Optimize repo insertion and add repo src id

augur/api/view/api.py

@@ -1,8 +1,8 @@
 from flask import request, jsonify, redirect, url_for, flash, current_app
 import re
 from flask_login import current_user, login_required
 from augur.application.db.models import Repo, RepoGroup, UserGroup, UserRepo
-from augur.tasks.frontend import add_org_repo_list, parse_org_and_repo_name, parse_org_name
+from augur.tasks.frontend import add_github_orgs_and_repos, parse_org_and_repo_name, parse_org_name, add_gitlab_repos
 from .utils import *
 from ..server import app
 from augur.application.db.session import DatabaseSession
@@ -14,33 +14,21 @@
         return redirect(url_for('static', filename="cache"))
     return redirect(url_for('static', filename="cache/" + toCacheFilename(file, False)))
 
-
-def add_existing_repo_to_group(session, user_id, group_name, repo_id):
-
-    logger.info("Adding existing repo to group")
-
-    group_id = UserGroup.convert_group_name_to_id(session, user_id, group_name)
-    if group_id is None:
-        return False
-
-    result = UserRepo.insert(session, repo_id, group_id)
-    if not result:
-        return False
 
 def add_existing_org_to_group(session, user_id, group_name, rg_id):
 
     logger.info("Adding existing org to group")
 
     group_id = UserGroup.convert_group_name_to_id(session, user_id, group_name)
     if group_id is None:
         return False
 
     repos = session.query(Repo).filter(Repo.repo_group_id == rg_id).all()
     logger.info("Length of repos in org: " + str(len(repos)))
     for repo in repos:
         result = UserRepo.insert(session, repo.repo_id, group_id)
         if not result:
             logger.info("Failed to add repo to group")
 
 
 
@@ -48,6 +36,8 @@
 @login_required
 def av_add_user_repo():
 
+    print("Adding user repos")
+
     urls = request.form.get('urls')
     group = request.form.get("group_name")
 
@@ -68,58 +58,51 @@
 
     invalid_urls = []
 
-    with DatabaseSession(logger, current_app.engine) as session:
-        for url in urls:  
-
-            # matches https://github.com/{org}/ or htts://github.com/{org}
-            if (org_name := Repo.parse_github_org_url(url)):
-                rg_obj = RepoGroup.get_by_name(session, org_name)
-                if rg_obj:
-                    # add the orgs repos to the group
-                    add_existing_org_to_group(session, current_user.user_id, group, rg_obj.repo_group_id)
-
-            # matches https://github.com/{org}/{repo}/ or htts://github.com/{org}/{repo}
-            elif Repo.parse_github_repo_url(url)[0]:
-                org_name, repo_name = Repo.parse_github_repo_url(url)
-                repo_git = f"https://github.com/{org_name}/{repo_name}"
-                repo_obj = Repo.get_by_repo_git(session, repo_git)
-                if repo_obj:
-                    add_existing_repo_to_group(session, current_user.user_id, group, repo_obj.repo_id)
-
-            # matches /{org}/{repo}/ or /{org}/{repo} or {org}/{repo}/ or {org}/{repo}
-            elif (match := parse_org_and_repo_name(url)):
-                org, repo = match.groups()
-                repo_git = f"https://github.com/{org}/{repo}"
-                repo_obj = Repo.get_by_repo_git(session, repo_git)
-                if repo_obj:
-                    add_existing_repo_to_group(session, current_user.user_id, group, repo_obj.repo_id)
+    orgs = []
+    repo_urls = []
+    gitlab_repo_urls = []
+    for url in urls:  
+
+        # matches https://github.com/{org}/ or htts://github.com/{org}
+        if (org_name := Repo.parse_github_org_url(url)):
+            orgs.append(org_name)
+
+        # matches https://github.com/{org}/{repo}/ or htts://github.com/{org}/{repo}
+        elif Repo.parse_github_repo_url(url)[0]:
+            repo_urls.append(url)
+
+        # matches /{org}/{repo}/ or /{org}/{repo} or {org}/{repo}/ or {org}/{repo}
+        elif (match := parse_org_and_repo_name(url)):
+            org, repo = match.groups()
+            repo_git = f"https://github.com/{org}/{repo}"
+            repo_urls.append(repo_git)
+
+        # matches /{org}/ or /{org} or {org}/ or {org}
+        elif (match := parse_org_name(url)):
+            org_name = match.group(1)
+            orgs.append(org_name)
+
+        # matches https://gitlab.com/{org}/{repo}/ or http://gitlab.com/{org}/{repo}
+        elif Repo.parse_gitlab_repo_url(url)[0]:
+
+            org_name, repo_name = Repo.parse_gitlab_repo_url(url)
+            repo_git = f"https://gitlab.com/{org_name}/{repo_name}"
 
-            # matches /{org}/ or /{org} or {org}/ or {org}
-            elif (match := parse_org_name(url)):
-                org_name = match.group(1)
-                rg_obj = RepoGroup.get_by_name(session, org_name)
-                logger.info(rg_obj)
-                if rg_obj:
-                    # add the orgs repos to the group
-                    add_existing_org_to_group(session, current_user.user_id, group, rg_obj.repo_group_id)
-
-            # matches https://gitlab.com/{org}/{repo}/ or http://gitlab.com/{org}/{repo}
-            elif Repo.parse_gitlab_repo_url(url)[0]:
-
-                org_name, repo_name = Repo.parse_gitlab_repo_url(url)
-                repo_git = f"https://gitlab.com/{org_name}/{repo_name}"
-
-                # TODO: gitlab ensure the whole repo git is inserted so it can be found here
-                repo_obj = Repo.get_by_repo_git(session, repo_git)
-                if repo_obj:
-                    add_existing_repo_to_group(session, current_user.user_id, group, repo_obj.repo_id)
-
-            else:
-                invalid_urls.append(url)
-
-    if urls:
-        urls = [url.lower() for url in urls]
-        add_org_repo_list.si(current_user.user_id, group, urls).apply_async()
+            gitlab_repo_urls.append(repo_git)
+        else:
+            invalid_urls.append(url)
+
+
+
+    if orgs or repo_urls:
+        repo_urls = [url.lower() for url in repo_urls]
+        orgs = [url.lower() for url in orgs]
+        flash(f"Adding repos: {repo_urls}")
+        flash(f"Adding orgs: {orgs}")
+        add_github_orgs_and_repos.si(current_user.user_id, group, orgs, repo_urls).apply_async()
+
+    if gitlab_repo_urls:
+        add_gitlab_repos(current_user.user_id, group, gitlab_repo_urls)
 
     flash("Adding repos and orgs in the background")
 
@@ -226,5 +209,5 @@
 """
 @app.route('/requests/report/wait/<id>')
 def wait_for_report_request(id):
     requestReports(id)
     return jsonify(report_requests[id])

augur/application/db/lib.py

@@ -9,7 +9,7 @@
 from psycopg2.errors import DeadlockDetected
 from typing import List, Any, Optional, Union
 
-from augur.application.db.models import Config, Repo, Commit, WorkerOauth, Issue, PullRequest, PullRequestReview, ContributorsAlias,UnresolvedCommitEmail, Contributor, CollectionStatus
+from augur.application.db.models import Config, Repo, Commit, WorkerOauth, Issue, PullRequest, PullRequestReview, ContributorsAlias,UnresolvedCommitEmail, Contributor, CollectionStatus, UserGroup, RepoGroup
 from augur.tasks.util.collection_state import CollectionState
 from augur.application.db import get_session, get_engine
 from augur.application.db.util import execute_session_query
@@ -17,7 +17,7 @@
 
 logger = logging.getLogger("db_lib")
 
 def convert_type_of_value(config_dict, logger=None):
 
 
     data_type = config_dict["type"]
@@ -144,6 +144,25 @@
 
         return repo
 
+def get_github_repo_by_src_id(src_id):
+
+    with get_session() as session:
+
+        query = session.query(Repo).filter(Repo.repo_src_id == src_id, Repo.repo_git.ilike(f'%https://github.com%'))
+        repo = execute_session_query(query, 'first')
+
+        return repo
+
+def get_gitlab_repo_by_src_id(src_id):
+
+    with get_session() as session:
+
+        query = session.query(Repo).filter(Repo.repo_src_id == src_id, Repo.repo_git.ilike(f'%https://gitlab.com%'))
+        repo = execute_session_query(query, 'first')
+
+        return repo
+
+
 def remove_working_commits_by_repo_id_and_hashes(repo_id, commit_hashes):
 
     remove_working_commits = s.sql.text("""DELETE FROM working_commits 
@@ -177,7 +196,7 @@
 
     try:
         working_commits = fetchall_data_from_sql_text(query)
     except:
         working_commits = []
 
     return working_commits
@@ -193,7 +212,7 @@
 
     try:
         missing_commit_hashes = fetchall_data_from_sql_text(fetch_missing_hashes_sql)
     except:
         missing_commit_hashes = []
 
     return missing_commit_hashes
@@ -213,7 +232,7 @@
         return session.query(CollectionStatus).filter(getattr(CollectionStatus,f"{collection_type}_status" ) == CollectionState.COLLECTING.value).count()
 
 
 def facade_bulk_insert_commits(logger, records):
 
     with get_session() as session:
 
@@ -553,3 +572,24 @@
     with get_session() as session:
         return session.query(Issue).filter(Issue.repo_id == repo_id, Issue.updated_at >= since).order_by(Issue.gh_issue_number).all()
 
+
+
+def get_group_by_name(user_id, group_name):
+
+
+    with get_session() as session:
+
+        try:
+            user_group = session.query(UserGroup).filter(UserGroup.user_id == user_id, UserGroup.name == group_name).one()
+        except s.orm.exc.NoResultFound:
+            return None
+
+        return user_group
+
+def get_repo_group_by_name(name):
+
+
+    with get_session() as session:
+
+        return  session.query(RepoGroup).filter(RepoGroup.rg_name == name).first()
+

augur/application/db/models/augur_data.py

@@ -870,6 +870,7 @@ class Repo(Base):
     data_collection_date = Column(
         TIMESTAMP(precision=0), server_default=text("CURRENT_TIMESTAMP")
     )
+    repo_src_id = Column(BigInteger)
 
     repo_group = relationship("RepoGroup", back_populates="repo")
     user_repo = relationship("UserRepo", back_populates="repo")
@@ -1064,7 +1065,7 @@ def parse_github_org_url(url):
         return result.groups()[0]
 
     @staticmethod
-    def insert_gitlab_repo(session, url: str, repo_group_id: int, tool_source):
+    def insert_gitlab_repo(session, url: str, repo_group_id: int, tool_source, repo_src_id):
         """Add a repo to the repo table.
 
         Args:
@@ -1098,7 +1099,8 @@ def insert_gitlab_repo(session, url: str, repo_group_id: int, tool_source):
             "repo_type": None,
             "tool_source": tool_source,
             "tool_version": "1.0",
-            "data_source": "Git"
+            "data_source": "Git",
+            "repo_src_id": repo_src_id
         }
 
         repo_unique = ["repo_git"]
@@ -1111,7 +1113,7 @@ def insert_gitlab_repo(session, url: str, repo_group_id: int, tool_source):
         return result[0]["repo_id"]
 
     @staticmethod
-    def insert_github_repo(session, url: str, repo_group_id: int, tool_source, repo_type):
+    def insert_github_repo(session, url: str, repo_group_id: int, tool_source, repo_type, repo_src_id):
         """Add a repo to the repo table.
 
         Args:
@@ -1146,7 +1148,8 @@ def insert_github_repo(session, url: str, repo_group_id: int, tool_source, repo_
             "repo_type": repo_type,
             "tool_source": tool_source,
             "tool_version": "1.0",
-            "data_source": "Git"
+            "data_source": "Git",
+            "repo_src_id": repo_src_id
         }
 
         repo_unique = ["repo_git"]

augur/application/schema/alembic/versions/30_add_repo_src_id.py

@@ -0,0 +1,25 @@
+"""Add repo src id
+
+Revision ID: 30
+Revises: 29
+Create Date: 2024-08-30
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '30'
+down_revision = '29'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('repo', sa.Column('repo_src_id', sa.BigInteger(), nullable=True), schema='augur_data')
+    op.create_unique_constraint('repo_src_id_unique', 'repo', ['repo_src_id'], schema='augur_data')
+
+
+def downgrade():
+    op.drop_constraint('repo_src_id_unique', 'repo', schema='augur_data', type_='unique')
+    op.drop_column('repo', 'repo_src_id', schema='augur_data')