From 57c3c808d41ea5a95113d2f9c1c02ca4d3f0335f Mon Sep 17 00:00:00 2001 From: Andrew Brain Date: Mon, 20 Jan 2025 21:15:06 -0600 Subject: [PATCH] speed up pr review message collection --- augur/tasks/github/pull_requests/tasks.py | 7 ++++++- augur/tasks/start_tasks.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/augur/tasks/github/pull_requests/tasks.py b/augur/tasks/github/pull_requests/tasks.py index c581ceb35..b65da7f4f 100644 --- a/augur/tasks/github/pull_requests/tasks.py +++ b/augur/tasks/github/pull_requests/tasks.py @@ -219,7 +219,7 @@ def process_pull_request_review_contributor(pr_review: dict, tool_source: str, t return pr_review_cntrb @celery.task(base=AugurSecondaryRepoCollectionTask) -def collect_pull_request_review_comments(repo_git: str) -> None: +def collect_pull_request_review_comments(repo_git: str, full_collection: bool) -> None: owner, repo = get_owner_repo(repo_git) @@ -230,6 +230,11 @@ def collect_pull_request_review_comments(repo_git: str) -> None: repo_id = get_repo_by_repo_git(repo_git).repo_id + if not full_collection: + # subtract 2 days to ensure all data is collected + core_data_last_collected = (get_core_data_last_collected(repo_id) - timedelta(days=2)).replace(tzinfo=timezone.utc) + review_msg_url += f"?since={core_data_last_collected.isoformat()}" + pr_reviews = get_pull_request_reviews_by_repo_id(repo_id) # maps the github pr_review id to the auto incrementing pk that augur stores as pr_review id diff --git a/augur/tasks/start_tasks.py b/augur/tasks/start_tasks.py index 8aa767ece..714ef2c73 100644 --- a/augur/tasks/start_tasks.py +++ b/augur/tasks/start_tasks.py @@ -120,7 +120,7 @@ def secondary_repo_collect_phase(repo_git, full_collection): repo_task_group = group( process_pull_request_files.si(repo_git, full_collection), process_pull_request_commits.si(repo_git, full_collection), - chain(collect_pull_request_reviews.si(repo_git, full_collection), collect_pull_request_review_comments.si(repo_git)), + chain(collect_pull_request_reviews.si(repo_git, full_collection), collect_pull_request_review_comments.si(repo_git, full_collection)), process_ossf_dependency_metrics.si(repo_git) )