From 529905d7ab7ac780704ade6122abe4c219e159e8 Mon Sep 17 00:00:00 2001 From: Ulincsys Date: Mon, 3 Jul 2023 18:47:03 -0500 Subject: [PATCH 001/182] Upgrade SQLAlchemy version - Switch to SQLAlchemy ORM syntax for repo filtering - Will replace the large SQL string in repo_load_controller.py - Current implementation located in augur/api/view/utils.py - Add clearer error message when page number is incorrect on repo view - Upgrade SQLAlchemy to v2.0.17 - Required for new syntax that the old version does not support - Specifically `ilike` and `regexp_replace` Signed-off-by: Ulincsys --- augur/api/view/routes.py | 16 ++++--- augur/api/view/utils.py | 70 +++++++++++++++++++++++++++--- augur/application/util.py | 3 -- augur/templates/repos-table.j2 | 4 ++ augur/util/repo_load_controller.py | 1 - setup.py | 2 +- 6 files changed, 80 insertions(+), 16 deletions(-) diff --git a/augur/api/view/routes.py b/augur/api/view/routes.py index 3e62dbfd9..889076fcb 100644 --- a/augur/api/view/routes.py +++ b/augur/api/view/routes.py @@ -74,14 +74,18 @@ def repo_table_view(): pagination_offset = config.get_value("frontend", "pagination_offset") if current_user.is_authenticated: - data = current_user.get_repos(page = page, sort = sorting, direction = direction, search=query)[0] - page_count = (current_user.get_repo_count(search = query)[0] or 0) // pagination_offset + data = load_repos_test(user = current_user, search = query, page = page, sort = sorting, direction = direction, source = "user") + page_count = load_repos_test(user = current_user, search = query, count = True, source = "user") + # data = current_user.get_repos(page = page, sort = sorting, direction = direction, search=query)[0] + # page_count = (current_user.get_repo_count(search = query)[0] or 0) // pagination_offset else: - data = get_all_repos(page = page, sort = sorting, direction = direction, search=query)[0] - page_count = (get_all_repos_count(search = query)[0] or 0) // pagination_offset + data = load_repos_test(search = query, page = page, sort = sorting, direction = direction) + page_count = load_repos_test(search = query, count = True) + # data = get_all_repos(page = page, sort = sorting, direction = direction, search=query)[0] + # page_count = (get_all_repos_count(search = query)[0] or 0) // pagination_offset - #if not cacheFileExists("repos.json"): - # return renderLoading("repos/views/table", query, "repos.json") + if not data.count(): + data = None # return renderRepos("table", query, data, sorting, rev, page, True) return render_module("repos-table", title="Repos", repos=data, query_key=query, activePage=page, pages=page_count, offset=pagination_offset, PS="repo_table_view", reverse = rev, sorting = sorting) diff --git a/augur/api/view/utils.py b/augur/api/view/utils.py index a8e6998f9..3d8f5bcff 100644 --- a/augur/api/view/utils.py +++ b/augur/api/view/utils.py @@ -2,10 +2,17 @@ from concurrent.futures import ThreadPoolExecutor from flask import render_template, flash, url_for, Flask from .init import * -from .server import User from ..server import app, db_session from augur.application.config import AugurConfig -import urllib.request, urllib.error, json, os, math, yaml, urllib3, time, logging, re +import urllib.request, urllib.error, json, os, math, yaml, urllib3, time, logging, re, math + +from augur.application.db.session import DatabaseSession +from augur.application.db.engine import DatabaseEngine +from augur.application.db.models import User, Repo, RepoGroup, UserGroup, UserRepo +from sqlalchemy import Column, Table, Integer, MetaData, or_, Label +from sqlalchemy.sql.operators import ilike_op, distinct_op +from sqlalchemy.sql.functions import coalesce +from augur.application.db.models.base import Base init_logging() @@ -141,9 +148,6 @@ def getSetting(key, section = "View"): loadSettings() -User.api = getSetting("serving") -User.logger = logger - version_check(settings) """ ---------------------------------------------------------------- @@ -456,3 +460,59 @@ def render_module(module, **args): def renderLoading(dest, query, request): cache_files_requested.append(request) return render_template('index.j2', body="loading", title="Loading", d=dest, query_key=query, api_url=getSetting('serving')) + +with DatabaseEngine() as engine: + augur_data_schema = MetaData(schema = "augur_data") + augur_data_schema.reflect(bind = engine, views = True) + + commits_materialized_view: Table = augur_data_schema.tables["augur_data.api_get_all_repos_commits"] + issues_materialized_view: Table = augur_data_schema.tables["augur_data.api_get_all_repos_issues"] + +""" ---------------------------------------------------------------- +""" +def load_repos_test(count = False, source = None, **kwargs): + columns: list[Label] = [ + Repo.repo_id.distinct().label("repo_id"), + Repo.description.label("description"), + Repo.repo_git.label("url"), + coalesce(commits_materialized_view.columns.commits_all_time, 0).label("commits_all_time"), + coalesce(issues_materialized_view.columns.issues_all_time, 0).label("issues_all_time"), + RepoGroup.rg_name.label("rg_name"), + Repo.repo_git.regexp_replace('.*github\.com\/[A-Za-z0-9 \- _]+\/([A-Za-z0-9 \- _ .]+)$', "\\1").label("repo_name"), + Repo.repo_git.regexp_replace('.*github\.com\/([A-Za-z0-9 \- _]+)\/[A-Za-z0-9 \- _ .]+$', "\\1").label("repo_owner"), + RepoGroup.repo_group_id.label("repo_group_id") + ] + + def get_colum_by_label(label: str)-> Label: + for column in columns: + if column.name == label: + return column + + repos = db_session.query(*columns)\ + .outerjoin(commits_materialized_view, Repo.repo_id == commits_materialized_view.columns.repo_id)\ + .outerjoin(issues_materialized_view, Repo.repo_id == issues_materialized_view.columns.repo_id)\ + .join(RepoGroup, Repo.repo_group_id == RepoGroup.repo_group_id) + + user: User = kwargs.get("user") + if user: + repos = repos.join(UserRepo, Repo.repo_id == UserRepo.repo_id)\ + .join(UserGroup, UserGroup.group_id == UserRepo.group_id)\ + .filter(UserGroup.user_id == user.user_id) + + search = kwargs.get("search") + qkey = kwargs.get("query_key") or ["repo_name", "repo_owner"] + if search: + if isinstance(qkey, list) and len(qkey) > 0: + repos = repos.filter(or_(ilike_op(get_colum_by_label(filter_column), f"%{search}%") for filter_column in qkey)) + else: + repos = repos.filter(ilike_op(get_colum_by_label(qkey), f"%{search}%")) + + page_size: int = kwargs.get("page_size") or 25 + if count: + c = repos.count() + return math.ceil(c / page_size) - 1 + + page: int = kwargs.get("page") or 0 + offset = page * page_size + + return repos.slice(offset, offset + page_size) \ No newline at end of file diff --git a/augur/application/util.py b/augur/application/util.py index 1915abdeb..03e591df9 100644 --- a/augur/application/util.py +++ b/augur/application/util.py @@ -25,6 +25,3 @@ def get_all_repos_count(**kwargs): result = controller.get_repo_count(source="all", **kwargs) return result - - - diff --git a/augur/templates/repos-table.j2 b/augur/templates/repos-table.j2 index a0593c73e..fce58eca0 100644 --- a/augur/templates/repos-table.j2 +++ b/augur/templates/repos-table.j2 @@ -86,6 +86,10 @@ {% elif query_key %}

Your search did not match any repositories

+{% elif pages %} +

Pagination Error

+

Something went wrong displaying the requested section of results.

+

Please go back and try again.

{% elif current_user.is_authenticated %}

No Repos Tracked

Add repos to your personal tracker in your profile page

diff --git a/augur/util/repo_load_controller.py b/augur/util/repo_load_controller.py index 3436167ae..d95a0817a 100644 --- a/augur/util/repo_load_controller.py +++ b/augur/util/repo_load_controller.py @@ -305,7 +305,6 @@ def generate_repo_query(self, source, count, **kwargs): query_args["page_size"] = page_size query_args["offset"] = offset - query_args["order_by"] = order_by return query, query_args, {"status": "success"} diff --git a/setup.py b/setup.py index 9177110f1..632828835 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ "alembic==1.8.1", # 1.8.1 "coloredlogs==15.0", # 15.0.1 "Beaker==1.11.0", # 1.11.0 - "SQLAlchemy==1.3.23", # 1.4.40 + "SQLAlchemy==2.0.17", # 1.4.40 "itsdangerous==2.0.1", # 2.1.2 'Jinja2~=3.0.3', "Flask==2.0.2", # 2.2.2 From f2620ce73cf4bff7aea38c5908eebfaa3a5d423a Mon Sep 17 00:00:00 2001 From: sgoggins Date: Sat, 8 Jul 2023 17:48:21 -0500 Subject: [PATCH 002/182] fixing order --- augur/tasks/start_tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/augur/tasks/start_tasks.py b/augur/tasks/start_tasks.py index 1918efbf8..c8f5dc59f 100644 --- a/augur/tasks/start_tasks.py +++ b/augur/tasks/start_tasks.py @@ -88,7 +88,8 @@ def primary_repo_collect_phase(repo_git): #facade_phase(logger,repo_git), collect_linux_badge_info.si(repo_git), collect_releases.si(repo_git), - grab_comitters.si(repo_git) + grab_comitters.si(repo_git), + process_ossf_dependency_metrics.si(repo_git) ) return repo_task_group @@ -102,7 +103,6 @@ def secondary_repo_collect_phase(repo_git): repo_task_group = group( process_pull_request_files.si(repo_git), process_pull_request_commits.si(repo_git), - process_ossf_dependency_metrics.si(repo_git), chain(collect_pull_request_reviews.si(repo_git), collect_pull_request_review_comments.si(repo_git)) ) From 005d3452b67c660fab35c66752e1a35f1fc56ae3 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Sun, 9 Jul 2023 00:05:02 +0000 Subject: [PATCH 003/182] email test Signed-off-by: Sean P. Goggins --- sendgridtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sendgridtest.py b/sendgridtest.py index 0aea1e25c..8e3139f2a 100644 --- a/sendgridtest.py +++ b/sendgridtest.py @@ -7,7 +7,7 @@ message = Mail( from_email='metrix@goggins.com', to_emails='gogginss@missouri.edu', - subject='Sending with Twilio SendGrid is Fun', + subject='Hi, Attached is your report of recommendations provided as a part of your DEI Badge! Contained are specific suggestions you may consider for making your project more open and welcoming to newcomers from diverse backgrounds. .... etc.', html_content='and easy to do anywhere, even with Python') try: sg = SendGridAPIClient(os.environ.get('SENDGRID_API_KEY')) From 8e2876f85228e48742fb84deb099f07de1e26eec Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Sun, 9 Jul 2023 00:06:19 +0000 Subject: [PATCH 004/182] test Signed-off-by: Sean P. Goggins --- sendgridtest.py | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 sendgridtest.py diff --git a/sendgridtest.py b/sendgridtest.py deleted file mode 100644 index 8e3139f2a..000000000 --- a/sendgridtest.py +++ /dev/null @@ -1,19 +0,0 @@ -# using SendGrid's Python Library -# https://github.com/sendgrid/sendgrid-python -import os -from sendgrid import SendGridAPIClient -from sendgrid.helpers.mail import Mail - -message = Mail( - from_email='metrix@goggins.com', - to_emails='gogginss@missouri.edu', - subject='Hi, Attached is your report of recommendations provided as a part of your DEI Badge! Contained are specific suggestions you may consider for making your project more open and welcoming to newcomers from diverse backgrounds. .... etc.', - html_content='and easy to do anywhere, even with Python') -try: - sg = SendGridAPIClient(os.environ.get('SENDGRID_API_KEY')) - response = sg.send(message) - print(response.status_code) - print(response.body) - print(response.headers) -except Exception as e: - print(e.message) From 0954a837ac1ea4bec7074a91868f22c05e216f36 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Sun, 9 Jul 2023 00:08:21 +0000 Subject: [PATCH 005/182] update version Signed-off-by: Sean P. Goggins --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 811157100..ce06f3462 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Augur NEW Release v0.51.1 +# Augur NEW Release v0.51.2 [![first-timers-only](https://img.shields.io/badge/first--timers--only-friendly-blue.svg?style=flat-square)](https://www.firsttimersonly.com/) We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy of tagging issues for first timers only, and walking one newcomer through the resolution process weekly. [You can find these issues tagged with "first timers only" on our issues list.](https://github.com/chaoss/augur/labels/first-timers-only). @@ -8,7 +8,7 @@ ### [If you want to jump right in, updated docker build/compose and bare metal installation instructions are available here](docs/new-install.md) -Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.51.1 +Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.51.2 - The `main` branch is a stable version of our new architecture, which features: - Dramatic improvement in the speed of large scale data collection (100,000+ repos). All data is obtained for 100k+ repos within 2 weeks. - A new job management architecture that uses Celery and Redis to manage queues, and enables users to run a Flower job monitoring dashboard From b8e08923a70cb87a7ff64d4a10cde5eca3e654aa Mon Sep 17 00:00:00 2001 From: sgoggins Date: Thu, 13 Jul 2023 10:55:43 -0500 Subject: [PATCH 006/182] updating materialized view refresh code. --- augur/tasks/db/refresh_materialized_views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/augur/tasks/db/refresh_materialized_views.py b/augur/tasks/db/refresh_materialized_views.py index 53b29ddbd..5c592645f 100644 --- a/augur/tasks/db/refresh_materialized_views.py +++ b/augur/tasks/db/refresh_materialized_views.py @@ -18,12 +18,14 @@ def refresh_materialized_views(): refresh_view_query = s.sql.text(""" REFRESH MATERIALIZED VIEW augur_data.api_get_all_repos_issues with data; REFRESH MATERIALIZED VIEW augur_data.explorer_commits_and_committers_daily_count with data; + REFRESH MATERIALIZED VIEW augur_data.api_get_all_repos_prs with data; REFRESH MATERIALIZED VIEW augur_data.api_get_all_repos_commits with data; REFRESH MATERIALIZED VIEW augur_data.augur_new_contributors with data; REFRESH MATERIALIZED VIEW augur_data.explorer_contributor_actions with data; REFRESH MATERIALIZED VIEW augur_data.explorer_libyear_all with data; REFRESH MATERIALIZED VIEW augur_data.explorer_libyear_detail with data; REFRESH MATERIALIZED VIEW augur_data.explorer_new_contributors with data; + REFRESH MATERIALIZED VIEW augur_data.explorer_entry_list with data; REFRESH MATERIALIZED VIEW augur_data.explorer_libyear_summary with data; """) From b81a07e3906b66434dadbfbff1b8fb8a63e2083d Mon Sep 17 00:00:00 2001 From: sgoggins Date: Thu, 13 Jul 2023 11:38:59 -0500 Subject: [PATCH 007/182] materialized view refresh script: Started. Needs to be parameterized to use db.config.json --- scripts/control/refresh-matviews.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 scripts/control/refresh-matviews.sh diff --git a/scripts/control/refresh-matviews.sh b/scripts/control/refresh-matviews.sh new file mode 100644 index 000000000..576466b2d --- /dev/null +++ b/scripts/control/refresh-matviews.sh @@ -0,0 +1,11 @@ +#!/bin/sh +psql -U augur -h localhost -p 5432 -d padres -c 'REFRESH MATERIALIZED VIEW augur_data.api_get_all_repos_issues with data;' +psql -U augur -h localhost -p 5432 -d padres -c 'REFRESH MATERIALIZED VIEW augur_data.explorer_commits_and_committers_daily_count with data;' +psql -U augur -h localhost -p 5432 -d padres -c 'REFRESH MATERIALIZED VIEW augur_data.api_get_all_repo_prs with data;' +psql -U augur -h localhost -p 5432 -d padres -c 'REFRESH MATERIALIZED VIEW augur_data.api_get_all_repos_commits with data;' +psql -U augur -h localhost -p 5432 -d padres -c 'REFRESH MATERIALIZED VIEW augur_data.augur_new_contributors with data;' +psql -U augur -h localhost -p 5432 -d padres -c 'REFRESH MATERIALIZED VIEW augur_data.explorer_contributor_actions with data;' +psql -U augur -h localhost -p 5432 -d padres -c 'REFRESH MATERIALIZED VIEW augur_data.explorer_libyear_all with data;' +psql -U augur -h localhost -p 5432 -d padres -c 'REFRESH MATERIALIZED VIEW augur_data.explorer_libyear_detail with data;' +psql -U augur -h localhost -p 5432 -d padres -c 'REFRESH MATERIALIZED VIEW augur_data.explorer_new_contributors with data;' +psql -U augur -h localhost -p 5432 -d padres -c 'REFRESH MATERIALIZED VIEW augur_data.explorer_entry_list with data;' \ No newline at end of file From 7df2d08162384cc527894be08173f929f75757ee Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Tue, 25 Jul 2023 17:39:59 -0500 Subject: [PATCH 008/182] fix fetchall_data_from_sql_text for sqlalchemy 2.x Signed-off-by: Isaac Milarsky --- augur/application/db/models/augur_data.py | 10 ++++++---- augur/application/db/models/augur_operations.py | 3 ++- augur/application/db/session.py | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/augur/application/db/models/augur_data.py b/augur/application/db/models/augur_data.py index 676a71dee..ee9f561a2 100644 --- a/augur/application/db/models/augur_data.py +++ b/augur/application/db/models/augur_data.py @@ -1195,10 +1195,12 @@ class Commit(Base): "Contributor", primaryjoin="Commit.cmt_author_platform_username == Contributor.cntrb_login", ) - contributor1 = relationship( - "Contributor", - primaryjoin="Commit.cmt_author_platform_username == Contributor.cntrb_login", - ) + + # Redundant? + #contributor1 = relationship( + # "Contributor", + # primaryjoin="Commit.cmt_author_platform_username == Contributor.cntrb_login", + #) repo = relationship("Repo") diff --git a/augur/application/db/models/augur_operations.py b/augur/application/db/models/augur_operations.py index e2100196d..25bb05eda 100644 --- a/augur/application/db/models/augur_operations.py +++ b/augur/application/db/models/augur_operations.py @@ -275,7 +275,8 @@ class User(Base): {"schema": "augur_operations"} ) - groups = relationship("UserGroup") + # Redundant. + #groups = relationship("UserGroup") tokens = relationship("UserSessionToken") applications = relationship("ClientApplication") diff --git a/augur/application/db/session.py b/augur/application/db/session.py index 2212c1fdc..7e9a6da54 100644 --- a/augur/application/db/session.py +++ b/augur/application/db/session.py @@ -95,8 +95,8 @@ def fetchall_data_from_sql_text(self,sql_text): with self.engine.connect() as connection: - result = connection.execute(sql_text).fetchall() - return [dict(zip(row.keys(), row)) for row in result] + result = connection.execute(sql_text) + return [dict(row) for row in result.mappings()] def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[str], return_columns: Optional[List[str]] = None, string_fields: Optional[List[str]] = None, on_conflict_update:bool = True) -> Optional[List[dict]]: From 6d01398d8c60c9e3baaa1297dc352367e6afddf2 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Tue, 25 Jul 2023 17:50:08 -0500 Subject: [PATCH 009/182] new method of converting row to dict in sqlalchemy 2.x Signed-off-by: Isaac Milarsky --- augur/tasks/github/facade_github/tasks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/augur/tasks/github/facade_github/tasks.py b/augur/tasks/github/facade_github/tasks.py index 04ccc16e1..f0e8845fb 100644 --- a/augur/tasks/github/facade_github/tasks.py +++ b/augur/tasks/github/facade_github/tasks.py @@ -251,8 +251,8 @@ def insert_facade_contributors(repo_id): """).bindparams(repo_id=repo_id) #Execute statement with session. - result = manifest.augur_db.execute_sql(new_contrib_sql).fetchall() - new_contribs = [dict(zip(row.keys(), row)) for row in result] + result = manifest.augur_db.execute_sql(new_contrib_sql) + new_contribs = [dict(row) for row in result.mappings()] #print(new_contribs) @@ -302,8 +302,8 @@ def insert_facade_contributors(repo_id): #existing_cntrb_emails = json.loads(pd.read_sql(resolve_email_to_cntrb_id_sql, self.db, params={ # 'repo_id': repo_id}).to_json(orient="records")) - result = session.execute_sql(resolve_email_to_cntrb_id_sql).fetchall() - existing_cntrb_emails = [dict(zip(row.keys(), row)) for row in result] + result = session.execute_sql(resolve_email_to_cntrb_id_sql) + existing_cntrb_emails = [dict(row) for row in result.mappings()] print(existing_cntrb_emails) link_commits_to_contributor(session,list(existing_cntrb_emails)) From fb5091ef053c2549722043d51bda15a1698dfde2 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Mon, 7 Aug 2023 11:27:49 -0500 Subject: [PATCH 010/182] deal with redundant relationship definition Signed-off-by: Isaac Milarsky --- augur/application/db/models/augur_operations.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/augur/application/db/models/augur_operations.py b/augur/application/db/models/augur_operations.py index e1228d8b5..8507133b0 100644 --- a/augur/application/db/models/augur_operations.py +++ b/augur/application/db/models/augur_operations.py @@ -740,7 +740,8 @@ class UserRepo(Base): ForeignKey("augur_data.repo.repo_id", name="user_repo_user_id_fkey"), primary_key=True, nullable=False ) - repo = relationship("Repo") + #made redundant by user_repo relationship in Repo orm class. + #repo = relationship("Repo") group = relationship("UserGroup") @staticmethod From 20e2f26d82e726080d3006c4d531153bd59f7683 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Mon, 7 Aug 2023 11:39:44 -0500 Subject: [PATCH 011/182] patch contributor breadth worker to use 2.x version of sqlalchemy execute() Signed-off-by: Isaac Milarsky --- .../contributor_breadth_worker.py | 7 +++++-- augur/tasks/db/refresh_materialized_views.py | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py b/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py index 44a6761cf..9fdcf434a 100644 --- a/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py +++ b/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py @@ -33,7 +33,8 @@ def contributor_breadth_model() -> None: WHERE gh_login IS NOT NULL """) - result = engine.execute(cntrb_login_query) + with engine.connect() as connection: + result = connection.execute(cntrb_login_query) current_cntrb_logins = [dict(row) for row in result] @@ -45,7 +46,9 @@ def contributor_breadth_model() -> None: GROUP BY c.gh_login; """) - cntrb_newest_events_list = engine.execute(cntrb_newest_events_query) + with engine.connect() as connection: + cntrb_newest_events_list = connection.execute(cntrb_newest_events_query) + cntrb_newest_events_list = [dict(row) for row in cntrb_newest_events_list] cntrb_newest_events_map = {} diff --git a/augur/tasks/db/refresh_materialized_views.py b/augur/tasks/db/refresh_materialized_views.py index 5c592645f..74f636192 100644 --- a/augur/tasks/db/refresh_materialized_views.py +++ b/augur/tasks/db/refresh_materialized_views.py @@ -15,10 +15,12 @@ def refresh_materialized_views(): logger = logging.getLogger(refresh_materialized_views.__name__) + #Removed to make it work: + # REFRESH MATERIALIZED VIEW augur_data.api_get_all_repos_prs with data; refresh_view_query = s.sql.text(""" REFRESH MATERIALIZED VIEW augur_data.api_get_all_repos_issues with data; REFRESH MATERIALIZED VIEW augur_data.explorer_commits_and_committers_daily_count with data; - REFRESH MATERIALIZED VIEW augur_data.api_get_all_repos_prs with data; + REFRESH MATERIALIZED VIEW augur_data.api_get_all_repos_commits with data; REFRESH MATERIALIZED VIEW augur_data.augur_new_contributors with data; REFRESH MATERIALIZED VIEW augur_data.explorer_contributor_actions with data; From fd2b4c5b226ee526838e530db023b077bd547203 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Mon, 7 Aug 2023 13:34:25 -0500 Subject: [PATCH 012/182] fix insert_data to work with sqlalchemy 2.x by using new begin method with connection object. This allows a commit to be implicit within the resulting context manager and emulates the functionality of the now removed autocommit Signed-off-by: Isaac Milarsky --- augur/application/db/session.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/augur/application/db/session.py b/augur/application/db/session.py index 7e9a6da54..bcc988991 100644 --- a/augur/application/db/session.py +++ b/augur/application/db/session.py @@ -174,7 +174,9 @@ def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[s while attempts < 10: try: - with EngineConnection(self.engine) as connection: + #begin keyword is needed for sqlalchemy 2.x + #this is because autocommit support was removed in 2.0 + with EngineConnection(self.engine).begin() as connection: connection.execute(stmnt) break except OperationalError as e: From 9e4445cda42dff9c2a4b9cfac9088b8402b8d136 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Mon, 7 Aug 2023 14:34:11 -0500 Subject: [PATCH 013/182] fix missing argument in splitting insert_data calls in recursion in insert_data method. fix missing .begin() call in block to deal with return columns. fix dict conversion for list of rows in sqlalchemy 2.x in insert_data and contributor breadth worker Signed-off-by: Isaac Milarsky --- augur/application/db/session.py | 17 ++++++++++------- .../contributor_breadth_worker.py | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/augur/application/db/session.py b/augur/application/db/session.py index bcc988991..23cf8abc8 100644 --- a/augur/application/db/session.py +++ b/augur/application/db/session.py @@ -176,7 +176,7 @@ def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[s try: #begin keyword is needed for sqlalchemy 2.x #this is because autocommit support was removed in 2.0 - with EngineConnection(self.engine).begin() as connection: + with self.engine.begin() as connection: connection.execute(stmnt) break except OperationalError as e: @@ -193,14 +193,15 @@ def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[s raise e except Exception as e: + #self.logger.info(e) if(len(data) == 1): raise e else: first_half = data[:len(data)//2] second_half = data[len(data)//2:] - self.insert_data(first_half, natural_keys, return_columns, string_fields, on_conflict_update) - self.insert_data(second_half, natural_keys, return_columns, string_fields, on_conflict_update) + self.insert_data(first_half, table,natural_keys, return_columns, string_fields, on_conflict_update) + self.insert_data(second_half,table, natural_keys, return_columns, string_fields, on_conflict_update) else: self.logger.error("Unable to insert data in 10 attempts") @@ -215,7 +216,7 @@ def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[s # othewise it gets the requested return columns and returns them as a list of dicts while attempts < 10: try: - with EngineConnection(self.engine) as connection: + with self.engine.begin() as connection: return_data_tuples = connection.execute(stmnt).fetchall() break except OperationalError as e: @@ -246,9 +247,11 @@ def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[s if deadlock_detected is True: self.logger.error("Made it through even though Deadlock was detected") - return_data = [] - for data_tuple in return_data_tuples: - return_data.append(dict(data_tuple)) + return_data = [dict(row) for row in return_data_tuples.mappings()] + + #no longer working in sqlalchemy 2.x + #for data_tuple in return_data_tuples: + # return_data.append(dict(data_tuple)) # using on confilict do nothing does not return the # present values so this does gets the return values diff --git a/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py b/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py index 9fdcf434a..19053b2f0 100644 --- a/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py +++ b/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py @@ -36,7 +36,7 @@ def contributor_breadth_model() -> None: with engine.connect() as connection: result = connection.execute(cntrb_login_query) - current_cntrb_logins = [dict(row) for row in result] + current_cntrb_logins = [dict(row) for row in result.mappings()] cntrb_newest_events_query = s.sql.text(""" From a39eb0b7edba1bc8df8427bdc7c184d67b29e7b3 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Mon, 7 Aug 2023 14:45:00 -0500 Subject: [PATCH 014/182] update mappings syntax Signed-off-by: Isaac Milarsky --- augur/application/db/session.py | 2 +- .../contributor_breadth_worker/contributor_breadth_worker.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/augur/application/db/session.py b/augur/application/db/session.py index 23cf8abc8..cd5788df9 100644 --- a/augur/application/db/session.py +++ b/augur/application/db/session.py @@ -217,7 +217,7 @@ def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[s while attempts < 10: try: with self.engine.begin() as connection: - return_data_tuples = connection.execute(stmnt).fetchall() + return_data_tuples = connection.execute(stmnt) break except OperationalError as e: if isinstance(e.orig, DeadlockDetected): diff --git a/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py b/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py index 19053b2f0..bca949df8 100644 --- a/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py +++ b/augur/tasks/data_analysis/contributor_breadth_worker/contributor_breadth_worker.py @@ -49,7 +49,7 @@ def contributor_breadth_model() -> None: with engine.connect() as connection: cntrb_newest_events_list = connection.execute(cntrb_newest_events_query) - cntrb_newest_events_list = [dict(row) for row in cntrb_newest_events_list] + cntrb_newest_events_list = [dict(row) for row in cntrb_newest_events_list.mappings()] cntrb_newest_events_map = {} for cntrb_event in cntrb_newest_events_list: From f9ad429c00d09ec7c15b54afa981611318bdea93 Mon Sep 17 00:00:00 2001 From: Seltyk Date: Mon, 7 Aug 2023 16:17:09 -0400 Subject: [PATCH 015/182] Add endpoint to get repo by ID Also wrote API documentation for it using an example 3scale repo in my testing Signed-off-by: Seltyk --- augur/api/routes/util.py | 62 +++++++++++++--- docs/source/rest-api/spec.yml | 129 ++++++++++++++++++++++------------ 2 files changed, 137 insertions(+), 54 deletions(-) diff --git a/augur/api/routes/util.py b/augur/api/routes/util.py index cd6a8ad3b..8a3261855 100644 --- a/augur/api/routes/util.py +++ b/augur/api/routes/util.py @@ -1,10 +1,11 @@ #SPDX-License-Identifier: MIT +from augur.api.routes import AUGUR_API_VERSION +from ..server import app, engine import base64 import sqlalchemy as s import pandas as pd import json from flask import Response -import logging from augur.application.db.session import DatabaseSession from augur.application.logs import AugurLogger @@ -12,10 +13,6 @@ logger = AugurLogger("augur").get_logger() -from augur.api.routes import AUGUR_API_VERSION -from ..server import app, engine - - @app.route('/{}/repo-groups'.format(AUGUR_API_VERSION)) def get_all_repo_groups(): #TODO: make this name automatic - wrapper? repoGroupsSQL = s.sql.text(""" @@ -52,9 +49,9 @@ def get_all_repos(): (select * from api_get_all_repos_issues) b on repo.repo_id = b.repo_id - left outer join - (select * from api_get_all_repo_prs) c - on repo.repo_id=c.repo_id + left outer join + (select * from api_get_all_repo_prs) c + on repo.repo_id=c.repo_id JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id order by repo_name """) @@ -91,9 +88,9 @@ def get_repos_in_repo_group(repo_group_id): (select * from api_get_all_repos_issues) b on repo.repo_id = b.repo_id - left outer join - (select * from api_get_all_repo_prs) c - on repo.repo_id=c.repo_id + left outer join + (select * from api_get_all_repo_prs) c + on repo.repo_id=c.repo_id JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id WHERE repo_groups.repo_group_id = :repo_group_id @@ -106,6 +103,49 @@ def get_repos_in_repo_group(repo_group_id): status=200, mimetype="application/json") +@app.route('/{}/repos/'.format(AUGUR_API_VERSION)) +def get_repo_by_id(repo_id: int) -> Response: + repo_by_id_SQL = s.sql.text(""" + SELECT + repo.repo_id, + repo.repo_name, + repo.description, + repo.repo_git AS url, + a.commits_all_time, + b.issues_all_time, + c.pull_requests_all_time, + rg_name, + repo.repo_group_id + FROM + repo + LEFT OUTER JOIN + (SELECT * FROM api_get_all_repos_commits) a + ON repo.repo_id = a.repo_id + LEFT OUTER JOIN + (SELECT * FROM api_get_all_repos_issues) b + ON repo.repo_id = b.repo_id + LEFT OUTER JOIN + (SELECT * FROM api_get_all_repo_prs) c + ON repo.repo_id = c.repo_id + JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id + WHERE + repo.repo_id = :id + """) + + results = pd.read_sql(repo_by_id_SQL, engine, params={"id": repo_id}) + results["url"] = results["url"].apply(lambda datum: datum.split("//")[1]) # cut "https://" off the URL + results["base64_url"] = [base64.b64encode(results.at[i, "url"].encode()) for i in results.index] + data = results.to_json(orient="records", date_format="iso", date_unit="ms") + + if not data or data == "[]": + return Response(response='{"status": "Repository ' + str(repo_id) + ' does not exist"}', + status=400, + mimetype="application/json") + + return Response(response=data[1:-1], # cut off brackets at each end, turns list of length 1 into single value + status=200, + mimetype="application/json") + @app.route('/{}/owner//repo/'.format(AUGUR_API_VERSION)) def get_repo_by_git_name(owner, repo): diff --git a/docs/source/rest-api/spec.yml b/docs/source/rest-api/spec.yml index ae583dbbb..50643d916 100644 --- a/docs/source/rest-api/spec.yml +++ b/docs/source/rest-api/spec.yml @@ -96,6 +96,49 @@ paths: type: array tags: - utility + /repos/:id: + get: + description: Get a downloaded repo by its ID in Augur. + operationId: Get Repo By ID + responses: + '200': + description: OK + scema: + items: + properties: + base64_url: + description: 'Base64 encode of the full URL. Example Z2l0aHViLmNvbS8zc2NhbGUvM3NjYWxlLW9wZXJhdG9yLW1ldGFkYXRh' + type: string + description: + description: 'Repository description. Example: null' + type: string + commits_all_time: + description: 'How many commits have been made to this respository. Example: 24' + type: integer + issues_all_time: + description: 'How many issues have been raised on this respository. Example: 1' + type: integer + pull_requests_all_time: + description: 'How many pull requests have been made to this reqpository. Example: 7' + type: integer + repo_id: + description: 'Repository ID, should match provided URL parameter. Example: 25551' + type: integer + repo_name: + description: 'Name of the provided rpository. Example: 3scale-operator-metadata' + type: string + rg_name: + description: 'Name of the repository group containing this repo. Example: 3scale' + type: string + repo_group_id: + description: 'ID of the repository group containing this repo. Example: 25431' + type: integer + url: + description: 'URL of this repository, sans leading protocol. Example: github.com/3scale/3scale-operator-metadata' + type: string + type: object + tags: + - utility /owner/:owner/repo/:repo: get: description: Get the repo_group_id and repo_id of a particular repo. @@ -284,12 +327,12 @@ paths: tags: - utility #risk endpoints - /metadata/repo_info: - get: - description: 'Returns the metadata about all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the default branch name, repository license file, forks, stars, watchers, and committers. Also includes metadata about current repository issue and pull request status and counts.' + /metadata/repo_info: + get: + description: 'Returns the metadata about all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the default branch name, repository license file, forks, stars, watchers, and committers. Also includes metadata about current repository issue and pull request status and counts.' externalDocs: description: CHAOSS Metric Definition - url: https://github.com/chaoss/wg-risk/blob/main/focus-areas/business-risk.md + url: https://github.com/chaoss/wg-risk/blob/main/focus-areas/business-risk.md operationId: Activity Metadata (Repo) responses: '200': @@ -315,45 +358,45 @@ paths: fork_count: description: 'Example: 554' type: integer - watchers_count: + watchers_count: description: 'Example: 424' type: integer - stars_count: + stars_count: description: 'Example: 443' type: integer - commits_count: + commits_count: description: '4434' - type: integer - committers_count: + type: integer + committers_count: description: 'Example: 42' type: integer - open_issues: + open_issues: description: 'Example: 7' - type: integer - issues_count: + type: integer + issues_count: description: 'Example: 23332' type: integer - issues_closed: + issues_closed: description: 'Example: 23322' type: integer - pull_request_count: + pull_request_count: description: 'Example: 19445' type: integer - pull_requests_open: + pull_requests_open: description: 'Example: 10' type: integer - pull_requests_closed: + pull_requests_closed: description: 'Example: 19435' type: integer - pull_requests_merged: + pull_requests_merged: description: 'Example: 17473' type: integer type: array tags: - risk - /metadata/contributions_count: - get: - description: 'Returns a list of repositories contributed to by all the contributors in an Augur Instance: INCLUDING all repositories on a platform, *not* merely those repositories in the Augur Instance. Numerical totals represent total CONTRIBUTIONS.' + /metadata/contributions_count: + get: + description: 'Returns a list of repositories contributed to by all the contributors in an Augur Instance: INCLUDING all repositories on a platform, *not* merely those repositories in the Augur Instance. Numerical totals represent total CONTRIBUTIONS.' externalDocs: description: CHAOSS Metric Definition url: https://github.com/chaoss/wg-risk/blob/main/focus-areas/business-risk.md @@ -373,9 +416,9 @@ paths: type: array tags: - risk - /metadata/contributors_count: - get: - description: 'Returns a list of repositories contributed to by all the contributors in an Augur Instance: INCLUDING all repositories on a platform, *not* merely those repositories in the Augur Instance. Numerical totals represent total CONTRIBUTORS.' + /metadata/contributors_count: + get: + description: 'Returns a list of repositories contributed to by all the contributors in an Augur Instance: INCLUDING all repositories on a platform, *not* merely those repositories in the Augur Instance. Numerical totals represent total CONTRIBUTORS.' externalDocs: description: CHAOSS Metric Definition url: https://github.com/chaoss/wg-risk/blob/main/focus-areas/business-risk.md @@ -5144,9 +5187,9 @@ paths: type: object tags: - visualizations - /complexity/project_lines: - get: - description: 'Returns project line data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the total and average number of lines in the project repository.' + /complexity/project_lines: + get: + description: 'Returns project line data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the total and average number of lines in the project repository.' operationId: Total Lines (repo) responses: '200': @@ -5172,9 +5215,9 @@ paths: type: array tags: - complexity - /complexity/project_file_complexity: - get: - description: 'Returns project file complexity data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the total and average file complexity of the project repository.' + /complexity/project_file_complexity: + get: + description: 'Returns project file complexity data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the total and average file complexity of the project repository.' operationId: File Complexity (repo) responses: '200': @@ -5200,9 +5243,9 @@ paths: type: array tags: - complexity - /complexity/project_blank_lines: - get: - description: 'Returns project blank line data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the total and average number of blank lines in the project repository.' + /complexity/project_blank_lines: + get: + description: 'Returns project blank line data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the total and average number of blank lines in the project repository.' operationId: Total Blank Lines (repo) responses: '200': @@ -5228,9 +5271,9 @@ paths: type: array tags: - complexity - /complexity/project_comment_lines: - get: - description: 'Returns project comment line data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the total and average number of comment lines in the project repository.' + /complexity/project_comment_lines: + get: + description: 'Returns project comment line data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the total and average number of comment lines in the project repository.' operationId: Total Comment Lines (repo) responses: '200': @@ -5256,9 +5299,9 @@ paths: type: array tags: - complexity - /complexity/project_files: - get: - description: 'Returns project file data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the total number of files in the project repository.' + /complexity/project_files: + get: + description: 'Returns project file data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the total number of files in the project repository.' operationId: Total Files (repo) responses: '200': @@ -5281,9 +5324,9 @@ paths: type: array tags: - complexity - /complexity/project_languages: - get: - description: 'Returns project language data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the lines and files of a language in a repository.' + /complexity/project_languages: + get: + description: 'Returns project language data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the lines and files of a language in a repository.' operationId: Project Languages (repo) responses: '200': @@ -5317,7 +5360,7 @@ paths: description: 'The number of messages exchanged for a repository group over a specified period.' externalDocs: description: CHAOSS Metric Definition - url: + url: operationId: Repository Messages (Repo Group) parameters: - description: Repository Group ID @@ -5633,4 +5676,4 @@ paths: type: string enum: ["Missing argument"] tags: - - DEI Badging \ No newline at end of file + - DEI Badging From c390c93042e5075b40a4efa33f47850043384ca1 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Mon, 7 Aug 2023 15:57:43 -0500 Subject: [PATCH 016/182] update alembic scripts to work with sqlalchemy 2.x Signed-off-by: Isaac Milarsky --- augur/application/db/session.py | 4 ++-- .../schema/alembic/versions/1_augur_new_changes.py | 10 ++++++---- .../schema/alembic/versions/3_oauth_and_user_groups.py | 4 ++-- .../versions/9_add_repo_group_for_frontend_repos.py | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/augur/application/db/session.py b/augur/application/db/session.py index cd5788df9..f1d1e64dd 100644 --- a/augur/application/db/session.py +++ b/augur/application/db/session.py @@ -85,7 +85,7 @@ def __del__(self): def execute_sql(self, sql_text): - with self.engine.connect() as connection: + with self.engine.begin() as connection: return_data = connection.execute(sql_text) @@ -93,7 +93,7 @@ def execute_sql(self, sql_text): def fetchall_data_from_sql_text(self,sql_text): - with self.engine.connect() as connection: + with self.engine.begin() as connection: result = connection.execute(sql_text) return [dict(row) for row in result.mappings()] diff --git a/augur/application/schema/alembic/versions/1_augur_new_changes.py b/augur/application/schema/alembic/versions/1_augur_new_changes.py index 0be3780a3..2e8440294 100644 --- a/augur/application/schema/alembic/versions/1_augur_new_changes.py +++ b/augur/application/schema/alembic/versions/1_augur_new_changes.py @@ -300,8 +300,9 @@ def change_cntrb_id_to_uuid_5(upgrade=True): """ INSERT INTO "augur_data"."contributors"("cntrb_id", "cntrb_login", "cntrb_email", "cntrb_company", "cntrb_created_at", "cntrb_type", "cntrb_fake", "cntrb_deleted", "cntrb_long", "cntrb_lat", "cntrb_country_code", "cntrb_state", "cntrb_city", "cntrb_location", "cntrb_canonical", "gh_user_id", "gh_login", "gh_url", "gh_html_url", "gh_node_id", "gh_avatar_url", "gh_gravatar_id", "gh_followers_url", "gh_following_url", "gh_gists_url", "gh_starred_url", "gh_subscriptions_url", "gh_organizations_url", "gh_repos_url", "gh_events_url", "gh_received_events_url", "gh_type", "gh_site_admin", "tool_source", "tool_version", "data_source", "data_collection_date") VALUES (:cntrb_uuid, 'not-provided', NULL, NULL, '2019-06-13 11:33:39', NULL, 0, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 1, 'nobody', 'http://fake.me', 'http://fake.me', 'x', 'http://fake.me', NULL, 'http://fake.me', 'http://fake.me', 'http://fake.me', 'http://fake.me', 'http://fake.me', 'http://fake.me', 'http://fake.me', 'http://fake.me', NULL, NULL, NULL, NULL, NULL, NULL, '2019-06-13 16:35:25'); """ - ), - cntrb_uuid=UnresolvableUUID().to_UUID() + ).bindparams( + cntrb_uuid=UnresolvableUUID().to_UUID() + ) ) conn.execute( @@ -309,8 +310,9 @@ def change_cntrb_id_to_uuid_5(upgrade=True): """ INSERT INTO "augur_data"."contributors" ("cntrb_id", "cntrb_login", "cntrb_email", "cntrb_full_name", "cntrb_company", "cntrb_created_at", "cntrb_type", "cntrb_fake", "cntrb_deleted", "cntrb_long", "cntrb_lat", "cntrb_country_code", "cntrb_state", "cntrb_city", "cntrb_location", "cntrb_canonical", "cntrb_last_used", "gh_user_id", "gh_login", "gh_url", "gh_html_url", "gh_node_id", "gh_avatar_url", "gh_gravatar_id", "gh_followers_url", "gh_following_url", "gh_gists_url", "gh_starred_url", "gh_subscriptions_url", "gh_organizations_url", "gh_repos_url", "gh_events_url", "gh_received_events_url", "gh_type", "gh_site_admin", "gl_web_url", "gl_avatar_url", "gl_state", "gl_username", "gl_full_name", "gl_id", "tool_source", "tool_version", "data_source", "data_collection_date") VALUES (:cntrb_uuid, 'nan', 'kannayoshihiro@gmail.com', 'KANNA Yoshihiro', 'UTMC', '2009-04-17 12:43:58', NULL, 0, 0, NULL, NULL, NULL, NULL, NULL, NULL, 'kannayoshihiro@gmail.com', '2021-01-28 21:56:10-06', 74832, 'nan', 'https://api.github.com/users/nan', 'https://github.com/nan', 'MDQ6VXNlcjc0ODMy', 'https://avatars.githubusercontent.com/u/74832?v=4', '', 'https://api.github.com/users/nan/followers', 'https://api.github.com/users/nan/following{/other_user}', 'https://api.github.com/users/nan/gists{/gist_id}', 'https://api.github.com/users/nan/starred{/owner}{/repo}', 'https://api.github.com/users/nan/subscriptions', 'https://api.github.com/users/nan/orgs', 'https://api.github.com/users/nan/repos', 'https://api.github.com/users/nan/events{/privacy}', 'https://api.github.com/users/nan/received_events', 'User', 'false', NULL, NULL, NULL, NULL, NULL, NULL, 'GitHub API Worker', '1.0.0', 'GitHub API', '2021-10-28 15:23:46'); """ - ), - cntrb_uuid=GithubUUID().to_UUID() + ).bindparams( + cntrb_uuid=GithubUUID().to_UUID() + ) ) else: diff --git a/augur/application/schema/alembic/versions/3_oauth_and_user_groups.py b/augur/application/schema/alembic/versions/3_oauth_and_user_groups.py index 8d75b7a70..0d9c6d744 100644 --- a/augur/application/schema/alembic/versions/3_oauth_and_user_groups.py +++ b/augur/application/schema/alembic/versions/3_oauth_and_user_groups.py @@ -85,9 +85,9 @@ def upgrade(): table_changes = """ - ALTER TABLE user_repos + ALTER TABLE augur_operations.user_repos ADD COLUMN group_id BIGINT, - ADD CONSTRAINT user_repos_group_id_fkey FOREIGN KEY (group_id) REFERENCES user_groups(group_id), + ADD CONSTRAINT user_repos_group_id_fkey FOREIGN KEY (group_id) REFERENCES augur_operations.user_groups(group_id), DROP COLUMN user_id, ADD PRIMARY KEY (group_id, repo_id); """ diff --git a/augur/application/schema/alembic/versions/9_add_repo_group_for_frontend_repos.py b/augur/application/schema/alembic/versions/9_add_repo_group_for_frontend_repos.py index 288f584cf..52a6e017d 100644 --- a/augur/application/schema/alembic/versions/9_add_repo_group_for_frontend_repos.py +++ b/augur/application/schema/alembic/versions/9_add_repo_group_for_frontend_repos.py @@ -25,7 +25,7 @@ def upgrade(): conn = op.get_bind() result = conn.execute(text(f"""SELECT * FROM "augur_data"."repo_groups" WHERE rg_name='{repo_group_name}';""")).fetchall() if len(result) == 0: - conn.execute(f"""INSERT INTO "augur_data"."repo_groups" ("rg_name", "rg_description", "rg_website", "rg_recache", "rg_last_modified", "rg_type", "tool_source", "tool_version", "data_source", "data_collection_date") VALUES ('{repo_group_name}', 'DO NOT DELETE OR FRONTEND REPOS WILL BREAK', '', 0, '2023-02-17 15:00:00', NULL, NULL, NULL, NULL, NULL);""") + conn.execute(text(f"""INSERT INTO "augur_data"."repo_groups" ("rg_name", "rg_description", "rg_website", "rg_recache", "rg_last_modified", "rg_type", "tool_source", "tool_version", "data_source", "data_collection_date") VALUES ('{repo_group_name}', 'DO NOT DELETE OR FRONTEND REPOS WILL BREAK', '', 0, '2023-02-17 15:00:00', NULL, NULL, NULL, NULL, NULL);""")) # ### end Alembic commands ### From a83fee5c2a5d71922277478763b8f1451501eb32 Mon Sep 17 00:00:00 2001 From: Seltyk Date: Tue, 8 Aug 2023 09:25:58 -0400 Subject: [PATCH 017/182] It would maybe help if I spelled "schema" right Signed-off-by: Seltyk --- docs/source/rest-api/spec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/rest-api/spec.yml b/docs/source/rest-api/spec.yml index 50643d916..c7ed3601a 100644 --- a/docs/source/rest-api/spec.yml +++ b/docs/source/rest-api/spec.yml @@ -103,7 +103,7 @@ paths: responses: '200': description: OK - scema: + schema: items: properties: base64_url: From 577c4f8994c350de7f6070fb9f9ecbb127fce242 Mon Sep 17 00:00:00 2001 From: Seltyk Date: Tue, 8 Aug 2023 09:59:26 -0400 Subject: [PATCH 018/182] Pretend the schema is an array for rendering I'm honestly not sure why "object" doesn't work out, so for the first time in my life, I had to write docs for the docs in the docs! Signed-off-by: Seltyk --- docs/source/rest-api/spec.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/rest-api/spec.yml b/docs/source/rest-api/spec.yml index c7ed3601a..15423e9a6 100644 --- a/docs/source/rest-api/spec.yml +++ b/docs/source/rest-api/spec.yml @@ -98,7 +98,7 @@ paths: - utility /repos/:id: get: - description: Get a downloaded repo by its ID in Augur. + description: Get a downloaded repo by its ID in Augur. The schema block below says it is an array, but it is not. operationId: Get Repo By ID responses: '200': @@ -136,7 +136,7 @@ paths: url: description: 'URL of this repository, sans leading protocol. Example: github.com/3scale/3scale-operator-metadata' type: string - type: object + type: array tags: - utility /owner/:owner/repo/:repo: From 98c0a5140cc4951245c1fee9e16307d6c8f165f3 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Tue, 8 Aug 2023 11:41:29 -0500 Subject: [PATCH 019/182] get rid of redundancies in relationship definitions in orm Signed-off-by: Isaac Milarsky --- .../application/db/models/augur_operations.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/augur/application/db/models/augur_operations.py b/augur/application/db/models/augur_operations.py index 8507133b0..54db0bb99 100644 --- a/augur/application/db/models/augur_operations.py +++ b/augur/application/db/models/augur_operations.py @@ -271,8 +271,7 @@ class User(Base): {"schema": "augur_operations"} ) - # Redundant. - #groups = relationship("UserGroup") + groups = relationship("UserGroup") tokens = relationship("UserSessionToken") applications = relationship("ClientApplication") @@ -629,7 +628,7 @@ class UserGroup(Base): {"schema": "augur_operations"} ) - user = relationship("User") + #user = relationship("User") repos = relationship("UserRepo") @staticmethod @@ -742,7 +741,7 @@ class UserRepo(Base): #made redundant by user_repo relationship in Repo orm class. #repo = relationship("Repo") - group = relationship("UserGroup") + #group = relationship("UserGroup") @staticmethod def insert(session, repo_id: int, group_id:int = 1) -> bool: @@ -951,7 +950,7 @@ class UserSessionToken(Base): application_id = Column(ForeignKey("augur_operations.client_applications.id", name="user_session_token_application_id_fkey"), nullable=False) created_at = Column(BigInteger) - user = relationship("User") + #user = relationship("User") application = relationship("ClientApplication") refresh_tokens = relationship("RefreshToken") @@ -993,8 +992,8 @@ class ClientApplication(Base): redirect_url = Column(String, nullable=False) api_key = Column(String, nullable=False) - user = relationship("User") - sessions = relationship("UserSessionToken") + #user = relationship("User") + #sessions = relationship("UserSessionToken") subscriptions = relationship("Subscription") def __eq__(self, other): @@ -1015,8 +1014,8 @@ class Subscription(Base): application_id = Column(ForeignKey("augur_operations.client_applications.id", name="subscriptions_application_id_fkey"), primary_key=True) type_id = Column(ForeignKey("augur_operations.subscription_types.id", name="subscriptions_type_id_fkey"), primary_key=True) - application = relationship("ClientApplication") - type = relationship("SubscriptionType") + #application = relationship("ClientApplication") + #type = relationship("SubscriptionType") class SubscriptionType(Base): __tablename__ = "subscription_types" @@ -1042,7 +1041,7 @@ class RefreshToken(Base): id = Column(String, primary_key=True) user_session_token = Column(ForeignKey("augur_operations.user_session_tokens.token", name="refresh_token_session_token_id_fkey"), nullable=False) - user_session = relationship("UserSessionToken") + #user_session = relationship("UserSessionToken") @staticmethod def create(session, user_session_token_id): From e92023b2721272f0965a90ec452feadce085dbd9 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Tue, 8 Aug 2023 12:35:20 -0500 Subject: [PATCH 020/182] use begin() syntax to implicitly commit results of sqlalchemy 2.x transactions Signed-off-by: Isaac Milarsky --- augur/application/cli/db.py | 4 ++-- augur/application/schema/alembic/env.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/augur/application/cli/db.py b/augur/application/cli/db.py index f09aaabbd..42d57ecc6 100644 --- a/augur/application/cli/db.py +++ b/augur/application/cli/db.py @@ -99,7 +99,7 @@ def add_repo_groups(filename): """ Create new repo groups in Augur's database """ - with DatabaseEngine() as engine, engine.connect() as connection: + with DatabaseEngine() as engine, engine.begin() as connection: df = pd.read_sql( s.sql.text("SELECT repo_group_id FROM augur_data.repo_groups"), @@ -248,7 +248,7 @@ def update_api_key(api_key): """ ) - with DatabaseEngine() as engine, engine.connect() as connection: + with DatabaseEngine() as engine, engine.begin() as connection: connection.execute(update_api_key_sql, api_key=api_key) logger.info(f"Updated Augur API key to: {api_key}") diff --git a/augur/application/schema/alembic/env.py b/augur/application/schema/alembic/env.py index d170ef243..85b02eac9 100644 --- a/augur/application/schema/alembic/env.py +++ b/augur/application/schema/alembic/env.py @@ -60,7 +60,7 @@ def run_migrations_online(): """ - with DatabaseEngine() as connectable, connectable.connect() as connection: + with DatabaseEngine() as connectable, connectable.begin() as connection: context.configure( connection=connection, target_metadata=target_metadata, From 82ba6f87780e5ef8575f7d2c3f774343f6713a81 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Fri, 11 Aug 2023 12:48:17 -0500 Subject: [PATCH 021/182] missed a dictionary conversion that used the 1.4 method instead of 2.x method sqlalchemy Signed-off-by: Isaac Milarsky --- augur/tasks/github/pull_requests/files_model/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/augur/tasks/github/pull_requests/files_model/core.py b/augur/tasks/github/pull_requests/files_model/core.py index e7ebcd945..81b4c4397 100644 --- a/augur/tasks/github/pull_requests/files_model/core.py +++ b/augur/tasks/github/pull_requests/files_model/core.py @@ -20,8 +20,8 @@ def pull_request_files_model(repo_id,logger, augur_db, key_auth): pr_numbers = [] #pd.read_sql(pr_number_sql, self.db, params={}) - result = augur_db.execute_sql(pr_number_sql).fetchall() - pr_numbers = [dict(zip(row.keys(), row)) for row in result] + result = augur_db.execute_sql(pr_number_sql)#.fetchall() + pr_numbers = [dict(row) for row in result.mappings()] query = augur_db.session.query(Repo).filter(Repo.repo_id == repo_id) repo = execute_session_query(query, 'one') From b31bd78a3c764200b1f14db4bc9c16aa40d7bc6a Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Mon, 14 Aug 2023 10:50:05 -0500 Subject: [PATCH 022/182] remove call to undefined method in gunicorn start that was there for some reason Signed-off-by: Isaac Milarsky --- augur/api/view/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/augur/api/view/utils.py b/augur/api/view/utils.py index ea5ec80bd..76551a6ab 100644 --- a/augur/api/view/utils.py +++ b/augur/api/view/utils.py @@ -74,7 +74,7 @@ def getSetting(key, section = "View"): loadSettings() -version_check(settings) +#version_check(settings) """ ---------------------------------------------------------------- """ From 34fbeee2250877aecda2600c2655691897c71b17 Mon Sep 17 00:00:00 2001 From: Shiva953 Date: Sat, 7 Oct 2023 11:55:37 +0530 Subject: [PATCH 023/182] add pull_requests_new API endpoint Signed-off-by: Shiva953 --- augur/api/metrics/pull_request.py | 47 +++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/augur/api/metrics/pull_request.py b/augur/api/metrics/pull_request.py index 9fbcc6175..e26e7103a 100644 --- a/augur/api/metrics/pull_request.py +++ b/augur/api/metrics/pull_request.py @@ -10,6 +10,53 @@ from ..server import engine +@register_metric() +def pull_requests_new(repo_group_id, repo_id=None, period='day', begin_date=None, end_date=None): + """ + Returns a time series of the number of new Pull Requests opened during a certain period. + + :param repo_id: The repository's id + :param repo_group_id: The repository's group id + :param period: To set the periodicity to 'day', 'week', 'month' or 'year', defaults to 'day' + :param begin_date: Specifies the begin date, defaults to '1970-1-1 00:00:01' + :param end_date: Specifies the end date, defaults to datetime.now() + :return: DataFrame of new Pull Requests/period + """ + if not begin_date: + begin_date = '1970-1-1 00:00:01' + if not end_date: + end_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + + if repo_id: + new_pull_requests_query = s.sql.text(""" + SELECT DATE_TRUNC(:period, pr_created_at) AS created_date, + COUNT(pr_id) AS new_pull_requests + FROM pull_requests + WHERE repo_id = :repo_id + AND pr_created_at BETWEEN :begin_date AND :end_date + GROUP BY created_date + """) + + results = pd.read_sql(new_pull_requests_query, engine, params={'repo_id': repo_id, 'period': period, + 'begin_date': begin_date, + 'end_date': end_date}) + else: + new_pull_requests_query = s.sql.text(""" + SELECT DATE_TRUNC(:period, pr_created_at) AS created_date, + COUNT(pr_id) AS new_pull_requests + FROM pull_requests + WHERE repo_id IN (SELECT repo_id FROM repo WHERE repo_group_id = :repo_group_id) + AND pr_created_at BETWEEN :begin_date AND :end_date + GROUP BY created_date + """) + + results = pd.read_sql(new_pull_requests_query, engine, + params={'repo_group_id': repo_group_id, 'period': period, + 'begin_date': begin_date, + 'end_date': end_date}) + + return results + @register_metric() def pull_requests_merge_contributor_new(repo_group_id, repo_id=None, period='day', begin_date=None, end_date=None): """ From ed6915fd372365df5bcffc25797d6c044d18d01e Mon Sep 17 00:00:00 2001 From: Shiva953 Date: Mon, 16 Oct 2023 18:51:06 +0530 Subject: [PATCH 024/182] added API documentation for the endpoints Signed-off-by: Shiva953 --- docs/source/rest-api/spec.yml | 84 +++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/docs/source/rest-api/spec.yml b/docs/source/rest-api/spec.yml index ae583dbbb..e60135e8d 100644 --- a/docs/source/rest-api/spec.yml +++ b/docs/source/rest-api/spec.yml @@ -1678,6 +1678,90 @@ paths: type: array tags: - experimental + /repo-groups/:repo_group_id/pull-requests-new: + get: + description: Returns a time series of the number of new Pull Requests opened during a certain period for a specific repository group. + operationId: Pull Requests New (Repo Group) + parameters: + - name: repo_group_id + in: path + description: The repository group's id. + required: true + schema: + type: string + - name: period + in: query + description: Sets the periodicity to 'day', 'week', 'month', or 'year'. + schema: + type: string + - name: begin_date + in: query + description: Specifies the begin date. + schema: + type: string + - name: end_date + in: query + description: Specifies the end date. + schema: + type: string + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + created_date: + type: string + description: Date in the format 'YYYY-MM-DD'. + new_pull_requests: + type: integer + description: The number of new pull requests opened on the date. + tags: + - experimental + /repo-groups/:repo_id/pull-requests-new: + get: + description: Returns a time series of the number of new Pull Requests opened during a certain period for a specific repository. + operationId: Pull Requests New (Repo) + parameters: + - name: repo_id + in: path + description: The repository's id. + required: true + schema: + type: string + - name: period + in: query + description: Sets the periodicity to 'day', 'week', 'month', or 'year'. + schema: + type: string + - name: begin_date + in: query + description: Specifies the begin date. + schema: + type: string + - name: end_date + in: query + description: Specifies the end date. + schema: + type: string + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + created_date: + type: string + description: Date in the format 'YYYY-MM-DD'. + new_pull_requests: + type: integer + description: The number of new pull requests opened on the date. + tags: + - experimental /repo-groups/:repo_group_id/pull-requests-closed-no-merge: get: description: Timeseries of pull request which were closed but not merged From ec5d543c24018840aa1242efc3e1b6fcfdd178ca Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Thu, 26 Oct 2023 17:19:17 -0500 Subject: [PATCH 025/182] fix legacy alembic upgrade Signed-off-by: Isaac Milarsky --- augur/application/schema/alembic/env.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/augur/application/schema/alembic/env.py b/augur/application/schema/alembic/env.py index 85b02eac9..83c1ee17c 100644 --- a/augur/application/schema/alembic/env.py +++ b/augur/application/schema/alembic/env.py @@ -5,7 +5,9 @@ from alembic import context from augur.application.db.models.base import Base -from augur.application.db.engine import DatabaseEngine +from augur.application.db.engine import DatabaseEngine, get_database_string +from sqlalchemy import create_engine +from sqlalchemy.pool import NullPool # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -59,8 +61,10 @@ def run_migrations_online(): and associate a connection with the context. """ + url = get_database_string() + engine = create_engine(url) - with DatabaseEngine() as connectable, connectable.begin() as connection: + with engine.connect() as connection: context.configure( connection=connection, target_metadata=target_metadata, From 740847789e899e7d0a2427959da601ec112d9db7 Mon Sep 17 00:00:00 2001 From: Isaac Milarsky Date: Thu, 26 Oct 2023 17:51:37 -0500 Subject: [PATCH 026/182] upgrade sqlalchemy version and give context for schemas for alembic Signed-off-by: Isaac Milarsky --- augur/application/schema/alembic/env.py | 12 +++++++++++- setup.py | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/augur/application/schema/alembic/env.py b/augur/application/schema/alembic/env.py index 83c1ee17c..94127a43b 100644 --- a/augur/application/schema/alembic/env.py +++ b/augur/application/schema/alembic/env.py @@ -6,7 +6,7 @@ from alembic import context from augur.application.db.models.base import Base from augur.application.db.engine import DatabaseEngine, get_database_string -from sqlalchemy import create_engine +from sqlalchemy import create_engine, event from sqlalchemy.pool import NullPool # this is the Alembic Config object, which provides @@ -64,6 +64,16 @@ def run_migrations_online(): url = get_database_string() engine = create_engine(url) + @event.listens_for(engine, "connect", insert=True) + def set_search_path(dbapi_connection, connection_record): + existing_autocommit = dbapi_connection.autocommit + dbapi_connection.autocommit = True + cursor = dbapi_connection.cursor() + cursor.execute("SET SESSION search_path=public,augur_data,augur_operations,spdx") + cursor.close() + dbapi_connection.autocommit = existing_autocommit + + with engine.connect() as connection: context.configure( connection=connection, diff --git a/setup.py b/setup.py index f48474c3d..c8b0da67c 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ "alembic==1.8.1", # 1.8.1 "coloredlogs==15.0", # 15.0.1 "Beaker==1.11.0", # 1.11.0 - "SQLAlchemy==2.0.17", # 1.4.40 + "SQLAlchemy==2.0.22", # 1.4.40 "itsdangerous==2.0.1", # 2.1.2 'Jinja2~=3.0.3', "Flask==2.0.2", # 2.2.2 From 740e4184fec8ce93ffaaa7bf91d574e3ab520b01 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Thu, 26 Oct 2023 17:56:56 -0500 Subject: [PATCH 027/182] fixing go install on OSX where brew is used Signed-off-by: Sean P. Goggins --- scripts/install/workers.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install/workers.sh b/scripts/install/workers.sh index 4f6dd7a62..b0fdb6d2d 100755 --- a/scripts/install/workers.sh +++ b/scripts/install/workers.sh @@ -39,7 +39,7 @@ do done -if [ -f "/usr/local/go/bin/go" ] || [ -f "/usr/local/bin/go" ] || [ -f "/usr/bin/go" ] || [ -f "/snap/bin/go" ]; then +if [ -f "/usr/local/go/bin/go" ] || [ -f "/usr/local/bin/go" ] || [ -f "/usr/bin/go" ] || [ -f "/snap/bin/go" ] || [ -f "/opt/homebrew/bin/go" ]; then echo "found go!" else echo "Installing go!" @@ -96,4 +96,4 @@ cd $CURRENT_DIR cd $HOME/scc; go build; echo "scc build done" -cd $CURRENT_DIR \ No newline at end of file +cd $CURRENT_DIR From 5d2b51576f7636d1c6f89af66125f72b972de3ff Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Thu, 26 Oct 2023 18:16:31 -0500 Subject: [PATCH 028/182] library version updates and fixed OSSF Scorecard Cloning (I think) Signed-off-by: Sean P. Goggins --- .../data_analysis/message_insights/setup.py | 8 +++---- scripts/install/workers.sh | 24 +++++++++---------- setup.py | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/augur/tasks/data_analysis/message_insights/setup.py b/augur/tasks/data_analysis/message_insights/setup.py index 311eb9b6f..0dc5dfab4 100644 --- a/augur/tasks/data_analysis/message_insights/setup.py +++ b/augur/tasks/data_analysis/message_insights/setup.py @@ -36,10 +36,10 @@ def read(filename): 'nltk==3.6.6', 'pandas==1.3.5', 'emoji==1.2.0', - 'Keras<2.9.0rc0', - 'Keras-Preprocessing==1.1.2', - 'tensorflow==2.8.0', - 'h5py~=3.6.0', + 'Keras==2.13.1', + 'Keras-Preprocessing', + 'tensorflow==2.13.1', + 'h5py==3.10.0', 'scikit-image==0.19.1', 'joblib==1.0.1', 'xgboost', diff --git a/scripts/install/workers.sh b/scripts/install/workers.sh index b0fdb6d2d..3401509e1 100755 --- a/scripts/install/workers.sh +++ b/scripts/install/workers.sh @@ -52,23 +52,23 @@ fi if [ -d "$HOME/scorecard" ]; then echo " Scorecard already exists, skipping cloning ..." echo " Updating Scorecard ... " - rm -rf $HOME/scorecard - echo "Cloning OSSF Scorecard to generate scorecard data ..." - git clone https://github.com/ossf/scorecard $HOME/scorecard - cd $HOME/scorecard + #rm -rf $HOME/scorecard + #echo "Cloning OSSF Scorecard to generate scorecard data ..." + #git clone https://github.com/ossf/scorecard $HOME/scorecard + #cd $HOME/scorecard CURRENT_DIR=$PWD; cd $CURRENT_DIR cd $HOME/scorecard; - go build; - echo "scorecard build done" - cd $CURRENT_DIR + #go build; + #echo "scorecard build done" + #cd $CURRENT_DIR #CURRENT_DIR=$PWD; #cd $HOME/scorecard; - #git pull; - #go mod tidy; - #go build; - #echo "Scorecard build done." - #cd $CURRENT_DIR + git pull; + go mod tidy; + go build; + echo "Scorecard build done." + cd $CURRENT_DIR else echo "Cloning OSSF Scorecard to generate scorecard data ..." git clone https://github.com/ossf/scorecard $HOME/scorecard diff --git a/setup.py b/setup.py index c8b0da67c..3eaa27eb4 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,7 @@ "partd >= 0.3.10", # 1.3.0 "distributed >= 2021.03.0", # 2022.8.1 "nltk==3.6.6", # 3.7 - "h5py~=3.6.0", # 3.7 + "h5py==3.10.0", # 3.7 "scipy==1.7.3", # 1.9.0 "blinker==1.4", # 1.5 "protobuf<3.22", # 4.21.5 From 523753feb51340945a9c5c378317d5ab0e997eb7 Mon Sep 17 00:00:00 2001 From: kaxada Date: Mon, 30 Oct 2023 15:42:51 +0300 Subject: [PATCH 029/182] fixed focus field when 'register' is clicked --- augur/templates/login.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/augur/templates/login.j2 b/augur/templates/login.j2 index c71d02d50..faaab620e 100644 --- a/augur/templates/login.j2 +++ b/augur/templates/login.j2 @@ -108,7 +108,7 @@