From c6b2cb5e4957e675a84f06e643b7fa64e63f2da6 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Wed, 12 Feb 2025 15:20:37 -0600 Subject: [PATCH 01/25] Update gsoc-ideas.md Signed-off-by: Sean P. Goggins --- gsoc-ideas.md | 148 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) diff --git a/gsoc-ideas.md b/gsoc-ideas.md index 1cf74e11e..d237d9e37 100644 --- a/gsoc-ideas.md +++ b/gsoc-ideas.md @@ -50,6 +50,154 @@ The aims of the project are as follows: - (Stretch Goal) Improve the operation of the overall machine learning insights pipeline in CHAOSS/augur, and generalize these capabilities. + +## IDEA: Implement Conversion Rate Metric in CHAOSS Software + +**Hours: 350** + +[Micro-tasks and place for questions](https://github.com/chaoss/community/issues/305) + +### Conversion Rate + +Question: What are the rates at which new contributors become more sustained contributors? + +### Description + +The conversion rate metric is primarily aimed at identifying how new community members become more sustained contributors over time. However, the conversion rate metric can also help understand the changing roles of contributors, how a community is growing or declining, and paths to maintainership within an open source community. + +### Objectives (why) + - Observe if new members are becoming more involved with an open source project + - Observe if new members are taking on leadership roles within an open source project + - Observe if outreach efforts are generating new contributors to an open source project + - Observe if outreach efforts are impacting roles of existing community members + - Observe if community conflict results in changing roles within an open source community + - Identify casual, regular, and core contributors + +### Implementation + +This project could be implemented using either the CHAOSS/Augur, or CHAOSS/Grimoirelab (including stack components noted in references) technology stacks. + +The aims of the project are as follows: + - Implement the Conversion Rate Metric in CHAOSS Software + - After discussion, consider which CHAOSS Software Stack you wish to work with + - In collaboration with mentors, define the technology framework, and initial path to a "hello world" version of the metric + - Iterative development of the metric + - Assist in the deployment of this metric for a pre-determined collection of repositories in a publicly viewable website linked to the CHAOSS project. + - Advance the work of the [chaoss metrics models working group](https://github.com/chaoss/wg-metrics-models). + +* _Difficulty:_ Medium +* _Requirements:_ Knowledge of Python is desired. Some knowledge of Javascript or twitter/bootstrap is also desired. Key requirement is a keenness to dig into this challenge! +* _Recommended:_ Python experience. +* _Mentors:_ Sean Goggins, Daniel Izquerdo + +#### Filters (optional) + - Commits + - Issue creation + - Issue comments + - Change request creation + - Change request comments + - Merged change requests + - Code Reviews + - Code Review Comments + - Reactions (emoji) + - Chat platform messages + - Maillist messages + - Meetup attendance + +#### Visualizations + +![](./images/gsoc-1.png) + +Source: https://chaoss.github.io/grimoirelab-sigils/assets/images/screenshots/sigils/overall-community-structure.png + +![](./images/gsoc-2.png) + +Source: https://opensource.com/sites/default/files/uploads/2021-09-15-developer-level-02.png + +#### Tools Providing the Metric + - GrimoireLab + - Augur + - openEuler Infra + +#### Data Collection Strategies + +The following is an example from the [openEuler](https://www.openeuler.org/en/) community: + - A group of people who attended an offline event A held by the community, can be identified as Group A. Demographic information of Group A could be fetched from an on-line survey when people register for the event. To identify the conversation rate of these participants: + - Some people from Group A started watching and forking the repos, indicating they have shown some interest in this community. We marked them as subgroup D0 (Developer Level 0) as a subset of Group A. + - Conversion rate from the total number of people in Group A to the number of people in subgroup D0 is: D0/Group A + - Some people from subgroup D0 make more contributions beyond just watching or forking, including creating issues, making comments on an issue, or performed a code review. We marked them as subgroup D1 (Developer Level 1) as a subset of D0. + - Conversion rate from the total number of people in Subgroup D0 to the number of people in subgroup D1 is: D1/D0. + - Some people from subgroup D1 continue to make more contributions, like code contributions, to the project. This could include creating merge requests and merging new project code. We marked them as subgroup D2 (Developer Level 2) as a subset of D1. + - Conversion rate from the total number of people in subgroup D1 to the number of people in subgroup D2 is: D2/D1. + +![](./images/gsoc-3.png) + + Definition: + - Developer Level 0 (D0) example: Contributors who have given the project a star, or are watching or have forked the repository + - Developer Level 1 (D1): Contributors who have created issues, made comments on an issue, or performed a code review + - Developer Level 2 (D2): Contributors who have created a merge request and successfully merged code + - Conversion Rate (Group A -> D0): CR (Group A -> D2) = D0/Group A + - Conversion Rate (D0 -> D1): CR (D0 -> D1) = D1/D0 + - Conversion Rate (D1 -> D2): CR (D1 -> D2) = D2/D1 + +### References + - https://opensource.com/article/21/11/data-open-source-contributors + - https://github.com/chaoss/augur + - https://gitee.com/openeuler/website-v2/blob/master/web-ui/docs/en/blog/zhongjun/2021-09-15-developer-level.md + - https://chaoss.github.io/grimoirelab-sigils/common/onion_analysis/ + - https://mikemcquaid.com/2018/08/14/the-open-source-contributor-funnel-why-people-dont-contribute-to-your-open-source-project/ +### Contributors + - Yehui Wang + - Clement Li + - zhongjun + - Xiaoya Xia + - Matt Germonprez + - Sean Goggins + - King Gao + + + +## IDEA: Open Source Software Health Metrics Visualization Exploration + +**Hours: 300** + +[Micro-tasks and place for questions](https://github.com/chaoss/augur-community-reports/issues/34) + +The CHAOSS Community currently delivers pre-packaged visualizations of open source software health data through Augur APIs (https://github.com/chaoss/augur/blob/main/augur/routes/pull_request_reports.py and https://github.com/chaoss/augur/blob/main/augur/routes/contributor_reports.py), and the https://github.com/chaoss/augur-community-reports repository. This project seeks to expand, refine, and standardize the visualization of different classes of community health metrics data. Specifically, some analyses are temporal, others are anomaly driven, and in some cases contrasts across repositories and communities are required. In each case, the visualization of data is an essential component for metrics, and what we are now referring to as metrics models (https://github.com/chaoss/wg-metrics-models). + +Additional resources include: http://new.augurlabs.io/ && https://github.com/augurlabs/augur_view which demonsrate the updated twitter/bootstrap Augur frontend. + +The aims of the project are as follows: + - Experiment with standard metrics visualizations using direct Augur database connections, or through the Augur API. + - Refine metrics, and metrics model visualizations using Jupyter Notebooks are similar technology. + - Transform visualizations, as they are completed, into Augur API endpoints, following the pull request, and contributor reports examples. + +* _Difficulty:_ Medium +* _Requirements:_ Strong interest in data visualization. +* _Recommended:_ Experience with Python is desirable, and experience designing, or developing visualizations is desirable. +* _Mentors:_ Isaac Milarsky, Andrew Brain + +## Idea: Build Knowledgebase Application on CHAOSS Website + +**Hours: 350** + +Micro-tasks and place for questions: https://github.com/chaoss/website/issues/708 + +While much of the work of CHAOSS is done in GitHub, the CHAOSS website is often the first place people visit to get information [https://chaoss.community/](https://chaoss.community/). The goal of the website is to create clear paths for new members who want to contribute, metrics users who want information about metrics, and existing members who need information about project operations. As project grows, there is a need for alternative display and categorization options for knowledgebase topics to reduce the burden on website visitors in finding the information that they need. + +Work on this project would require the student to work closely with the mentors and the community to come up with different display and categorization options for CHAOSS Knowledge Base topics. CHAOSS knowledgebase topics that are currently under consideration for this application are released metrics, metrics models, and contributor handbook information. Information about these topics are captured and stored in GitHub repositories by the relevant working groups. The application will need to pull information from github markdown documents to display on the website knowledge base application (we have existing code that does this). + +The aims of the project are as follows: +* Use Wordpress to implement a knowledge application (example knowledgebase plugin up for consideration - https://wordpress.org/plugins/basepress/) +* Research and ideate different display options and categorizations for knowledgebase topics. +* Build web pages to display different knowledgebase topics. + +The aims will require working with front-end web development technologies and WordPress to build a knowledgebase application that can display information about knowledgebase topics. + +* _Difficulty:_ Low +* _Requirements:_ Interest in front-end web development +* _Recommended:_ Experience with Wordpress, HTML, CSS, JavaScript, and GitHub Markdown +* _Mentors:_ John McGinness, Sean Goggins * _Difficulty:_ Medium * _Requirements:_ Interest in software analytics. Python programming. Conceptual understanding of machine learning, and an eagerness to learn maching learning, and SQL knowledge. * _Recommended:_ Experience with Python From f98b5dbcab8864ac21e83d9e840639259bba7e6f Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Wed, 12 Feb 2025 15:23:12 -0600 Subject: [PATCH 02/25] Update gsoc-ideas.md Signed-off-by: Sean P. Goggins --- gsoc-ideas.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gsoc-ideas.md b/gsoc-ideas.md index d237d9e37..19e91c5f1 100644 --- a/gsoc-ideas.md +++ b/gsoc-ideas.md @@ -55,7 +55,7 @@ The aims of the project are as follows: **Hours: 350** -[Micro-tasks and place for questions](https://github.com/chaoss/community/issues/305) +[Micro-tasks and place for questions]([https://github.com/chaoss/community/issues/305](https://github.com/chaoss/augur/issues/2992)) ### Conversion Rate From 8b1a9396e1c104e24c698075b1991d93d094e638 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Wed, 12 Feb 2025 15:25:52 -0600 Subject: [PATCH 03/25] Update gsoc-ideas.md Signed-off-by: Sean P. Goggins --- gsoc-ideas.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gsoc-ideas.md b/gsoc-ideas.md index 19e91c5f1..719de62ce 100644 --- a/gsoc-ideas.md +++ b/gsoc-ideas.md @@ -55,7 +55,7 @@ The aims of the project are as follows: **Hours: 350** -[Micro-tasks and place for questions]([https://github.com/chaoss/community/issues/305](https://github.com/chaoss/augur/issues/2992)) +[Micro-tasks and place for questions](https://github.com/chaoss/augur/issues/2992) ### Conversion Rate @@ -161,7 +161,7 @@ The following is an example from the [openEuler](https://www.openeuler.org/en/) **Hours: 300** -[Micro-tasks and place for questions](https://github.com/chaoss/augur-community-reports/issues/34) +[Micro-tasks and place for questions]([https://github.com/chaoss/augur-community-reports/issues/34](https://github.com/chaoss/augur/issues/2993)) The CHAOSS Community currently delivers pre-packaged visualizations of open source software health data through Augur APIs (https://github.com/chaoss/augur/blob/main/augur/routes/pull_request_reports.py and https://github.com/chaoss/augur/blob/main/augur/routes/contributor_reports.py), and the https://github.com/chaoss/augur-community-reports repository. This project seeks to expand, refine, and standardize the visualization of different classes of community health metrics data. Specifically, some analyses are temporal, others are anomaly driven, and in some cases contrasts across repositories and communities are required. In each case, the visualization of data is an essential component for metrics, and what we are now referring to as metrics models (https://github.com/chaoss/wg-metrics-models). From f847358047d2857a7ac6c293a66179b71999f8f0 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Wed, 12 Feb 2025 15:27:19 -0600 Subject: [PATCH 04/25] Update gsoc-ideas.md Signed-off-by: Sean P. Goggins --- gsoc-ideas.md | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/gsoc-ideas.md b/gsoc-ideas.md index 719de62ce..b61faaec9 100644 --- a/gsoc-ideas.md +++ b/gsoc-ideas.md @@ -161,7 +161,7 @@ The following is an example from the [openEuler](https://www.openeuler.org/en/) **Hours: 300** -[Micro-tasks and place for questions]([https://github.com/chaoss/augur-community-reports/issues/34](https://github.com/chaoss/augur/issues/2993)) +[Micro-tasks and place for questions](https://github.com/chaoss/augur/issues/2993) The CHAOSS Community currently delivers pre-packaged visualizations of open source software health data through Augur APIs (https://github.com/chaoss/augur/blob/main/augur/routes/pull_request_reports.py and https://github.com/chaoss/augur/blob/main/augur/routes/contributor_reports.py), and the https://github.com/chaoss/augur-community-reports repository. This project seeks to expand, refine, and standardize the visualization of different classes of community health metrics data. Specifically, some analyses are temporal, others are anomaly driven, and in some cases contrasts across repositories and communities are required. In each case, the visualization of data is an essential component for metrics, and what we are now referring to as metrics models (https://github.com/chaoss/wg-metrics-models). @@ -176,29 +176,3 @@ The aims of the project are as follows: * _Requirements:_ Strong interest in data visualization. * _Recommended:_ Experience with Python is desirable, and experience designing, or developing visualizations is desirable. * _Mentors:_ Isaac Milarsky, Andrew Brain - -## Idea: Build Knowledgebase Application on CHAOSS Website - -**Hours: 350** - -Micro-tasks and place for questions: https://github.com/chaoss/website/issues/708 - -While much of the work of CHAOSS is done in GitHub, the CHAOSS website is often the first place people visit to get information [https://chaoss.community/](https://chaoss.community/). The goal of the website is to create clear paths for new members who want to contribute, metrics users who want information about metrics, and existing members who need information about project operations. As project grows, there is a need for alternative display and categorization options for knowledgebase topics to reduce the burden on website visitors in finding the information that they need. - -Work on this project would require the student to work closely with the mentors and the community to come up with different display and categorization options for CHAOSS Knowledge Base topics. CHAOSS knowledgebase topics that are currently under consideration for this application are released metrics, metrics models, and contributor handbook information. Information about these topics are captured and stored in GitHub repositories by the relevant working groups. The application will need to pull information from github markdown documents to display on the website knowledge base application (we have existing code that does this). - -The aims of the project are as follows: -* Use Wordpress to implement a knowledge application (example knowledgebase plugin up for consideration - https://wordpress.org/plugins/basepress/) -* Research and ideate different display options and categorizations for knowledgebase topics. -* Build web pages to display different knowledgebase topics. - -The aims will require working with front-end web development technologies and WordPress to build a knowledgebase application that can display information about knowledgebase topics. - -* _Difficulty:_ Low -* _Requirements:_ Interest in front-end web development -* _Recommended:_ Experience with Wordpress, HTML, CSS, JavaScript, and GitHub Markdown -* _Mentors:_ John McGinness, Sean Goggins -* _Difficulty:_ Medium -* _Requirements:_ Interest in software analytics. Python programming. Conceptual understanding of machine learning, and an eagerness to learn maching learning, and SQL knowledge. -* _Recommended:_ Experience with Python -* _Mentors:_ Sean Goggins, Andrew Brain, Isaac Milarsky From 1ffc551c3d5ecf93652a76a9190e644ac0585504 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Wed, 12 Feb 2025 15:45:52 -0600 Subject: [PATCH 05/25] Update gsoc-interest.md Signed-off-by: Sean P. Goggins --- gsoc-interest.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gsoc-interest.md b/gsoc-interest.md index 04a237581..e737687a0 100644 --- a/gsoc-interest.md +++ b/gsoc-interest.md @@ -1,5 +1,7 @@ # Google Summer of Code 2025 Interested Candidates +[IDEAS PAGE](https://github.com/chaoss/augur/blob/main/gsoc-ideas.md) + Hi potential GSoC students, You can ask questions and meet the community on Slack here: https://join.slack.com/t/chaoss-workspace/shared_invite/zt-289zxh6tu-3oQaFlutPFY039MjKpnWcA ... look for the `wg-augur-8knot` channel. From ffec2844ff4917b5e32c2453659a7bdd27325a3c Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Wed, 12 Feb 2025 15:50:55 -0600 Subject: [PATCH 06/25] Update README.md Signed-off-by: Sean P. Goggins --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index f65deb753..b7baa5784 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,6 @@ We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy o ## NEW RELEASE ALERT! **If you want to jump right in, the updated docker, docker-compose and bare metal installation instructions are available [here](docs/new-install.md)**. -<<<<<<< HEAD Augur is now releasing a dramatically improved new version to the ```main``` branch. It is also available [here](https://github.com/chaoss/augur/releases/tag/v0.81.0). From a653027954617ac9d3cdf96d775a65037862f2ac Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Wed, 12 Feb 2025 15:51:47 -0600 Subject: [PATCH 07/25] Update README.md Signed-off-by: Sean P. Goggins --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b7baa5784..3b6f08fb1 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ We strongly believe that much of what makes open source so great is the incredib ## License, Copyright, and Funding -Copyright © 2023 University of Nebraska at Omaha, University of Missouri, Brian Warner, and the CHAOSS Project. +Copyright © 2025 University of Nebraska at Omaha, University of Missouri, Brian Warner, and the CHAOSS Project. Augur is free software: you can redistribute it and/or modify it under the terms of the MIT License as published by the Open Source Initiative. See the [LICENSE](LICENSE) file for more details. From 9500e7367e0715d49f2d39cc3061f67e41697f62 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Thu, 13 Feb 2025 10:40:23 -0600 Subject: [PATCH 08/25] Update gsoc-ideas.md Signed-off-by: Sean P. Goggins --- gsoc-ideas.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gsoc-ideas.md b/gsoc-ideas.md index b61faaec9..c376908c5 100644 --- a/gsoc-ideas.md +++ b/gsoc-ideas.md @@ -159,7 +159,7 @@ The following is an example from the [openEuler](https://www.openeuler.org/en/) ## IDEA: Open Source Software Health Metrics Visualization Exploration -**Hours: 300** +**Hours: 350** [Micro-tasks and place for questions](https://github.com/chaoss/augur/issues/2993) From 32d9a8ce9822e10bc017547b54e1b01212ddb76f Mon Sep 17 00:00:00 2001 From: Ulincsys Date: Thu, 13 Feb 2025 12:42:07 -0600 Subject: [PATCH 09/25] Fix key orchestrator missing in Docker build - Copy keyman directory in Dockerfile - Update graphical dockerfile to reflect the current build process Signed-off-by: Ulincsys --- docker/backend/Dockerfile | 1 + docker/backend/graphical | 91 ++++++++++++++++++++++++++++++++++----- 2 files changed, 82 insertions(+), 10 deletions(-) diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index 220bb8ff4..03dae6b01 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -80,6 +80,7 @@ COPY ./augur/ augur/ COPY ./metadata.py . COPY ./setup.py . COPY ./scripts/ scripts/ +COPY ./keyman/ keyman/ # Add rust and cargo to PATH ENV PATH="/usr/bin/:/root/.cargo/bin:/usr/local/bin:${PATH}" diff --git a/docker/backend/graphical b/docker/backend/graphical index a1c6b95d3..b5c223bef 100644 --- a/docker/backend/graphical +++ b/docker/backend/graphical @@ -1,22 +1,75 @@ -#SPDX-License-Identifier: MIT -FROM python:3.9-slim-bullseye +# SPDX-License-Identifier: MIT +FROM python:3.11-slim-bullseye LABEL maintainer="outdoors@acm.org" -LABEL version="0.51.1" +LABEL version="0.76.6" ENV DEBIAN_FRONTEND=noninteractive +ENV PATH="/usr/bin/:/usr/local/bin:/usr/lib:${PATH}" RUN set -x \ && apt-get update \ - && apt-get -y install --no-install-recommends \ + && apt-get -y install \ git \ bash \ curl \ gcc \ - python3-pip \ + software-properties-common \ + postgresql-contrib \ + musl-dev \ + python3-dev \ + python3-distutils \ + python3-venv \ wget \ postgresql-client \ - && rm -rf /var/lib/apt/lists/* + libpq-dev \ + build-essential \ + rustc \ + cargo \ + chromium \ + tar \ + jq \ + chromium-driver \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + +# Install Firefox from Debian repositories for ARM64 architecture +RUN set -x \ + && apt-get update \ + && apt-get install -y firefox-esr + +# Install Geckodriver +RUN GECKODRIVER_VERSION=$(curl -s https://api.github.com/repos/mozilla/geckodriver/releases/latest | jq -r '.tag_name' | sed 's/v//') \ + && ARCH=$(uname -m) \ + && if [ "$ARCH" = "aarch64" ]; then \ + GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/v${GECKODRIVER_VERSION}/geckodriver-v${GECKODRIVER_VERSION}-linux-aarch64.tar.gz"; \ + GECKODRIVER_FILE="geckodriver-v${GECKODRIVER_VERSION}-linux-aarch64.tar.gz"; \ + else \ + GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/v${GECKODRIVER_VERSION}/geckodriver-v${GECKODRIVER_VERSION}-linux64.tar.gz"; \ + GECKODRIVER_FILE="geckodriver-v${GECKODRIVER_VERSION}-linux64.tar.gz"; \ + fi \ + && wget $GECKODRIVER_URL \ + && tar -xzf $GECKODRIVER_FILE \ + && mv geckodriver /usr/local/bin/ \ + && rm $GECKODRIVER_FILE + +# Verify installations +RUN firefox --version +RUN geckodriver --version + +# Ensure Rust directories are writable +RUN mkdir -p /root/.rustup/downloads /root/.cargo/registry && \ + chmod -R 777 /root/.rustup /root/.cargo + +# Add rust and cargo to PATH +ENV PATH="/root/.cargo/bin:${PATH}" + +# Install the specific version of Rust +RUN set -x \ + && rustup install 1.78.0 +RUN set -x \ + && rustup default 1.78.0 EXPOSE 5000 @@ -27,20 +80,38 @@ COPY ./augur/ augur/ COPY ./metadata.py . COPY ./setup.py . COPY ./scripts/ scripts/ +COPY ./keyman/ keyman/ + +# Add rust and cargo to PATH +ENV PATH="/usr/bin/:/root/.cargo/bin:/usr/local/bin:${PATH}" -#COPY ./docker/backend/docker.config.json . RUN python3 -m venv /opt/venv +RUN set -x \ + && /opt/venv/bin/pip install --upgrade pip + +RUN set -x \ + && /opt/venv/bin/pip install wheel + RUN set -x \ && /opt/venv/bin/pip install . -RUN ./scripts/docker/install-workers-deps.sh +RUN set -x \ + && /opt/venv/bin/pip install --upgrade pip \ + && /opt/venv/bin/pip install wheel \ + && /opt/venv/bin/pip install . RUN ./scripts/docker/install-go.sh +ENV PATH="${PATH}:/usr/local/go/bin" +RUN ./scripts/docker/install-workers-deps.sh + # RUN ./scripts/install/workers.sh RUN mkdir -p repos/ logs/ /augur/facade/ COPY ./docker/backend/graphical.sh / -RUN chmod +x /graphical.sh -ENTRYPOINT /graphical.sh +COPY ./docker/backend/init.sh / +RUN chmod +x /entrypoint.sh /init.sh +ENTRYPOINT ["/bin/bash", "/graphical.sh"] +#ENTRYPOINT ["/entrypoint.sh"] +CMD /init.sh From 1393440af0916afb1106669ca603fb5d17c5420c Mon Sep 17 00:00:00 2001 From: Ulincsys Date: Fri, 14 Feb 2025 03:41:45 -0600 Subject: [PATCH 10/25] Add __init__.py to fix module import error Signed-off-by: Ulincsys --- keyman/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 keyman/__init__.py diff --git a/keyman/__init__.py b/keyman/__init__.py new file mode 100644 index 000000000..e69de29bb From def43f17079293793718bc73b3f0717aa2dda4f8 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Fri, 14 Feb 2025 09:07:57 -0600 Subject: [PATCH 11/25] Update metadata.py Signed-off-by: Sean P. Goggins --- metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata.py b/metadata.py index 2ac76c87d..5a3d190a9 100644 --- a/metadata.py +++ b/metadata.py @@ -5,8 +5,8 @@ __short_description__ = "Python 3 package for free/libre and open-source software community metrics, models & data collection" -__version__ = "0.81.0" -__release__ = "v0.81.0 (Super Soaker)" +__version__ = "0.81.1" +__release__ = "v0.81.1 (Super Soaker)" __license__ = "MIT" __copyright__ = "University of Missouri, University of Nebraska-Omaha, CHAOSS, Sean Goggins, Brian Warner & Augurlabs 2112" From c6646ceef1bc2a479a8471097464c393bac42c94 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Fri, 14 Feb 2025 09:09:42 -0600 Subject: [PATCH 12/25] Update README.md Signed-off-by: Sean P. Goggins --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3b6f08fb1..f33f9e5ae 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Augur NEW Release v0.81.0 +# Augur NEW Release v0.81.1 Augur is primarily a data engineering tool that makes it possible for data scientists to gather open source software community data - less data carpentry for everyone else! The primary way of looking at Augur data is through [8Knot](https://github.com/oss-aspen/8knot), a public instance of 8Knot is available [here](https://metrix.chaoss.io) - this is tied to a public instance of [Augur](https://ai.chaoss.io). @@ -11,7 +11,7 @@ We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy o ## NEW RELEASE ALERT! **If you want to jump right in, the updated docker, docker-compose and bare metal installation instructions are available [here](docs/new-install.md)**. -Augur is now releasing a dramatically improved new version to the ```main``` branch. It is also available [here](https://github.com/chaoss/augur/releases/tag/v0.81.0). +Augur is now releasing a dramatically improved new version to the ```main``` branch. It is also available [here](https://github.com/chaoss/augur/releases/tag/v0.81.1). - The `main` branch is a stable version of our new architecture, which features: @@ -43,9 +43,9 @@ For more information on [how to get involved on the CHAOSS website](https://chao ## Collecting Data -Augur supports ```Python3.6``` through ```Python3.9``` on all platforms. ```Python3.10``` and above do not yet work because of machine learning worker dependencies. On OSX, you can create a ```Python3.9``` environment, by running: +Augur supports ```Python3.7``` through ```Python3.8``` on all platforms. ```Python3.12``` and above do not yet work because of machine learning worker dependencies. On OSX, you can create a ```Python3.11``` environment, by running: ``` -$ python3.9 -m venv path/to/venv +$ python3.11 -m venv path/to/venv ``` Augur's main focus is to measure the overall health and sustainability of open source projects. From 03196c5354e26779fbf6bfbe16afb4445e2c1bc7 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Fri, 14 Feb 2025 09:10:18 -0600 Subject: [PATCH 13/25] Update README.md Signed-off-by: Sean P. Goggins --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f33f9e5ae..5ef9deb95 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ For more information on [how to get involved on the CHAOSS website](https://chao ## Collecting Data -Augur supports ```Python3.7``` through ```Python3.8``` on all platforms. ```Python3.12``` and above do not yet work because of machine learning worker dependencies. On OSX, you can create a ```Python3.11``` environment, by running: +Augur supports ```Python3.7``` through ```Python3.11``` on all platforms. ```Python3.12``` and above do not yet work because of machine learning worker dependencies. On OSX, you can create a ```Python3.11``` environment, by running: ``` $ python3.11 -m venv path/to/venv ``` From 46bf0343e5f0512d16b493533f89df18be30cd53 Mon Sep 17 00:00:00 2001 From: Ulincsys Date: Mon, 17 Feb 2025 10:52:03 -0600 Subject: [PATCH 14/25] Show tracebacks for network exceptions Signed-off-by: Ulincsys --- augur/application/cli/__init__.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/augur/application/cli/__init__.py b/augur/application/cli/__init__.py index 00f41a553..b3bdae779 100644 --- a/augur/application/cli/__init__.py +++ b/augur/application/cli/__init__.py @@ -6,6 +6,7 @@ import re import json import httpx +import traceback from augur.application.db.engine import DatabaseEngine from augur.application.db import get_engine, dispose_database_engine @@ -24,13 +25,16 @@ def new_func(ctx, *args, **kwargs): return ctx.invoke(function_internet_connection, *args, **kwargs) except (TimeoutError, httpx.TimeoutException): print("Request timed out.") - except httpx.NetworkError: + except httpx.NetworkError as e: print(f"Network Error: {httpx.NetworkError}") - except httpx.ProtocolError: + print(traceback.format_exc(e)) + except httpx.ProtocolError as e: print(f"Protocol Error: {httpx.ProtocolError}") + print(traceback.format_exc(e)) + print(f"\n\n{usage} command setup failed\n \ - You are not connected to the internet.\n \ - Please connect to the internet to run Augur\n \ + There was an error while testing for network connectivity.\n \ + Please check your connection to the internet to run Augur\n \ Consider setting http_proxy variables for limited access installations.") sys.exit(-1) From 1d9583d3373794240c529df23b0b4083d582150c Mon Sep 17 00:00:00 2001 From: Ulincsys Date: Mon, 17 Feb 2025 11:27:54 -0600 Subject: [PATCH 15/25] Fix incorrect usage of format_exc Signed-off-by: Ulincsys --- augur/application/cli/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/augur/application/cli/__init__.py b/augur/application/cli/__init__.py index b3bdae779..d20aaad5a 100644 --- a/augur/application/cli/__init__.py +++ b/augur/application/cli/__init__.py @@ -27,10 +27,10 @@ def new_func(ctx, *args, **kwargs): print("Request timed out.") except httpx.NetworkError as e: print(f"Network Error: {httpx.NetworkError}") - print(traceback.format_exc(e)) + print(traceback.format_exc()) except httpx.ProtocolError as e: print(f"Protocol Error: {httpx.ProtocolError}") - print(traceback.format_exc(e)) + print(traceback.format_exc()) print(f"\n\n{usage} command setup failed\n \ There was an error while testing for network connectivity.\n \ From 53c3c14db9daff59262cab2d28b7560e018fe38f Mon Sep 17 00:00:00 2001 From: Ulincsys Date: Mon, 17 Feb 2025 11:51:12 -0600 Subject: [PATCH 16/25] Have decorator pass secondary exceptions transparently Signed-off-by: Ulincsys --- augur/application/cli/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/augur/application/cli/__init__.py b/augur/application/cli/__init__.py index d20aaad5a..998f341e2 100644 --- a/augur/application/cli/__init__.py +++ b/augur/application/cli/__init__.py @@ -21,8 +21,6 @@ def new_func(ctx, *args, **kwargs): try: _ = client.request( method="GET", url="http://chaoss.community", timeout=10, follow_redirects=True) - - return ctx.invoke(function_internet_connection, *args, **kwargs) except (TimeoutError, httpx.TimeoutException): print("Request timed out.") except httpx.NetworkError as e: @@ -36,7 +34,9 @@ def new_func(ctx, *args, **kwargs): There was an error while testing for network connectivity.\n \ Please check your connection to the internet to run Augur\n \ Consider setting http_proxy variables for limited access installations.") - sys.exit(-1) + sys.exit(-1) + + return ctx.invoke(function_internet_connection, *args, **kwargs) return update_wrapper(new_func, function_internet_connection) From 6954f9c491d81e35a9ec5d22e2aa2de34b93a2f7 Mon Sep 17 00:00:00 2001 From: Ulincsys Date: Mon, 17 Feb 2025 12:00:53 -0600 Subject: [PATCH 17/25] Logical fixes and general improvements - Fix logical issue with try block - Use multiline string for logging Signed-off-by: Ulincsys --- augur/application/cli/__init__.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/augur/application/cli/__init__.py b/augur/application/cli/__init__.py index 998f341e2..e68af307b 100644 --- a/augur/application/cli/__init__.py +++ b/augur/application/cli/__init__.py @@ -17,10 +17,12 @@ def test_connection(function_internet_connection): @click.pass_context def new_func(ctx, *args, **kwargs): usage = re.search(r"Usage:\s(.*)\s\[OPTIONS\]", str(ctx.get_usage())).groups()[0] + success = False with httpx.Client() as client: try: _ = client.request( method="GET", url="http://chaoss.community", timeout=10, follow_redirects=True) + success = True except (TimeoutError, httpx.TimeoutException): print("Request timed out.") except httpx.NetworkError as e: @@ -29,12 +31,16 @@ def new_func(ctx, *args, **kwargs): except httpx.ProtocolError as e: print(f"Protocol Error: {httpx.ProtocolError}") print(traceback.format_exc()) - - print(f"\n\n{usage} command setup failed\n \ - There was an error while testing for network connectivity.\n \ - Please check your connection to the internet to run Augur\n \ - Consider setting http_proxy variables for limited access installations.") - sys.exit(-1) + + if not success: + print( + f""" + \n\n{usage} command setup failed. + There was an error while testing for network connectivity + Please check your connection to the internet to run Augur + Consider setting http_proxy variables for limited access installations.""" + ) + sys.exit(-1) return ctx.invoke(function_internet_connection, *args, **kwargs) From 641f72f1655fbba998f15b1204fada0fb0177e1f Mon Sep 17 00:00:00 2001 From: Ulincsys Date: Mon, 17 Feb 2025 13:03:27 -0600 Subject: [PATCH 18/25] Strip whitespace from keys while loading Signed-off-by: Ulincsys --- augur/tasks/github/util/github_api_key_handler.py | 2 ++ augur/tasks/gitlab/gitlab_api_key_handler.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/augur/tasks/github/util/github_api_key_handler.py b/augur/tasks/github/util/github_api_key_handler.py index 9ca777a1b..47933e67d 100644 --- a/augur/tasks/github/util/github_api_key_handler.py +++ b/augur/tasks/github/util/github_api_key_handler.py @@ -107,6 +107,8 @@ def get_api_keys(self) -> List[str]: if len(keys) == 0: return [] + + keys = [key.strip() for key in keys] valid_keys = [] with httpx.Client() as client: diff --git a/augur/tasks/gitlab/gitlab_api_key_handler.py b/augur/tasks/gitlab/gitlab_api_key_handler.py index 03c0cf66c..72b0ace14 100644 --- a/augur/tasks/gitlab/gitlab_api_key_handler.py +++ b/augur/tasks/gitlab/gitlab_api_key_handler.py @@ -110,6 +110,8 @@ def get_api_keys(self) -> List[str]: if len(keys) == 0: return [] + keys = [key.strip() for key in keys] + valid_keys = [] with httpx.Client() as client: From e6940c6b99cb60298cb51a76e4c35dd1c0c7afe3 Mon Sep 17 00:00:00 2001 From: Ulincsys Date: Tue, 18 Feb 2025 12:51:26 -0600 Subject: [PATCH 19/25] Filter worker_oauth keys by platform in cli Signed-off-by: Ulincsys --- augur/application/cli/github.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/augur/application/cli/github.py b/augur/application/cli/github.py index cad13be79..4896bf05f 100644 --- a/augur/application/cli/github.py +++ b/augur/application/cli/github.py @@ -32,7 +32,7 @@ def update_api_key(): """ SELECT value as github_key from config Where section_name='Keys' AND setting_name='github_api_key' UNION All - SELECT access_token as github_key from worker_oauth ORDER BY github_key DESC; + SELECT access_token as github_key from worker_oauth where platform='github' ORDER BY github_key DESC; """ ) From 9a4b8660e5d45cd7dd1dbf37459b98507a03d7ad Mon Sep 17 00:00:00 2001 From: Ulincsys Date: Fri, 21 Feb 2025 13:30:01 -0600 Subject: [PATCH 20/25] Fix comparison with NoneType logical error Signed-off-by: Ulincsys --- augur/tasks/github/pull_requests/tasks.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/augur/tasks/github/pull_requests/tasks.py b/augur/tasks/github/pull_requests/tasks.py index b65da7f4f..d88a50394 100644 --- a/augur/tasks/github/pull_requests/tasks.py +++ b/augur/tasks/github/pull_requests/tasks.py @@ -231,9 +231,13 @@ def collect_pull_request_review_comments(repo_git: str, full_collection: bool) - repo_id = get_repo_by_repo_git(repo_git).repo_id if not full_collection: - # subtract 2 days to ensure all data is collected - core_data_last_collected = (get_core_data_last_collected(repo_id) - timedelta(days=2)).replace(tzinfo=timezone.utc) - review_msg_url += f"?since={core_data_last_collected.isoformat()}" + last_collected_date = get_core_data_last_collected(repo_id) + + # the full_collection parameter is hardcoded to False, so we must check if there actually is a last_collected date + if last_collected_date: + # subtract 2 days to ensure all data is collected + core_data_last_collected = (last_collected_date - timedelta(days=2)).replace(tzinfo=timezone.utc) + review_msg_url += f"?since={core_data_last_collected.isoformat()}" pr_reviews = get_pull_request_reviews_by_repo_id(repo_id) From 1e417caeb39acaa678745c1ff09093712b978f73 Mon Sep 17 00:00:00 2001 From: Ulincsys Date: Fri, 21 Feb 2025 18:16:01 -0600 Subject: [PATCH 21/25] Secondary task to use get_secondary_data_last_collected Signed-off-by: Ulincsys --- augur/tasks/github/pull_requests/tasks.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/augur/tasks/github/pull_requests/tasks.py b/augur/tasks/github/pull_requests/tasks.py index d88a50394..21ef9d8eb 100644 --- a/augur/tasks/github/pull_requests/tasks.py +++ b/augur/tasks/github/pull_requests/tasks.py @@ -231,13 +231,14 @@ def collect_pull_request_review_comments(repo_git: str, full_collection: bool) - repo_id = get_repo_by_repo_git(repo_git).repo_id if not full_collection: - last_collected_date = get_core_data_last_collected(repo_id) + last_collected_date = get_secondary_data_last_collected(repo_id) - # the full_collection parameter is hardcoded to False, so we must check if there actually is a last_collected date if last_collected_date: # subtract 2 days to ensure all data is collected core_data_last_collected = (last_collected_date - timedelta(days=2)).replace(tzinfo=timezone.utc) - review_msg_url += f"?since={core_data_last_collected.isoformat()}" + review_msg_url += f"?since={core_data_last_collected.isoformat()}" + else: + logger.warning(f"core_data_last_collected is NULL for recollection on repo: {repo_git}") pr_reviews = get_pull_request_reviews_by_repo_id(repo_id) From 47a8c66e79a6c9751f09cdb35c12730e7225e0f5 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Sat, 22 Feb 2025 12:02:49 -0600 Subject: [PATCH 22/25] Update README.md Signed-off-by: Sean P. Goggins --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5ef9deb95..2f4fdb67a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Augur NEW Release v0.81.1 +# Augur NEW Release v0.81.2 Augur is primarily a data engineering tool that makes it possible for data scientists to gather open source software community data - less data carpentry for everyone else! The primary way of looking at Augur data is through [8Knot](https://github.com/oss-aspen/8knot), a public instance of 8Knot is available [here](https://metrix.chaoss.io) - this is tied to a public instance of [Augur](https://ai.chaoss.io). @@ -11,7 +11,7 @@ We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy o ## NEW RELEASE ALERT! **If you want to jump right in, the updated docker, docker-compose and bare metal installation instructions are available [here](docs/new-install.md)**. -Augur is now releasing a dramatically improved new version to the ```main``` branch. It is also available [here](https://github.com/chaoss/augur/releases/tag/v0.81.1). +Augur is now releasing a dramatically improved new version to the ```main``` branch. It is also available [here](https://github.com/chaoss/augur/releases/tag/v0.81.2). - The `main` branch is a stable version of our new architecture, which features: From 474c65c444aa03c5a2779de8db384e62e1a288b8 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Sat, 22 Feb 2025 12:03:40 -0600 Subject: [PATCH 23/25] Update metadata.py Signed-off-by: Sean P. Goggins --- metadata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metadata.py b/metadata.py index 5a3d190a9..7c91b77f5 100644 --- a/metadata.py +++ b/metadata.py @@ -5,8 +5,8 @@ __short_description__ = "Python 3 package for free/libre and open-source software community metrics, models & data collection" -__version__ = "0.81.1" -__release__ = "v0.81.1 (Super Soaker)" +__version__ = "0.81.2" +__release__ = "v0.81.2 (Basketball Jones)" __license__ = "MIT" -__copyright__ = "University of Missouri, University of Nebraska-Omaha, CHAOSS, Sean Goggins, Brian Warner & Augurlabs 2112" +__copyright__ = "University of Missouri, University of Nebraska-Omaha, CHAOSS, Sean Goggins, Brian Warner & Augurlabs 2025" From 21fae403c63caf969a7d93047c7f845223f3c9be Mon Sep 17 00:00:00 2001 From: Andrew Brain Date: Tue, 25 Feb 2025 19:17:52 -0600 Subject: [PATCH 24/25] reduce dependency on create_collection_status_record tasks --- augur/application/cli/backend.py | 3 +++ augur/tasks/frontend.py | 6 +++++- augur/tasks/init/celery_app.py | 7 +++++-- augur/tasks/start_tasks.py | 4 ++-- augur/util/repo_load_controller.py | 4 +++- 5 files changed, 18 insertions(+), 6 deletions(-) diff --git a/augur/application/cli/backend.py b/augur/application/cli/backend.py index 2b5ec6904..aeddbf463 100644 --- a/augur/application/cli/backend.py +++ b/augur/application/cli/backend.py @@ -86,6 +86,9 @@ def start(ctx, disable_collection, development, pidfile, port): worker_vmem_cap = get_value("Celery", 'worker_process_vmem_cap') + # create rabbit messages so if it failed on shutdown the queues are clean + clear_rabbitmq_messages() + gunicorn_command = f"gunicorn -c {gunicorn_location} -b {host}:{port} augur.api.server:app --log-file gunicorn.log" server = subprocess.Popen(gunicorn_command.split(" ")) diff --git a/augur/tasks/frontend.py b/augur/tasks/frontend.py index f526d9041..cd2d3e32b 100644 --- a/augur/tasks/frontend.py +++ b/augur/tasks/frontend.py @@ -9,7 +9,7 @@ from augur.tasks.github.util.github_graphql_data_access import GithubGraphQlDataAccess from augur.application.db.lib import get_group_by_name, get_repo_by_repo_git, get_github_repo_by_src_id, get_gitlab_repo_by_src_id from augur.tasks.github.util.util import get_owner_repo -from augur.application.db.models.augur_operations import retrieve_owner_repos, FRONTEND_REPO_GROUP_NAME, RepoGroup +from augur.application.db.models.augur_operations import retrieve_owner_repos, FRONTEND_REPO_GROUP_NAME, RepoGroup, CollectionStatus from augur.tasks.github.util.github_paginator import hit_api from augur.application.db.models import UserRepo, Repo @@ -235,6 +235,8 @@ def add_github_repo(logger, session, url, repo_group_id, group_id, repo_type, re logger.error(f"Error while adding repo: Failed to insert user repo record. A record with a repo_id of {repo_id} and a group id of {group_id} needs to be added to the user repo table so that this repo shows up in the users group") return + CollectionStatus.insert(session, logger, repo_id) + def get_gitlab_repo_data(gl_session, url: str, logger) -> bool: @@ -281,6 +283,8 @@ def add_gitlab_repo(logger, session, url, repo_group_id, group_id, repo_src_id): if not result: logger.error(f"Error while adding repo: Failed to insert user repo record. A record with a repo_id of {repo_id} and a group id of {group_id} needs to be added to the user repo table so that this repo shows up in the users group") return + + CollectionStatus.insert(session, logger, repo_id) # @celery.task # def add_org_repo_list(user_id, group_name, urls): diff --git a/augur/tasks/init/celery_app.py b/augur/tasks/init/celery_app.py index 1be45b1f0..5026244f8 100644 --- a/augur/tasks/init/celery_app.py +++ b/augur/tasks/init/celery_app.py @@ -201,8 +201,8 @@ def setup_periodic_tasks(sender, **kwargs): The tasks so that they are grouped by the module they are defined in """ from celery.schedules import crontab - from augur.tasks.start_tasks import augur_collection_monitor, augur_collection_update_weights - from augur.tasks.start_tasks import non_repo_domain_tasks, retry_errored_repos + from augur.tasks.start_tasks import augur_collection_monitor + from augur.tasks.start_tasks import non_repo_domain_tasks, retry_errored_repos, create_collection_status_records from augur.tasks.git.facade_tasks import clone_repos from augur.tasks.db.refresh_materialized_views import refresh_materialized_views from augur.tasks.data_analysis.contributor_breadth_worker.contributor_breadth_worker import contributor_breadth_model @@ -232,6 +232,9 @@ def setup_periodic_tasks(sender, **kwargs): logger.info(f"Setting 404 repos to be marked for retry on midnight each day") sender.add_periodic_task(crontab(hour=0, minute=0),retry_errored_repos.s()) + one_day_in_seconds = 24*60*60 + sender.add_periodic_task(one_day_in_seconds, create_collection_status_records.s()) + @after_setup_logger.connect def setup_loggers(*args,**kwargs): """Override Celery loggers with our own.""" diff --git a/augur/tasks/start_tasks.py b/augur/tasks/start_tasks.py index 3ba30ed70..2a697a0ea 100644 --- a/augur/tasks/start_tasks.py +++ b/augur/tasks/start_tasks.py @@ -378,5 +378,5 @@ def create_collection_status_records(self): CollectionStatus.insert(session, logger, repo[0]) repo = execute_sql(query).first() - #Check for new repos every seven minutes to be out of step with the clone_repos task - create_collection_status_records.si().apply_async(countdown=60*7) + # no longer recursively run this task because collection status records are added when repos are inserted + #create_collection_status_records.si().apply_async(countdown=60*7) diff --git a/augur/util/repo_load_controller.py b/augur/util/repo_load_controller.py index 7021a215f..af46ce326 100644 --- a/augur/util/repo_load_controller.py +++ b/augur/util/repo_load_controller.py @@ -5,7 +5,7 @@ from typing import Any, Dict from augur.application.db.engine import DatabaseEngine -from augur.application.db.models import Repo, UserRepo, RepoGroup, UserGroup, User +from augur.application.db.models import Repo, UserRepo, RepoGroup, UserGroup, User, CollectionStatus from augur.application.db.models.augur_operations import retrieve_owner_repos from augur.application.db.util import execute_session_query @@ -67,8 +67,10 @@ def add_cli_repo(self, repo_data: Dict[str, Any], from_org_list=False, repo_type # if the repo doesn't exist it adds it if "gitlab" in url: repo_id = Repo.insert_gitlab_repo(self.session, url, repo_group_id, "CLI") + CollectionStatus.insert(self.session, logger, repo_id) else: repo_id = Repo.insert_github_repo(self.session, url, repo_group_id, "CLI", repo_type) + CollectionStatus.insert(self.session, logger, repo_id) if not repo_id: logger.warning(f"Invalid repo group id specified for {url}, skipping.") From 9ca06bd2103a94d88d0e1ef06d762a5f869ab9ab Mon Sep 17 00:00:00 2001 From: Andrew Brain Date: Tue, 25 Feb 2025 19:25:00 -0600 Subject: [PATCH 25/25] cleanup everything --- augur/application/cli/backend.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/augur/application/cli/backend.py b/augur/application/cli/backend.py index aeddbf463..69f93aa6e 100644 --- a/augur/application/cli/backend.py +++ b/augur/application/cli/backend.py @@ -87,7 +87,7 @@ def start(ctx, disable_collection, development, pidfile, port): worker_vmem_cap = get_value("Celery", 'worker_process_vmem_cap') # create rabbit messages so if it failed on shutdown the queues are clean - clear_rabbitmq_messages() + cleanup_collection_status_and_rabbit(logger, ctx.obj.engine) gunicorn_command = f"gunicorn -c {gunicorn_location} -b {host}:{port} augur.api.server:app --log-file gunicorn.log" server = subprocess.Popen(gunicorn_command.split(" ")) @@ -183,7 +183,7 @@ def start(ctx, disable_collection, development, pidfile, port): try: keypub.shutdown() - cleanup_after_collection_halt(logger, ctx.obj.engine) + cleanup_collection_status_and_rabbit(logger, ctx.obj.engine) except RedisConnectionError: pass @@ -305,7 +305,7 @@ def stop_collection(ctx): logger.info(f"Waiting on [{', '.join(str(p.pid for p in alive))}]") time.sleep(0.5) - cleanup_after_collection_halt(logger, ctx.obj.engine) + cleanup_collection_status_and_rabbit(logger, ctx.obj.engine) @cli.command('kill') @test_connection @@ -333,10 +333,10 @@ def augur_stop(signal, logger, engine): _broadcast_signal_to_processes(augur_processes, broadcast_signal=signal, given_logger=logger) if "celery" in process_names: - cleanup_after_collection_halt(logger, engine) + cleanup_collection_status_and_rabbit(logger, engine) -def cleanup_after_collection_halt(logger, engine): +def cleanup_collection_status_and_rabbit(logger, engine): clear_redis_caches() connection_string = get_value("RabbitMQ", "connection_string")