From b68858c2720af86acb1cfd323208b2fd4c69a54c Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Thu, 7 Nov 2024 12:32:15 -0600 Subject: [PATCH 1/9] version bump Signed-off-by: Sean P. Goggins --- README.md | 4 ++-- docker/backend/Dockerfile | 2 +- docker/database/Dockerfile | 2 +- docker/rabbitmq/Dockerfile | 2 +- metadata.py | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 731dc9f38..b2aa2450c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Augur NEW Release v0.76.5 +# Augur NEW Release v0.76.6 Augur is primarily a data engineering tool that makes it possible for data scientists to gather open source software community data - less data carpentry for everyone else! The primary way of looking at Augur data is through [8Knot](https://github.com/oss-aspen/8knot), a public instance of 8Knot is available [here](https://metrix.chaoss.io) - this is tied to a public instance of [Augur](https://ai.chaoss.io). @@ -12,7 +12,7 @@ We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy o **If you want to jump right in, the updated docker, docker-compose and bare metal installation instructions are available [here](docs/new-install.md)**. -Augur is now releasing a dramatically improved new version to the ```main``` branch. It is also available [here](https://github.com/chaoss/augur/releases/tag/v0.76.5). +Augur is now releasing a dramatically improved new version to the ```main``` branch. It is also available [here](https://github.com/chaoss/augur/releases/tag/v0.76.6). - The `main` branch is a stable version of our new architecture, which features: diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index 90be0a624..9cbd95775 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -2,7 +2,7 @@ FROM python:3.11-slim-bullseye LABEL maintainer="outdoors@acm.org" -LABEL version="0.76.5" +LABEL version="0.76.6" ENV DEBIAN_FRONTEND=noninteractive ENV PATH="/usr/bin/:/usr/local/bin:/usr/lib:${PATH}" diff --git a/docker/database/Dockerfile b/docker/database/Dockerfile index 3ab62ce8a..c1d109887 100644 --- a/docker/database/Dockerfile +++ b/docker/database/Dockerfile @@ -2,7 +2,7 @@ FROM postgres:16 LABEL maintainer="outdoors@acm.org" -LABEL version="0.76.5" +LABEL version="0.76.6" ENV POSTGRES_DB "test" ENV POSTGRES_USER "augur" diff --git a/docker/rabbitmq/Dockerfile b/docker/rabbitmq/Dockerfile index f020a4243..d0ea2dea4 100644 --- a/docker/rabbitmq/Dockerfile +++ b/docker/rabbitmq/Dockerfile @@ -1,7 +1,7 @@ FROM rabbitmq:3.12-management-alpine LABEL maintainer="574/augur@simplelogin.com" -LABEL version="0.76.5" +LABEL version="0.76.6" ARG RABBIT_MQ_DEFAULT_USER=augur ARG RABBIT_MQ_DEFAULT_PASSWORD=password123 diff --git a/metadata.py b/metadata.py index 872cc1454..c168a801f 100644 --- a/metadata.py +++ b/metadata.py @@ -5,8 +5,8 @@ __short_description__ = "Python 3 package for free/libre and open-source software community metrics, models & data collection" -__version__ = "0.76.5" -__release__ = "v0.76.5 (Pumpkin Home Companion)" +__version__ = "0.76.6" +__release__ = "v0.76.6 (Turkey Drop 24)" __license__ = "MIT" __copyright__ = "University of Missouri, University of Nebraska-Omaha, CHAOSS, Brian Warner & Augurlabs 2112" From 5b33039662460cc9387529c4128af864b5e2a661 Mon Sep 17 00:00:00 2001 From: Greg Sutcliffe Date: Thu, 5 Dec 2024 16:03:05 +0000 Subject: [PATCH 2/9] Update Go version in Docker, and re-enable SCC/Scorecard --- docker/backend/Dockerfile | 3 ++- scripts/docker/install-go.sh | 16 ++++++++-------- scripts/docker/install-workers-deps.sh | 23 ++++++++++++++++++++++- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index 9cbd95775..220bb8ff4 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -100,9 +100,10 @@ RUN set -x \ && /opt/venv/bin/pip install wheel \ && /opt/venv/bin/pip install . +RUN ./scripts/docker/install-go.sh +ENV PATH="${PATH}:/usr/local/go/bin" RUN ./scripts/docker/install-workers-deps.sh -RUN ./scripts/docker/install-go.sh # RUN ./scripts/install/workers.sh RUN mkdir -p repos/ logs/ /augur/facade/ diff --git a/scripts/docker/install-go.sh b/scripts/docker/install-go.sh index 41cfc6b2c..61143cb35 100755 --- a/scripts/docker/install-go.sh +++ b/scripts/docker/install-go.sh @@ -1,9 +1,9 @@ #!/bin/bash -# install Go -installGo() ( - cd "$(mktemp -d)" - wget https://golang.org/dl/go1.16.5.linux-amd64.tar.gz - rm -rf /usr/local/go && tar -C /usr/local -xzf go1.16.5.linux-amd64.tar.gz -) -installGo -export PATH=$PATH:/usr/local/go/bin \ No newline at end of file + +export VERSION="1.22.9" + +cd "$(mktemp -d)" +wget https://golang.org/dl/go${VERSION}.linux-amd64.tar.gz +rm -rf /usr/local/go && tar -C /usr/local -xzf go${VERSION}.linux-amd64.tar.gz + +export PATH=$PATH:/usr/local/go/bin diff --git a/scripts/docker/install-workers-deps.sh b/scripts/docker/install-workers-deps.sh index 90d4d44e0..188ec4362 100755 --- a/scripts/docker/install-workers-deps.sh +++ b/scripts/docker/install-workers-deps.sh @@ -1,9 +1,9 @@ #!/bin/bash set -x + OLD=$(pwd) for i in $(find . | grep -i setup.py); do - cd $(dirname $i) /opt/venv/bin/pip install . cd $OLD @@ -14,3 +14,24 @@ done for i in stopwords punkt popular universal_tagset ; do /opt/venv/bin/python -m nltk.downloader $i done + +# Note this +CURRENT_DIR=$PWD; + +# Install scc +SCC_DIR="$HOME/scc" +echo "Cloning Sloc Cloc and Code (SCC) to generate value data ..." +git clone https://github.com/boyter/scc "$SCC_DIR" +cd $SCC_DIR +go build; +echo "scc build done" +cd $CURRENT_DIR + +# Install scorecard +SCORECARD_DIR="$HOME/scorecard" +echo "Cloning OSSF Scorecard to generate scorecard data ..." +git clone https://github.com/ossf/scorecard $SCORECARD_DIR +cd $SCORECARD_DIR +go build; +echo "scorecard build done" +cd $CURRENT_DIR From f6a954e68919aa7598abe453e7ecf1955f22bd78 Mon Sep 17 00:00:00 2001 From: Ulincsys Date: Tue, 7 Jan 2025 17:55:59 -0600 Subject: [PATCH 3/9] hotfix for issue 2963 --- augur/application/db/models/augur_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/augur/application/db/models/augur_data.py b/augur/application/db/models/augur_data.py index dfb64d1fe..c0b20d670 100644 --- a/augur/application/db/models/augur_data.py +++ b/augur/application/db/models/augur_data.py @@ -1116,7 +1116,7 @@ def insert_gitlab_repo(session, url: str, repo_group_id: int, tool_source, repo_ return result[0]["repo_id"] @staticmethod - def insert_github_repo(session, url: str, repo_group_id: int, tool_source, repo_type, repo_src_id): + def insert_github_repo(session, url: str, repo_group_id: int, tool_source, repo_type, repo_src_id = None): """Add a repo to the repo table. Args: From 13bd2ed9121a101fff5ca8a9f1ce8000663d4ffc Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Tue, 7 Jan 2025 18:04:53 -0600 Subject: [PATCH 4/9] full commit message patch Signed-off-by: Sean P. Goggins --- augur/tasks/git/facade_tasks.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/augur/tasks/git/facade_tasks.py b/augur/tasks/git/facade_tasks.py index 7ff2300a9..26ba2f4b4 100644 --- a/augur/tasks/git/facade_tasks.py +++ b/augur/tasks/git/facade_tasks.py @@ -174,18 +174,27 @@ def facade_fetch_missing_commit_messages(repo_git): for hash in missing_message_hashes: #Get the huge list of commits to process. + logger.debug(f"The hash object is: {hash}. It has a type of: {type(hash)}") + + try: + escaped_hash = hash['cmt_commit_hash'] + except (TypeError, IndexError): + escaped_hash = hash + absolute_path = get_absolute_repo_path(facade_helper.repo_base_directory, repo.repo_id, repo.repo_path, repo.repo_name) repo_loc = (f"{absolute_path}/.git") try: commit_message = check_output( - f"git --git-dir {repo_loc} log --format=%B -n 1 {hash}".split() + f"git --git-dir {repo_loc} log --format=%B -n 1 {escaped_hash}".split() + #f"git --git-dir {repo_loc} log --format=%B -n 1 {hash}".split() ).decode('utf-8').strip() msg_record = { 'repo_id' : repo.repo_id, 'cmt_msg' : commit_message, - 'cmt_hash' : hash, + #'cmt_hash' : hash, + 'cmt_hash': escaped_hash if isinstance(escaped_hash, str) else escaped_hash['cmt_commit_hash'], 'tool_source' : 'Facade', 'tool_version' : '0.78?', 'data_source' : 'git', From ee4e82cc6d7b9601f5db00e3e5d992bd09be862a Mon Sep 17 00:00:00 2001 From: Ulincsys Date: Tue, 7 Jan 2025 18:06:26 -0600 Subject: [PATCH 5/9] gitlab hotfix for issue 2963 Signed-off-by: Ulincsys --- augur/application/db/models/augur_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/augur/application/db/models/augur_data.py b/augur/application/db/models/augur_data.py index c0b20d670..9212bcc5e 100644 --- a/augur/application/db/models/augur_data.py +++ b/augur/application/db/models/augur_data.py @@ -1068,7 +1068,7 @@ def parse_github_org_url(url): return result.groups()[0] @staticmethod - def insert_gitlab_repo(session, url: str, repo_group_id: int, tool_source, repo_src_id): + def insert_gitlab_repo(session, url: str, repo_group_id: int, tool_source, repo_src_id = None): """Add a repo to the repo table. Args: From f114b3cbf2b78ba7aa3a5aada27f2bc3a8245677 Mon Sep 17 00:00:00 2001 From: Sean Goggins Date: Wed, 8 Jan 2025 01:17:57 +0000 Subject: [PATCH 6/9] test Signed-off-by: Sean Goggins --- test.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 test.md diff --git a/test.md b/test.md new file mode 100644 index 000000000..9daeafb98 --- /dev/null +++ b/test.md @@ -0,0 +1 @@ +test From 1f080a008b4302787db43f97463a0563ce1b3328 Mon Sep 17 00:00:00 2001 From: Andrew Brain Date: Tue, 7 Jan 2025 19:34:12 -0600 Subject: [PATCH 7/9] return 0 from populate repo_src_id to fix sum error --- augur/tasks/github/util/populate_repo_src_id.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/augur/tasks/github/util/populate_repo_src_id.py b/augur/tasks/github/util/populate_repo_src_id.py index e4aad54ab..4346bcc3d 100644 --- a/augur/tasks/github/util/populate_repo_src_id.py +++ b/augur/tasks/github/util/populate_repo_src_id.py @@ -18,7 +18,8 @@ def populate_repo_src_id_task(repo_git): repo_src_id = get_repo_src_id(owner, repo, logger) update_repo_src_id(repo_id, repo_src_id) - + + return 0 def update_repo_src_id(repo_id, repo_src_id): From 69c044fbad702416418ced0a518ea9308831e5a7 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Tue, 7 Jan 2025 19:48:24 -0600 Subject: [PATCH 8/9] Update README.md Version Update Signed-off-by: Sean P. Goggins --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b2aa2450c..184a578a0 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Augur NEW Release v0.76.6 +# Augur NEW Release v0.76.7 Augur is primarily a data engineering tool that makes it possible for data scientists to gather open source software community data - less data carpentry for everyone else! The primary way of looking at Augur data is through [8Knot](https://github.com/oss-aspen/8knot), a public instance of 8Knot is available [here](https://metrix.chaoss.io) - this is tied to a public instance of [Augur](https://ai.chaoss.io). @@ -12,7 +12,7 @@ We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy o **If you want to jump right in, the updated docker, docker-compose and bare metal installation instructions are available [here](docs/new-install.md)**. -Augur is now releasing a dramatically improved new version to the ```main``` branch. It is also available [here](https://github.com/chaoss/augur/releases/tag/v0.76.6). +Augur is now releasing a dramatically improved new version to the ```main``` branch. It is also available [here](https://github.com/chaoss/augur/releases/tag/v0.76.7). - The `main` branch is a stable version of our new architecture, which features: @@ -152,4 +152,4 @@ GSoC 2019 participants GSoC 2018 participants ----------------------- -- `Keanu Nichols `_ \ No newline at end of file +- `Keanu Nichols `_ From a8dc898441bdc03ef0e2a0f62eeca7f207792396 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Tue, 7 Jan 2025 19:50:26 -0600 Subject: [PATCH 9/9] Update metadata.py version update Signed-off-by: Sean P. Goggins --- metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata.py b/metadata.py index c168a801f..5b8dd83fe 100644 --- a/metadata.py +++ b/metadata.py @@ -5,8 +5,8 @@ __short_description__ = "Python 3 package for free/libre and open-source software community metrics, models & data collection" -__version__ = "0.76.6" -__release__ = "v0.76.6 (Turkey Drop 24)" +__version__ = "0.76.7" +__release__ = "v0.76.7 (Captain Tuttle)" __license__ = "MIT" __copyright__ = "University of Missouri, University of Nebraska-Omaha, CHAOSS, Brian Warner & Augurlabs 2112"