feat: migrate to turbo

.changeset/README.md

@@ -0,0 +1,8 @@
+# Changesets
+
+Hello and welcome! This folder has been automatically generated by `@changesets/cli`, a build tool that works
+with multi-package repos, or single-package repos to help you version and publish your code. You can
+find the full documentation for it [in our repository](https://github.com/changesets/changesets)
+
+We have a quick list of common questions to get you started engaging with this project in
+[our documentation](https://github.com/changesets/changesets/blob/main/docs/common-questions.md)

.changeset/config.json

@@ -0,0 +1,11 @@
+{
+  "$schema": "https://unpkg.com/@changesets/config@3.0.2/schema.json",
+  "changelog": "@changesets/cli/changelog",
+  "commit": false,
+  "fixed": [],
+  "linked": [],
+  "access": "restricted",
+  "baseBranch": "main",
+  "updateInternalDependencies": "patch",
+  "ignore": []
+}

.changeset/moody-poems-retire.md

@@ -0,0 +1,5 @@
+---
+"@platforma-open/milaboratories.genome-assets": minor
+---
+
+feat: move to build with turbo

.github/workflows/build.yaml

@@ -1,12 +1,13 @@
-name: Build, indexed genome references
+name: Build, Test and Release Assets
 on:
+  merge_group:
   pull_request:
     types: [opened, reopened, synchronize]
     branches:
-      - "main"
+      - 'main'
   push:
     branches:
-      - "main"
+      - 'main'
   workflow_dispatch: {}
 jobs:
   init:
@@ -16,50 +17,68 @@ jobs:
         with:
           version-canonize: false
           branch-versioning: main
-      - uses: milaboratory/github-ci/actions/context@v4
-        id: ctx
-    outputs:
-      is-release: ${{ steps.ctx.outputs.is-release }}
   run:
     needs:
       - init
-    uses: milaboratory/github-ci/.github/workflows/node-simple.yaml@v4-beta
+    uses: milaboratory/github-ci/.github/workflows/node-simple-pnpm.yaml@v4
     with:
-      app-name: SMART-indexed genome assets
-      app-name-slug: "smart-indexed-genome-assets"
+      app-name: 'Assets: Genome'
+      app-name-slug: 'assets-genome'
       notify-telegram: true
-      node-version: "20.x"
-      is-electron-application: "false"
-      always-auth: "true"
-      registry-url: "https://registry.npmjs.org"
-      build-script-name: ${{ needs.init.outputs.is-release == 'true' && 'build-skip' || 'build' }}
-      gha-runner-label: 'dev-assets'
+      node-version: '20.x'
+      build-script-name: 'build'
+      pnpm-recursive-build: false
       test: false
-      publish-to-public: true
-      aws-login-enable: true
+      test-script-name: 'test'
+      pnpm-recursive-tests: false
+      team-id: 'ciplopen'
+
+      publish-to-public: 'true'
+      package-path: 'catalogue'
+      create-tag: 'true'
+
+      gha-runner-label: 'ubuntu-2xlarge-amd64'
       aws-login-duration: 43199
-      gcp-login-enable: true
+      pre-calculated: true
+      pre-calculated-task-list: |
+        [
+            { "step": "@platforma-open/milaboratories.genome-assets.mus-musculus" },
+            { "step": "@platforma-open/milaboratories.genome-assets.test-species" },
+            { "step": "@platforma-open/milaboratories.genome-assets.sus-scrofa" },
+            { "step": "@platforma-open/milaboratories.genome-assets.saccharomyces-cerevisiae" },
+            { "step": "@platforma-open/milaboratories.genome-assets.rattus-norvegicus" },
+            { "step": "@platforma-open/milaboratories.genome-assets.homo-sapiens" },
+            { "step": "@platforma-open/milaboratories.genome-assets.gallus-gallus" },
+            { "step": "@platforma-open/milaboratories.genome-assets.drosophila-melanogaster" },
+            { "step": "@platforma-open/milaboratories.genome-assets.danio-rerio" },
+            { "step": "@platforma-open/milaboratories.genome-assets.caenorhabditis-elegans" },
+            { "step": "@platforma-open/milaboratories.genome-assets.bos-taurus" },
+            { "step": "@platforma-open/milaboratories.genome-assets.arabidopsis-thaliana" }
+        ]
+
       npmrc-config: |
         {
           "registries": {
             "https://registry.npmjs.org/": {
-              "scopes": ["platforma-open"],
+              "scopes": ["milaboratories", "platforma-sdk", "platforma-open"],
               "tokenVar": "NPMJS_TOKEN"
             }
           }
         }
     secrets:
       env: |
-        {
-          "PL_REGISTRY_PLATFORMA_OPEN_UPLOAD_URL": ${{ toJSON(secrets.PL_REGISTRY_PLOPEN_UPLOAD_URL) }},
-          "NPMJS_TOKEN": ${{ toJSON(secrets.NPMJS_TOKEN) }}
-        }
-      AWS_ASSUME_ROLE: ${{ secrets.AWS_CI_IAM_MONOREPO_SIMPLE_ROLE }}
-      AWS_ASSUME_REGION: "eu-central-1"
-      GCP_KMS_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_KMS_WORKLOAD_IDENTITY_PROVIDER }}
-      GCP_KMS_SERVICE_ACCOUNT: ${{ secrets.GCP_KMS_SERVICE_ACCOUNT }}
-      GCP_KMS_LOCATION: ${{ secrets.GCP_KMS_LOCATION }}
-      GCP_KMS_KEYRING: ${{ secrets.GCP_KMS_KEYRING }}
-      GCP_KMS_KEY_NAME: ${{ secrets.GCP_KMS_KEY_NAME }}
+        { "PL_LICENSE": ${{ toJSON(secrets.MI_LICENSE) }},
+          "MI_LICENSE": ${{ toJSON(secrets.MI_LICENSE) }},
+          "NPMJS_TOKEN": ${{ toJSON(secrets.NPMJS_TOKEN) }},
+          "PL_CI_TEST_USER": ${{ toJSON(secrets.PL_CI_TEST_USER) }},
+          "PL_CI_TEST_PASSWORD": ${{ toJSON(secrets.PL_CI_TEST_PASSWORD) }},
+
+          "AWS_CI_IAM_MONOREPO_SIMPLE_ROLE": ${{ toJSON(secrets.AWS_CI_IAM_MONOREPO_SIMPLE_ROLE) }},
+          "AWS_CI_TURBOREPO_S3_BUCKET": ${{ toJSON(secrets.AWS_CI_TURBOREPO_US_S3_BUCKET) }},
+          "PL_REGISTRY_PLATFORMA_OPEN_UPLOAD_URL": ${{ toJSON(secrets.PL_REGISTRY_PLOPEN_UPLOAD_URL) }} }
+
       TELEGRAM_NOTIFICATION_TARGET: ${{ secrets.TG_CHANNEL_MIBUILDS }}
       TELEGRAM_API_TOKEN: ${{ secrets.TG_CI_BOT_TOKEN }}
+
+      GH_ZEN_APP_ID: ${{ secrets.GH_ZEN_APP_ID }}
+      GH_ZEN_APP_PRIVATE_KEY: ${{ secrets.GH_ZEN_APP_PRIVATE_KEY }}

.gitignore

@@ -1,5 +1,9 @@
-/downloads/
-/indexed_genome/
-/pkg-*.tgz
-/pkg-*.zip
-/node_modules/
+downloads/
+indexed_genome/
+build/
+dist/
+.turbo
+node_modules/
+pkg-*.tgz
+pkg-*.zip
+package.sw.json

arabidopsis-thaliana/genomeFileUrls.json

@@ -0,0 +1,9 @@
+{
+    "Arabidopsis_thaliana": {
+      "assembly_version": "TAIR10",
+      "read_length": 75,
+      "genomeSAindexNbases": 12,
+      "genome_url": "https://ftp.ebi.ac.uk/ensemblgenomes/pub/release-60/plants/fasta/arabidopsis_thaliana/dna/Arabidopsis_thaliana.TAIR10.dna.toplevel.fa.gz",
+      "gtf_url": "https://ftp.ebi.ac.uk/ensemblgenomes/pub/release-60/plants/gtf/arabidopsis_thaliana/Arabidopsis_thaliana.TAIR10.60.gtf.gz"
+    }
+}

arabidopsis-thaliana/package.json

@@ -0,0 +1,28 @@
+{
+    "name": "@platforma-open/milaboratories.genome-assets.arabidopsis-thaliana",
+    "version": "1.0.0",
+    "description": "Assets for Arabidopsis Thaliana",
+    "scripts": {
+      "cleanup": "rm -rf ./pkg-*.tgz && rm -rf ./build/ && rm -rf ./dist/",
+      "build": "../build-everything.sh ./genomeFileUrls.json",
+      "postbuild": "pl-pkg build && ([ -z \"${CI}\" ] || pl-pkg publish)"
+    },
+    "block-software": {
+      "entrypoints": {
+        "main": {
+          "asset": {
+            "type": "asset",
+            "registry": "platforma-open",
+            "root": "./indexed_genome/Arabidopsis_thaliana"
+          }
+        }
+      }
+    },
+    "files": [
+      "dist/"
+    ],
+    "license": "UNLICENSED",
+    "devDependencies": {
+      "@platforma-sdk/package-builder": "catalog:"
+    }
+  }

bos-taurus/genomeFileUrls.json

@@ -0,0 +1,9 @@
+{
+    "Bos_taurus": {
+      "assembly_version": "ARS-UCD1.3",
+      "read_length": 150,
+      "genomeSAindexNbases": 14,
+      "genome_url": "https://ftp.ensembl.org/pub/release-113/fasta/bos_taurus/dna/Bos_taurus.ARS-UCD1.3.dna.toplevel.fa.gz",
+      "gtf_url": "https://ftp.ensembl.org/pub/release-113/gtf/bos_taurus/Bos_taurus.ARS-UCD1.3.113.gtf.gz"
+    }
+}

bos-taurus/package.json

@@ -0,0 +1,28 @@
+{
+    "name": "@platforma-open/milaboratories.genome-assets.bos-taurus",
+    "version": "1.0.0",
+    "description": "Assets for Bos Taurus",
+    "scripts": {
+      "cleanup": "rm -rf ./pkg-*.tgz && rm -rf ./build/ && rm -rf ./dist/",
+      "build": "../build-everything.sh ./genomeFileUrls.json",
+      "postbuild": "pl-pkg build && ([ -z \"${CI}\" ] || pl-pkg publish)"
+    },
+    "block-software": {
+      "entrypoints": {
+        "main": {
+          "asset": {
+            "type": "asset",
+            "registry": "platforma-open",
+            "root": "./indexed_genome/Bos_taurus"
+          }
+        }
+      }
+    },
+    "files": [
+      "dist/"
+    ],
+    "license": "UNLICENSED",
+    "devDependencies": {
+      "@platforma-sdk/package-builder": "catalog:"
+    }
+  }

build-everything.sh

@@ -8,5 +8,5 @@ JSON_FILE="${1}"
 jq -r 'keys[]' "${JSON_FILE}" | 
     while read -r species; do
          export PARENT_SCRIPT_PATH=$(dirname "$(realpath "$0")")
-        ./run_star_index.sh "${species}" ./genomeFileUrls_All.json "./indexed_genome/${species}"
+        ../run_star_index.sh "${species}" ./genomeFileUrls.json "./indexed_genome/${species}"
     done

caenorhabditis-elegans/genomeFileUrls.json

@@ -0,0 +1,9 @@
+{
+    "Caenorhabditis_elegans": {
+      "assembly_version": "WBcel235",
+      "read_length": 100,
+      "genomeSAindexNbases": 12,
+      "genome_url": "https://ftp.ensembl.org/pub/release-113/fasta/caenorhabditis_elegans/dna/Caenorhabditis_elegans.WBcel235.dna.toplevel.fa.gz",
+      "gtf_url": "https://ftp.ensembl.org/pub/release-113/gtf/caenorhabditis_elegans/Caenorhabditis_elegans.WBcel235.113.gtf.gz"
+    }
+}

caenorhabditis-elegans/package.json

@@ -0,0 +1,28 @@
+{
+    "name": "@platforma-open/milaboratories.genome-assets.caenorhabditis-elegans",
+    "version": "1.0.0",
+    "description": "Assets for Caenorhabditis Elegans",
+    "scripts": {
+      "cleanup": "rm -rf ./pkg-*.tgz && rm -rf ./build/ && rm -rf ./dist/",
+      "build": "../build-everything.sh ./genomeFileUrls.json",
+      "postbuild": "pl-pkg build && ([ -z \"${CI}\" ] || pl-pkg publish)"
+    },
+    "block-software": {
+      "entrypoints": {
+        "main": {
+          "asset": {
+            "type": "asset",
+            "registry": "platforma-open",
+            "root": "./indexed_genome/Caenorhabditis_elegans"
+          }
+        }
+      }
+    },
+    "files": [
+      "dist/"
+    ],
+    "license": "UNLICENSED",
+    "devDependencies": {
+      "@platforma-sdk/package-builder": "catalog:"
+    }
+  }

catalogue/package.json

@@ -0,0 +1,71 @@
+{
+
+    "name": "@platforma-open/milaboratories.genome-assets",
+    "version": "1.1.6",
+    "description": "STAR indexed genome references.",
+    "scripts": {
+        "cleanup": "rm -rf ./pkg-*.tgz && rm -rf ./dist/",
+        "build": "pl-pkg build"
+    },
+    "block-software": {
+        "entrypoints": {
+            "test-species": {
+                "reference": "@platforma-open/milaboratories.genome-assets.test-species/dist/tengo/asset/main.as.json"
+            },
+            "sus-scrofa": {
+                "reference": "@platforma-open/milaboratories.genome-assets.sus-scrofa/dist/tengo/asset/main.as.json"
+            },
+            "saccharomyces-cerevisiae": {
+                "reference": "@platforma-open/milaboratories.genome-assets.saccharomyces-cerevisiae/dist/tengo/asset/main.as.json"
+            },
+            "rattus-norvegicus": {
+                "reference": "@platforma-open/milaboratories.genome-assets.rattus-norvegicus/dist/tengo/asset/main.as.json"
+            },
+            "mus-musculus": {
+                "reference": "@platforma-open/milaboratories.genome-assets.mus-musculus/dist/tengo/asset/main.as.json"
+            },
+            "homo-sapiens": {
+                "reference": "@platforma-open/milaboratories.genome-assets.homo-sapiens/dist/tengo/asset/main.as.json"
+            },
+            "gallus-gallus": {
+                "reference": "@platforma-open/milaboratories.genome-assets.gallus-gallus/dist/tengo/asset/main.as.json"
+            },
+            "drosophila-melanogaster": {
+                "reference": "@platforma-open/milaboratories.genome-assets.drosophila-melanogaster/dist/tengo/asset/main.as.json"
+            },
+            "danio-rerio": {
+                "reference": "@platforma-open/milaboratories.genome-assets.danio-rerio/dist/tengo/asset/main.as.json"
+            },
+            "caenorhabditis-elegans": {
+                "reference": "@platforma-open/milaboratories.genome-assets.caenorhabditis-elegans/dist/tengo/asset/main.as.json"
+            },
+            "bos-taurus": {
+                "reference": "@platforma-open/milaboratories.genome-assets.bos-taurus/dist/tengo/asset/main.as.json"
+            },
+            "arabidopsis-thaliana": {
+                "reference": "@platforma-open/milaboratories.genome-assets.arabidopsis-thaliana/dist/tengo/asset/main.as.json"
+            }
+        }
+    },
+    "files": [
+        "dist/"
+    ],
+    "license": "UNLICENSED",
+    "dependencies": {
+        "@platforma-open/milaboratories.genome-assets.test-species": "workspace:*",
+        "@platforma-open/milaboratories.genome-assets.sus-scrofa": "workspace:*",
+        "@platforma-open/milaboratories.genome-assets.saccharomyces-cerevisiae": "workspace:*",
+        "@platforma-open/milaboratories.genome-assets.rattus-norvegicus": "workspace:*",
+        "@platforma-open/milaboratories.genome-assets.mus-musculus": "workspace:*",
+        "@platforma-open/milaboratories.genome-assets.homo-sapiens": "workspace:*",
+        "@platforma-open/milaboratories.genome-assets.gallus-gallus": "workspace:*",
+        "@platforma-open/milaboratories.genome-assets.drosophila-melanogaster": "workspace:*",
+        "@platforma-open/milaboratories.genome-assets.danio-rerio": "workspace:*",
+        "@platforma-open/milaboratories.genome-assets.caenorhabditis-elegans": "workspace:*",
+        "@platforma-open/milaboratories.genome-assets.bos-taurus": "workspace:*",
+        "@platforma-open/milaboratories.genome-assets.arabidopsis-thaliana": "workspace:*"
+    },
+    "devDependencies": {
+        "@platforma-sdk/package-builder": "catalog:"
+    }
+}

danio-rerio/genomeFileUrls.json

@@ -0,0 +1,9 @@
+{
+    "Danio_rerio": {
+      "assembly_version": "GRCz11",
+      "read_length": 150,
+      "genomeSAindexNbases": 14,
+      "genome_url": "https://ftp.ensembl.org/pub/release-113/fasta/danio_rerio/dna/Danio_rerio.GRCz11.dna.toplevel.fa.gz",
+      "gtf_url": "https://ftp.ensembl.org/pub/release-113/gtf/danio_rerio/Danio_rerio.GRCz11.113.gtf.gz"
+    }
+}

danio-rerio/package.json

@@ -0,0 +1,28 @@
+{
+    "name": "@platforma-open/milaboratories.genome-assets.danio-rerio",
+    "version": "1.0.0",
+    "description": "Assets for Danio Rerio",
+    "scripts": {
+      "cleanup": "rm -rf ./pkg-*.tgz && rm -rf ./build/ && rm -rf ./dist/",
+      "build": "../build-everything.sh ./genomeFileUrls.json",
+      "postbuild": "pl-pkg build && ([ -z \"${CI}\" ] || pl-pkg publish)"
+    },
+    "block-software": {
+      "entrypoints": {
+        "main": {
+          "asset": {
+            "type": "asset",
+            "registry": "platforma-open",
+            "root": "./indexed_genome/Danio_rerio"
+          }
+        }
+      }
+    },
+    "files": [
+      "dist/"
+    ],
+    "license": "UNLICENSED",
+    "devDependencies": {
+      "@platforma-sdk/package-builder": "catalog:"
+    }
+  }