Skip to content

Commit

Permalink
Merge pull request #1 from platforma-open/vladimir.antropov/migrate-t…
Browse files Browse the repository at this point in the history
…o-turbo

feat: migrate to turbo
  • Loading branch information
mike-ainsel authored Jan 7, 2025
2 parents 1a4b53b + 1922df2 commit d8840c2
Show file tree
Hide file tree
Showing 39 changed files with 3,648 additions and 3,710 deletions.
8 changes: 8 additions & 0 deletions .changeset/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# Changesets

Hello and welcome! This folder has been automatically generated by `@changesets/cli`, a build tool that works
with multi-package repos, or single-package repos to help you version and publish your code. You can
find the full documentation for it [in our repository](https://github.com/changesets/changesets)

We have a quick list of common questions to get you started engaging with this project in
[our documentation](https://github.com/changesets/changesets/blob/main/docs/common-questions.md)
11 changes: 11 additions & 0 deletions .changeset/config.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"$schema": "https://unpkg.com/@changesets/config@3.0.2/schema.json",
"changelog": "@changesets/cli/changelog",
"commit": false,
"fixed": [],
"linked": [],
"access": "restricted",
"baseBranch": "main",
"updateInternalDependencies": "patch",
"ignore": []
}
5 changes: 5 additions & 0 deletions .changeset/moody-poems-retire.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@platforma-open/milaboratories.genome-assets": minor
---

feat: move to build with turbo
81 changes: 50 additions & 31 deletions .github/workflows/build.yaml
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
name: Build, indexed genome references
name: Build, Test and Release Assets
on:
merge_group:
pull_request:
types: [opened, reopened, synchronize]
branches:
- "main"
- 'main'
push:
branches:
- "main"
- 'main'
workflow_dispatch: {}
jobs:
init:
Expand All @@ -16,50 +17,68 @@ jobs:
with:
version-canonize: false
branch-versioning: main
- uses: milaboratory/github-ci/actions/context@v4
id: ctx
outputs:
is-release: ${{ steps.ctx.outputs.is-release }}
run:
needs:
- init
uses: milaboratory/github-ci/.github/workflows/node-simple.yaml@v4-beta
uses: milaboratory/github-ci/.github/workflows/node-simple-pnpm.yaml@v4
with:
app-name: SMART-indexed genome assets
app-name-slug: "smart-indexed-genome-assets"
app-name: 'Assets: Genome'
app-name-slug: 'assets-genome'
notify-telegram: true
node-version: "20.x"
is-electron-application: "false"
always-auth: "true"
registry-url: "https://registry.npmjs.org"
build-script-name: ${{ needs.init.outputs.is-release == 'true' && 'build-skip' || 'build' }}
gha-runner-label: 'dev-assets'
node-version: '20.x'
build-script-name: 'build'
pnpm-recursive-build: false
test: false
publish-to-public: true
aws-login-enable: true
test-script-name: 'test'
pnpm-recursive-tests: false
team-id: 'ciplopen'

publish-to-public: 'true'
package-path: 'catalogue'
create-tag: 'true'

gha-runner-label: 'ubuntu-2xlarge-amd64'
aws-login-duration: 43199
gcp-login-enable: true
pre-calculated: true
pre-calculated-task-list: |
[
{ "step": "@platforma-open/milaboratories.genome-assets.mus-musculus" },
{ "step": "@platforma-open/milaboratories.genome-assets.test-species" },
{ "step": "@platforma-open/milaboratories.genome-assets.sus-scrofa" },
{ "step": "@platforma-open/milaboratories.genome-assets.saccharomyces-cerevisiae" },
{ "step": "@platforma-open/milaboratories.genome-assets.rattus-norvegicus" },
{ "step": "@platforma-open/milaboratories.genome-assets.homo-sapiens" },
{ "step": "@platforma-open/milaboratories.genome-assets.gallus-gallus" },
{ "step": "@platforma-open/milaboratories.genome-assets.drosophila-melanogaster" },
{ "step": "@platforma-open/milaboratories.genome-assets.danio-rerio" },
{ "step": "@platforma-open/milaboratories.genome-assets.caenorhabditis-elegans" },
{ "step": "@platforma-open/milaboratories.genome-assets.bos-taurus" },
{ "step": "@platforma-open/milaboratories.genome-assets.arabidopsis-thaliana" }
]
npmrc-config: |
{
"registries": {
"https://registry.npmjs.org/": {
"scopes": ["platforma-open"],
"scopes": ["milaboratories", "platforma-sdk", "platforma-open"],
"tokenVar": "NPMJS_TOKEN"
}
}
}
secrets:
env: |
{
"PL_REGISTRY_PLATFORMA_OPEN_UPLOAD_URL": ${{ toJSON(secrets.PL_REGISTRY_PLOPEN_UPLOAD_URL) }},
"NPMJS_TOKEN": ${{ toJSON(secrets.NPMJS_TOKEN) }}
}
AWS_ASSUME_ROLE: ${{ secrets.AWS_CI_IAM_MONOREPO_SIMPLE_ROLE }}
AWS_ASSUME_REGION: "eu-central-1"
GCP_KMS_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.GCP_KMS_WORKLOAD_IDENTITY_PROVIDER }}
GCP_KMS_SERVICE_ACCOUNT: ${{ secrets.GCP_KMS_SERVICE_ACCOUNT }}
GCP_KMS_LOCATION: ${{ secrets.GCP_KMS_LOCATION }}
GCP_KMS_KEYRING: ${{ secrets.GCP_KMS_KEYRING }}
GCP_KMS_KEY_NAME: ${{ secrets.GCP_KMS_KEY_NAME }}
{ "PL_LICENSE": ${{ toJSON(secrets.MI_LICENSE) }},
"MI_LICENSE": ${{ toJSON(secrets.MI_LICENSE) }},
"NPMJS_TOKEN": ${{ toJSON(secrets.NPMJS_TOKEN) }},
"PL_CI_TEST_USER": ${{ toJSON(secrets.PL_CI_TEST_USER) }},
"PL_CI_TEST_PASSWORD": ${{ toJSON(secrets.PL_CI_TEST_PASSWORD) }},
"AWS_CI_IAM_MONOREPO_SIMPLE_ROLE": ${{ toJSON(secrets.AWS_CI_IAM_MONOREPO_SIMPLE_ROLE) }},
"AWS_CI_TURBOREPO_S3_BUCKET": ${{ toJSON(secrets.AWS_CI_TURBOREPO_US_S3_BUCKET) }},
"PL_REGISTRY_PLATFORMA_OPEN_UPLOAD_URL": ${{ toJSON(secrets.PL_REGISTRY_PLOPEN_UPLOAD_URL) }} }
TELEGRAM_NOTIFICATION_TARGET: ${{ secrets.TG_CHANNEL_MIBUILDS }}
TELEGRAM_API_TOKEN: ${{ secrets.TG_CI_BOT_TOKEN }}

GH_ZEN_APP_ID: ${{ secrets.GH_ZEN_APP_ID }}
GH_ZEN_APP_PRIVATE_KEY: ${{ secrets.GH_ZEN_APP_PRIVATE_KEY }}
14 changes: 9 additions & 5 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
/downloads/
/indexed_genome/
/pkg-*.tgz
/pkg-*.zip
/node_modules/
downloads/
indexed_genome/
build/
dist/
.turbo
node_modules/
pkg-*.tgz
pkg-*.zip
package.sw.json
9 changes: 9 additions & 0 deletions arabidopsis-thaliana/genomeFileUrls.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"Arabidopsis_thaliana": {
"assembly_version": "TAIR10",
"read_length": 75,
"genomeSAindexNbases": 12,
"genome_url": "https://ftp.ebi.ac.uk/ensemblgenomes/pub/release-60/plants/fasta/arabidopsis_thaliana/dna/Arabidopsis_thaliana.TAIR10.dna.toplevel.fa.gz",
"gtf_url": "https://ftp.ebi.ac.uk/ensemblgenomes/pub/release-60/plants/gtf/arabidopsis_thaliana/Arabidopsis_thaliana.TAIR10.60.gtf.gz"
}
}
28 changes: 28 additions & 0 deletions arabidopsis-thaliana/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
"name": "@platforma-open/milaboratories.genome-assets.arabidopsis-thaliana",
"version": "1.0.0",
"description": "Assets for Arabidopsis Thaliana",
"scripts": {
"cleanup": "rm -rf ./pkg-*.tgz && rm -rf ./build/ && rm -rf ./dist/",
"build": "../build-everything.sh ./genomeFileUrls.json",
"postbuild": "pl-pkg build && ([ -z \"${CI}\" ] || pl-pkg publish)"
},
"block-software": {
"entrypoints": {
"main": {
"asset": {
"type": "asset",
"registry": "platforma-open",
"root": "./indexed_genome/Arabidopsis_thaliana"
}
}
}
},
"files": [
"dist/"
],
"license": "UNLICENSED",
"devDependencies": {
"@platforma-sdk/package-builder": "catalog:"
}
}
9 changes: 9 additions & 0 deletions bos-taurus/genomeFileUrls.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"Bos_taurus": {
"assembly_version": "ARS-UCD1.3",
"read_length": 150,
"genomeSAindexNbases": 14,
"genome_url": "https://ftp.ensembl.org/pub/release-113/fasta/bos_taurus/dna/Bos_taurus.ARS-UCD1.3.dna.toplevel.fa.gz",
"gtf_url": "https://ftp.ensembl.org/pub/release-113/gtf/bos_taurus/Bos_taurus.ARS-UCD1.3.113.gtf.gz"
}
}
28 changes: 28 additions & 0 deletions bos-taurus/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
"name": "@platforma-open/milaboratories.genome-assets.bos-taurus",
"version": "1.0.0",
"description": "Assets for Bos Taurus",
"scripts": {
"cleanup": "rm -rf ./pkg-*.tgz && rm -rf ./build/ && rm -rf ./dist/",
"build": "../build-everything.sh ./genomeFileUrls.json",
"postbuild": "pl-pkg build && ([ -z \"${CI}\" ] || pl-pkg publish)"
},
"block-software": {
"entrypoints": {
"main": {
"asset": {
"type": "asset",
"registry": "platforma-open",
"root": "./indexed_genome/Bos_taurus"
}
}
}
},
"files": [
"dist/"
],
"license": "UNLICENSED",
"devDependencies": {
"@platforma-sdk/package-builder": "catalog:"
}
}
2 changes: 1 addition & 1 deletion build-everything.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,5 @@ JSON_FILE="${1}"
jq -r 'keys[]' "${JSON_FILE}" |
while read -r species; do
export PARENT_SCRIPT_PATH=$(dirname "$(realpath "$0")")
./run_star_index.sh "${species}" ./genomeFileUrls_All.json "./indexed_genome/${species}"
../run_star_index.sh "${species}" ./genomeFileUrls.json "./indexed_genome/${species}"
done
9 changes: 9 additions & 0 deletions caenorhabditis-elegans/genomeFileUrls.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"Caenorhabditis_elegans": {
"assembly_version": "WBcel235",
"read_length": 100,
"genomeSAindexNbases": 12,
"genome_url": "https://ftp.ensembl.org/pub/release-113/fasta/caenorhabditis_elegans/dna/Caenorhabditis_elegans.WBcel235.dna.toplevel.fa.gz",
"gtf_url": "https://ftp.ensembl.org/pub/release-113/gtf/caenorhabditis_elegans/Caenorhabditis_elegans.WBcel235.113.gtf.gz"
}
}
28 changes: 28 additions & 0 deletions caenorhabditis-elegans/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
"name": "@platforma-open/milaboratories.genome-assets.caenorhabditis-elegans",
"version": "1.0.0",
"description": "Assets for Caenorhabditis Elegans",
"scripts": {
"cleanup": "rm -rf ./pkg-*.tgz && rm -rf ./build/ && rm -rf ./dist/",
"build": "../build-everything.sh ./genomeFileUrls.json",
"postbuild": "pl-pkg build && ([ -z \"${CI}\" ] || pl-pkg publish)"
},
"block-software": {
"entrypoints": {
"main": {
"asset": {
"type": "asset",
"registry": "platforma-open",
"root": "./indexed_genome/Caenorhabditis_elegans"
}
}
}
},
"files": [
"dist/"
],
"license": "UNLICENSED",
"devDependencies": {
"@platforma-sdk/package-builder": "catalog:"
}
}
71 changes: 71 additions & 0 deletions catalogue/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
{

"name": "@platforma-open/milaboratories.genome-assets",
"version": "1.1.6",
"description": "STAR indexed genome references.",
"scripts": {
"cleanup": "rm -rf ./pkg-*.tgz && rm -rf ./dist/",
"build": "pl-pkg build"
},
"block-software": {
"entrypoints": {
"test-species": {
"reference": "@platforma-open/milaboratories.genome-assets.test-species/dist/tengo/asset/main.as.json"
},
"sus-scrofa": {
"reference": "@platforma-open/milaboratories.genome-assets.sus-scrofa/dist/tengo/asset/main.as.json"
},
"saccharomyces-cerevisiae": {
"reference": "@platforma-open/milaboratories.genome-assets.saccharomyces-cerevisiae/dist/tengo/asset/main.as.json"
},
"rattus-norvegicus": {
"reference": "@platforma-open/milaboratories.genome-assets.rattus-norvegicus/dist/tengo/asset/main.as.json"
},
"mus-musculus": {
"reference": "@platforma-open/milaboratories.genome-assets.mus-musculus/dist/tengo/asset/main.as.json"
},
"homo-sapiens": {
"reference": "@platforma-open/milaboratories.genome-assets.homo-sapiens/dist/tengo/asset/main.as.json"
},
"gallus-gallus": {
"reference": "@platforma-open/milaboratories.genome-assets.gallus-gallus/dist/tengo/asset/main.as.json"
},
"drosophila-melanogaster": {
"reference": "@platforma-open/milaboratories.genome-assets.drosophila-melanogaster/dist/tengo/asset/main.as.json"
},
"danio-rerio": {
"reference": "@platforma-open/milaboratories.genome-assets.danio-rerio/dist/tengo/asset/main.as.json"
},
"caenorhabditis-elegans": {
"reference": "@platforma-open/milaboratories.genome-assets.caenorhabditis-elegans/dist/tengo/asset/main.as.json"
},
"bos-taurus": {
"reference": "@platforma-open/milaboratories.genome-assets.bos-taurus/dist/tengo/asset/main.as.json"
},
"arabidopsis-thaliana": {
"reference": "@platforma-open/milaboratories.genome-assets.arabidopsis-thaliana/dist/tengo/asset/main.as.json"
}
}
},
"files": [
"dist/"
],
"license": "UNLICENSED",
"dependencies": {
"@platforma-open/milaboratories.genome-assets.test-species": "workspace:*",
"@platforma-open/milaboratories.genome-assets.sus-scrofa": "workspace:*",
"@platforma-open/milaboratories.genome-assets.saccharomyces-cerevisiae": "workspace:*",
"@platforma-open/milaboratories.genome-assets.rattus-norvegicus": "workspace:*",
"@platforma-open/milaboratories.genome-assets.mus-musculus": "workspace:*",
"@platforma-open/milaboratories.genome-assets.homo-sapiens": "workspace:*",
"@platforma-open/milaboratories.genome-assets.gallus-gallus": "workspace:*",
"@platforma-open/milaboratories.genome-assets.drosophila-melanogaster": "workspace:*",
"@platforma-open/milaboratories.genome-assets.danio-rerio": "workspace:*",
"@platforma-open/milaboratories.genome-assets.caenorhabditis-elegans": "workspace:*",
"@platforma-open/milaboratories.genome-assets.bos-taurus": "workspace:*",
"@platforma-open/milaboratories.genome-assets.arabidopsis-thaliana": "workspace:*"
},
"devDependencies": {
"@platforma-sdk/package-builder": "catalog:"
}
}
9 changes: 9 additions & 0 deletions danio-rerio/genomeFileUrls.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"Danio_rerio": {
"assembly_version": "GRCz11",
"read_length": 150,
"genomeSAindexNbases": 14,
"genome_url": "https://ftp.ensembl.org/pub/release-113/fasta/danio_rerio/dna/Danio_rerio.GRCz11.dna.toplevel.fa.gz",
"gtf_url": "https://ftp.ensembl.org/pub/release-113/gtf/danio_rerio/Danio_rerio.GRCz11.113.gtf.gz"
}
}
28 changes: 28 additions & 0 deletions danio-rerio/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
"name": "@platforma-open/milaboratories.genome-assets.danio-rerio",
"version": "1.0.0",
"description": "Assets for Danio Rerio",
"scripts": {
"cleanup": "rm -rf ./pkg-*.tgz && rm -rf ./build/ && rm -rf ./dist/",
"build": "../build-everything.sh ./genomeFileUrls.json",
"postbuild": "pl-pkg build && ([ -z \"${CI}\" ] || pl-pkg publish)"
},
"block-software": {
"entrypoints": {
"main": {
"asset": {
"type": "asset",
"registry": "platforma-open",
"root": "./indexed_genome/Danio_rerio"
}
}
}
},
"files": [
"dist/"
],
"license": "UNLICENSED",
"devDependencies": {
"@platforma-sdk/package-builder": "catalog:"
}
}
Loading

0 comments on commit d8840c2

Please sign in to comment.