dvc.lock

schema: '2.0'
stages:
  extract_pref_labels:
    cmd: Rscript src/extract_pref_labels.r -g vocab/gnd.ttl -o vocab/gnd_pref_labels.arrow
    deps:
    - path: src/extract_pref_labels.r
      hash: md5
      md5: 61e10b4697a6f31dc84554c0891764d1
      size: 978
    - path: vocab/gnd.ttl
      hash: md5
      md5: a6a6db61f89d43c58dc2f5d4d6180c39
      size: 53388861
    outs:
    - path: vocab/gnd_pref_labels.arrow
      hash: md5
      md5: ebaeefdf6789769605bbe4feb6c404ca
      size: 7030402
  index_corpus@test:
    cmd:
    - mkdir -p results/test/
    - "python src/index_corpus.py  --corpus corpora/ft30k/test.tsv.gz --index corpora/ft30k/test.arrow\
      \  --pref_labels vocab/gnd_pref_labels.arrow --alpha 1.0 --top_k 100 --chunk_size
      500 --out results/test/predictions.arrow --n_jobs 40 --task ft30k --evalset
      test\n"
    deps:
    - path: results/test/collection_name.txt
      hash: md5
      md5: 56e2c40b7e81c2bab863e6ac0c9953fe
      size: 22
    - path: src/index_corpus.py
      hash: md5
      md5: 9cdfeaa49e04e1921c2a166cd3bffe47
      size: 7353
    params:
      params.yaml:
        search.alpha: 1.0
        search.chunk_size: 500
        search.top_k: 100
        task.name: ft30k
    outs:
    - path: results/test/predictions.arrow
      hash: md5
      md5: f2b85660b86a6dd9dc0d83f8683cec85
      size: 23655034
  compute_metrics@test:
    cmd:
    - mkdir -p results/test/
    - "Rscript src/compute_metrics.r  --kind ft  --evalset test  --test_index corpora/ftoa/test.arrow
      --predictions results/test/predictions.arrow --out_folder results/test/\n"
    deps:
    - path: corpora/ftoa/test.arrow
      hash: md5
      md5: edd596ba43b45b2c9641798990418636
      size: 20682
    - path: corpora/ground-truth.arrow
      hash: md5
      md5: 717791c69fedc7d364b38fdaf99fbce5
      size: 55898370
    - path: results/test/predictions.arrow
      hash: md5
      md5: 06ff139d081d40ad472e8a94d4a755c3
      size: 1483066
    - path: src/compute_metrics.r
      hash: md5
      md5: ba86f225aa8f62b37405b1d394fe366d
      size: 5420
    params:
      params.yaml:
        task.kind: ft
        task.name: ftoa
    outs:
    - path: results//test/prec_at_5.json
      hash: md5
      md5: c91a9213a59af99771d4e712ee027d28
      size: 37
    - path: results//test/rec_at_5.json
      hash: md5
      md5: a0f7eddddbd49252bf5336f224d3b1da
      size: 36
    - path: results/test/f1_at_5.json
      hash: md5
      md5: e27874956c5a3ee6a3aac853a47a0295
      size: 34
    - path: results/test/pr_auc.json
      hash: md5
      md5: eee6397cabd24ec4c19db8b133941bd1
      size: 29
    - path: results/test/pr_curve.csv
      hash: md5
      md5: f72c148eab52f67169875b73b7cdf2fa
      size: 3556
    - path: results/test/pr_curve_plot.svg
      hash: md5
      md5: 55b08f4d85a7d1e0c7ef8d4327dcdcc6
      size: 17770
    - path: results/test/rprec_at_5.json
      hash: md5
      md5: 716bec81b0c62a492d3058dcfe3185a5
      size: 38
  index_corpus@validate:
    cmd:
    - mkdir -p results/validate/
    - "python src/index_corpus.py  --corpus corpora/ft30k/validate.tsv.gz --index
      corpora/ft30k/validate.arrow  --pref_labels vocab/gnd_pref_labels.arrow --alpha
      1.0 --top_k 100 --chunk_size 500 --out results/validate/predictions.arrow --n_jobs
      40 --task ft30k --evalset validate\n"
    deps:
    - path: results/validate/collection_name.txt
      hash: md5
      md5: fece339858b5c76ea4a036677099c3aa
      size: 26
    - path: src/index_corpus.py
      hash: md5
      md5: 9cdfeaa49e04e1921c2a166cd3bffe47
      size: 7353
    params:
      params.yaml:
        search.alpha: 1.0
        search.chunk_size: 500
        search.top_k: 100
        task.name: ft30k
    outs:
    - path: results/validate/predictions.arrow
      hash: md5
      md5: f5161c3b13b4797a9f69c271513913fb
      size: 68127674
  compute_metrics@validate:
    cmd:
    - mkdir -p results/validate/
    - "Rscript src/compute_metrics.r  --kind ft  --evalset validate  --test_index
      corpora/ftoa/validate.arrow --predictions results/validate/predictions.arrow
      --out_folder results/validate/\n"
    deps:
    - path: corpora/ftoa/validate.arrow
      hash: md5
      md5: fed3c0bb52226e6a785ff9049069e949
      size: 31146
    - path: corpora/ground-truth.arrow
      hash: md5
      md5: 717791c69fedc7d364b38fdaf99fbce5
      size: 55898370
    - path: results/validate/predictions.arrow
      hash: md5
      md5: e61f5fafec5bdd7fc10387c57087f4d7
      size: 2380898
    - path: src/compute_metrics.r
      hash: md5
      md5: ba86f225aa8f62b37405b1d394fe366d
      size: 5420
    params:
      params.yaml:
        task.kind: ft
        task.name: ftoa
    outs:
    - path: results//validate/prec_at_5.json
      hash: md5
      md5: 6fa7daf3fb17eb1cfc6b88efa37daae4
      size: 37
    - path: results//validate/rec_at_5.json
      hash: md5
      md5: 1c61106f2789e0ec6d6dde56e287a44a
      size: 36
    - path: results/validate/f1_at_5.json
      hash: md5
      md5: 7995db16b10eda18a076f2bde1895c40
      size: 35
    - path: results/validate/pr_auc.json
      hash: md5
      md5: 6c4d3266d59a0ef91e58fd8fe33feb67
      size: 29
    - path: results/validate/pr_curve.csv
      hash: md5
      md5: 7038a3f4314849c9386c2f33305d6a30
      size: 3515
    - path: results/validate/pr_curve_plot.svg
      hash: md5
      md5: d3b577cc9217c614a72f687eadd9a2da
      size: 17758
    - path: results/validate/rprec_at_5.json
      hash: md5
      md5: 2f1f5d3ff956c536589127ea5dafa5fe
      size: 38
  create_collection@test:
    cmd:
    - mkdir -p results/test/
    - python src/create_collection.py --task ft30k --evalset test --collection_name_file
      results/test/collection_name.txt --overwrite false
    deps:
    - path: src/create_collection.py
      hash: md5
      md5: d560ab5d62f6b276cbb955df6bc2c548
      size: 3389
    params:
      params.yaml:
        collection.overwrite: false
        task.name: ft30k
    outs:
    - path: results/test/collection_name.txt
      hash: md5
      md5: 56e2c40b7e81c2bab863e6ac0c9953fe
      size: 22
  create_collection@validate:
    cmd:
    - mkdir -p results/validate/
    - python src/create_collection.py --task ft30k --evalset validate --collection_name_file
      results/validate/collection_name.txt --overwrite false
    deps:
    - path: src/create_collection.py
      hash: md5
      md5: d560ab5d62f6b276cbb955df6bc2c548
      size: 3389
    params:
      params.yaml:
        collection.overwrite: false
        task.name: ft30k
    outs:
    - path: results/validate/collection_name.txt
      hash: md5
      md5: fece339858b5c76ea4a036677099c3aa
      size: 26
  create_collection@train:
    cmd:
    - mkdir -p results/train/
    - python src/create_collection.py --task ft30k --evalset train --collection_name_file
      results/train/collection_name.txt --overwrite false
    deps:
    - path: src/create_collection.py
      hash: md5
      md5: d560ab5d62f6b276cbb955df6bc2c548
      size: 3389
    params:
      params.yaml:
        collection.overwrite: false
        task.name: ft30k
    outs:
    - path: results/train/collection_name.txt
      hash: md5
      md5: 98a683578d4202827f9b528be30bdaa1
      size: 23
  generate_candidates@train:
    cmd:
    - mkdir -p results/train/
    - "python src/index_corpus.py  --chunks results/train/chunks.txt --chunk_index
      results/train/chunk_index.arrow  --chunk_embeddings results/train/chunk_embeddings.npy
      --alpha 0.625 --top_k 100 --n_hits 20 --output results/train/candidates.arrow
      --n_jobs 40\n"
    deps:
    - path: results/train/chunk_embeddings.npy
      hash: md5
      md5: ab9351dbc1aaecc63441049a5557e232
      size: 1411264640
    - path: results/train/chunk_index.arrow
      hash: md5
      md5: ddc7396ad75e076eeb37e16b7c99feb0
      size: 15142650
    - path: results/train/chunks.txt
      hash: md5
      md5: f40341ae6608d4233579e6f398834a1a
      size: 100218015
    - path: src/index_corpus.py
      hash: md5
      md5: fd70f735e56aae7274717eceff304217
      size: 8311
    - path: vocab/gnd204k_w_altlabels.arrow
      hash: md5
      md5: bfc5257f2c7ce8c018a0938bfad2b340
      size: 17026962
    params:
      params.yaml:
        search.alpha: 0.625
        search.n_hits: 20
        search.top_k: 100
        task.name: ftoa
        vocab_config.collection_name: gnd204k_w_altlabels_no_phrase_jina_1024_retrieval
    outs:
    - path: results/train/candidates.arrow
      hash: md5
      md5: 347942ae3584e8265af6ba0ae2a0f881
      size: 15505402
  generate_candidates@validate:
    cmd:
    - mkdir -p results/validate/
    - "python src/index_corpus.py  --chunks results/validate/chunks.txt --chunk_index
      results/validate/chunk_index.arrow  --chunk_embeddings results/validate/chunk_embeddings.npy
      --alpha 0.625 --top_k 100 --n_hits 20 --output results/validate/candidates.arrow
      --n_jobs 40\n"
    deps:
    - path: results/validate/chunk_embeddings.npy
      hash: md5
      md5: 78d889f991222975caf2e08096821a7a
      size: 390213760
    - path: results/validate/chunk_index.arrow
      hash: md5
      md5: 9d93d89bc82c0db8316e938ef3776c79
      size: 4190058
    - path: results/validate/chunks.txt
      hash: md5
      md5: 979fa1434a30120b543a93891fd80676
      size: 27427659
    - path: src/index_corpus.py
      hash: md5
      md5: fd70f735e56aae7274717eceff304217
      size: 8311
    - path: vocab/gnd204k_w_altlabels.arrow
      hash: md5
      md5: bfc5257f2c7ce8c018a0938bfad2b340
      size: 17026962
    params:
      params.yaml:
        search.alpha: 0.625
        search.n_hits: 20
        search.top_k: 100
        task.name: ftoa
        vocab_config.collection_name: gnd204k_w_altlabels_no_phrase_jina_1024_retrieval
    outs:
    - path: results/validate/candidates.arrow
      hash: md5
      md5: 97e6821bb7dadb59cff4735ea7c53ba3
      size: 4250722
  generate_candidates@test:
    cmd:
    - mkdir -p results/test/
    - "python src/index_corpus.py  --chunks results/test/chunks.txt --chunk_index
      results/test/chunk_index.arrow  --chunk_embeddings results/test/chunk_embeddings.npy
      --alpha 0.625 --top_k 100 --n_hits 20 --output results/test/candidates.arrow
      --n_jobs 40\n"
    deps:
    - path: results/test/chunk_embeddings.npy
      hash: md5
      md5: d1ba3f2f3b0a3c852e2588aee8833739
      size: 235425920
    - path: results/test/chunk_index.arrow
      hash: md5
      md5: d3130a021bb94ba155169fe0fa91b888
      size: 2541346
    - path: results/test/chunks.txt
      hash: md5
      md5: 64a5d582e1debe87b0a6f45adb1e78e5
      size: 16563657
    - path: src/index_corpus.py
      hash: md5
      md5: fd70f735e56aae7274717eceff304217
      size: 8311
    - path: vocab/gnd204k_w_altlabels.arrow
      hash: md5
      md5: bfc5257f2c7ce8c018a0938bfad2b340
      size: 17026962
    params:
      params.yaml:
        search.alpha: 0.625
        search.n_hits: 20
        search.top_k: 100
        task.name: ftoa
        vocab_config.collection_name: gnd204k_w_altlabels_no_phrase_jina_1024_retrieval
    outs:
    - path: results/test/candidates.arrow
      hash: md5
      md5: 1ccb9c1e1944206b00907b2d0307e83e
      size: 2636234
  train_ranker:
    cmd:
    - mkdir -p results/train/
    - "Rscript src/train.r --ground_truth corpora/ground-truth.arrow --train_index
      corpora/ftoa/train.arrow --max_docs 30000 --train_candidates results/train/candidates.arrow
      --label_distribution assets/gnd-label-distribution.arrow --model_file results/train/model.rds
      --n_rounds 812 --subsample 0.62 --interaction_depth 7 --shrinkage 0.023 --verbose
      true --n_jobs 40\n"
    deps:
    - path: assets/gnd-label-distribution.arrow
      hash: md5
      md5: cf88a4cfcc09c976d6ca2dc193ae9efc
      size: 2316610
    - path: corpora/ftoa/train.arrow
      hash: md5
      md5: e95f42747020a8706f0185dbdce349c6
      size: 103066
    - path: corpora/ground-truth.arrow
      hash: md5
      md5: 717791c69fedc7d364b38fdaf99fbce5
      size: 55898370
    - path: results/train/candidates.arrow
      hash: md5
      md5: 347942ae3584e8265af6ba0ae2a0f881
      size: 15505402
    - path: src/prepare_data_for_gbm.r
      hash: md5
      md5: 8fbbba82ec32cfef38f3d8bf186f6a90
      size: 2047
    - path: src/train.r
      hash: md5
      md5: bcba722e41a9b7926b8da4f72f811f08
      size: 4131
    params:
      params.yaml:
        train.xgb.interaction_depth: 7
        train.xgb.n_rounds: 812
        train.xgb.shrinkage: 0.023
        train.xgb.subsample: 0.62
    outs:
    - path: results/train/model.rds
      hash: md5
      md5: 0df0fe48c64ade8c055365404652d650
      size: 1950397
  rank_candidates@validate:
    cmd:
    - mkdir -p results/validate/
    - "Rscript src/predict.r  --model_file results/train/model.rds --test_index corpora/ftoa/validate.arrow
      --test_candidates results/validate/candidates.arrow --max_docs 30000 --label_distribution
      assets/gnd-label-distribution.arrow --top_k 100 --out_file results/validate/predictions.arrow\n"
    deps:
    - path: assets/gnd-label-distribution.arrow
      hash: md5
      md5: cf88a4cfcc09c976d6ca2dc193ae9efc
      size: 2316610
    - path: corpora/ftoa/validate.arrow
      hash: md5
      md5: fed3c0bb52226e6a785ff9049069e949
      size: 31146
    - path: results/train/model.rds
      hash: md5
      md5: 0df0fe48c64ade8c055365404652d650
      size: 1950397
    - path: results/validate/candidates.arrow
      hash: md5
      md5: 97e6821bb7dadb59cff4735ea7c53ba3
      size: 4250722
    - path: src/predict.r
      hash: md5
      md5: f3346c8e00029e01f83ffb7140558e3f
      size: 2689
    - path: src/prepare_data_for_gbm.r
      hash: md5
      md5: 8fbbba82ec32cfef38f3d8bf186f6a90
      size: 2047
    params:
      params.yaml:
        chunk_opts.max_docs: 30000
        predict.top_k: 100
    outs:
    - path: results/validate/predictions.arrow
      hash: md5
      md5: e61f5fafec5bdd7fc10387c57087f4d7
      size: 2380898
  rank_candidates@test:
    cmd:
    - mkdir -p results/test/
    - "Rscript src/predict.r  --model_file results/train/model.rds --test_index corpora/ftoa/test.arrow
      --test_candidates results/test/candidates.arrow --max_docs 30000 --label_distribution
      assets/gnd-label-distribution.arrow --top_k 100 --out_file results/test/predictions.arrow\n"
    deps:
    - path: assets/gnd-label-distribution.arrow
      hash: md5
      md5: cf88a4cfcc09c976d6ca2dc193ae9efc
      size: 2316610
    - path: corpora/ftoa/test.arrow
      hash: md5
      md5: edd596ba43b45b2c9641798990418636
      size: 20682
    - path: results/test/candidates.arrow
      hash: md5
      md5: 1ccb9c1e1944206b00907b2d0307e83e
      size: 2636234
    - path: results/train/model.rds
      hash: md5
      md5: 0df0fe48c64ade8c055365404652d650
      size: 1950397
    - path: src/predict.r
      hash: md5
      md5: f3346c8e00029e01f83ffb7140558e3f
      size: 2689
    - path: src/prepare_data_for_gbm.r
      hash: md5
      md5: 8fbbba82ec32cfef38f3d8bf186f6a90
      size: 2047
    params:
      params.yaml:
        chunk_opts.max_docs: 30000
        predict.top_k: 100
    outs:
    - path: results/test/predictions.arrow
      hash: md5
      md5: 06ff139d081d40ad472e8a94d4a755c3
      size: 1483066
  create_vocab_collection:
    cmd:
    - "python src/create_vocab_collection.py \\\n  --ttl_file vocab/gnd.ttl \\\n \
      \ --collection_name gnd204k_w_altlabels_no_phrase_jina_1024_retrieval \\\n \
      \ --overwrite false \\\n  --TEI_port 8090 \\\n  --phrase \"\" \\\n  --use_altLabels
      true \\\n  --arrow_out vocab/gnd204k_w_altlabels.arrow\n"
    deps:
    - path: src/create_vocab_collection.py
      hash: md5
      md5: 0479a0d7f4539a01e1096a9ea179c4c0
      size: 7074
    - path: src/generate_embeddings.py
      hash: md5
      md5: 06f1d956bf4af6164d5bd63bc4f11d17
      size: 1668
    - path: vocab/gnd.ttl
      hash: md5
      md5: a6a6db61f89d43c58dc2f5d4d6180c39
      size: 53388861
    params:
      params.yaml:
        general.TEI_port: 8090
        general.embedding_dim: 1024
        general.embedding_model: jinaai/jina-embeddings-v3
        vocab_config.collection_name: gnd204k_w_altlabels_no_phrase_jina_1024_retrieval
        vocab_config.overwrite: false
        vocab_config.phrase: ''
        vocab_config.use_altLabels: true
    outs:
    - path: vocab/gnd204k_w_altlabels.arrow
      hash: md5
      md5: bfc5257f2c7ce8c018a0938bfad2b340
      size: 17026962
  chunk_texts@test:
    cmd:
    - mkdir -p results/test/
    - "python src/chunk_texts.py  --corpus corpora/ftoa/test.tsv.gz  --index corpora/ftoa/test.arrow
      --chunk_texts_output results/test/chunks.txt --chunk_index_output results/test/chunk_index.arrow
      --chunk_size 100 --max_chunks 100 --max_docs 30000 --max_sentences_per_doc 600
      --n_jobs 10\n"
    deps:
    - path: corpora/ftoa/test.arrow
      hash: md5
      md5: edd596ba43b45b2c9641798990418636
      size: 20682
    - path: corpora/ftoa/test.tsv.gz
      hash: md5
      md5: 1bb18b431fb9c9ed37afbb1485710e8e
      size: 6440270
    - path: src/chunk_texts.py
      hash: md5
      md5: cabeb7bbf48903154d121c1bed3d2aeb
      size: 5516
    params:
      params.yaml:
        chunk_opts.chunk_size: 100
        chunk_opts.max_chunks: 100
        chunk_opts.max_docs: 30000
        chunk_opts.max_sentences_per_doc: 600
    outs:
    - path: results/test/chunk_index.arrow
      hash: md5
      md5: d3130a021bb94ba155169fe0fa91b888
      size: 2541346
    - path: results/test/chunks.txt
      hash: md5
      md5: 64a5d582e1debe87b0a6f45adb1e78e5
      size: 16563657
  chunk_texts@train:
    cmd:
    - mkdir -p results/train/
    - "python src/chunk_texts.py  --corpus corpora/ftoa/train.tsv.gz  --index corpora/ftoa/train.arrow
      --chunk_texts_output results/train/chunks.txt --chunk_index_output results/train/chunk_index.arrow
      --chunk_size 100 --max_chunks 100 --max_docs 30000 --max_sentences_per_doc 600
      --n_jobs 10\n"
    deps:
    - path: corpora/ftoa/train.arrow
      hash: md5
      md5: e95f42747020a8706f0185dbdce349c6
      size: 103066
    - path: corpora/ftoa/train.tsv.gz
      hash: md5
      md5: ae5c8e95555deb7aa8c69b0ecddb33f5
      size: 37017585
    - path: src/chunk_texts.py
      hash: md5
      md5: cabeb7bbf48903154d121c1bed3d2aeb
      size: 5516
    params:
      params.yaml:
        chunk_opts.chunk_size: 100
        chunk_opts.max_chunks: 100
        chunk_opts.max_docs: 30000
        chunk_opts.max_sentences_per_doc: 600
    outs:
    - path: results/train/chunk_index.arrow
      hash: md5
      md5: ddc7396ad75e076eeb37e16b7c99feb0
      size: 15142650
    - path: results/train/chunks.txt
      hash: md5
      md5: f40341ae6608d4233579e6f398834a1a
      size: 100218015
  chunk_texts@validate:
    cmd:
    - mkdir -p results/validate/
    - "python src/chunk_texts.py  --corpus corpora/ftoa/validate.tsv.gz  --index corpora/ftoa/validate.arrow
      --chunk_texts_output results/validate/chunks.txt --chunk_index_output results/validate/chunk_index.arrow
      --chunk_size 100 --max_chunks 100 --max_docs 30000 --max_sentences_per_doc 600
      --n_jobs 10\n"
    deps:
    - path: corpora/ftoa/validate.arrow
      hash: md5
      md5: fed3c0bb52226e6a785ff9049069e949
      size: 31146
    - path: corpora/ftoa/validate.tsv.gz
      hash: md5
      md5: 086f20d34e8a56079e5b5fe83e823038
      size: 10194799
    - path: src/chunk_texts.py
      hash: md5
      md5: cabeb7bbf48903154d121c1bed3d2aeb
      size: 5516
    params:
      params.yaml:
        chunk_opts.chunk_size: 100
        chunk_opts.max_chunks: 100
        chunk_opts.max_docs: 30000
        chunk_opts.max_sentences_per_doc: 600
    outs:
    - path: results/validate/chunk_index.arrow
      hash: md5
      md5: 9d93d89bc82c0db8316e938ef3776c79
      size: 4190058
    - path: results/validate/chunks.txt
      hash: md5
      md5: 979fa1434a30120b543a93891fd80676
      size: 27427659
  embed_chunks@test:
    cmd:
    - mkdir -p results/test/
    - "python src/embed_chunks.py  --chunks results/test/chunks.txt  --out results/test/chunk_embeddings.npy\n"
    deps:
    - path: results/test/chunks.txt
      hash: md5
      md5: 64a5d582e1debe87b0a6f45adb1e78e5
      size: 16563657
    - path: src/embed_chunks.py
      hash: md5
      md5: 1226d6cbbe506bef5952845f05ade0f0
      size: 646
    - path: src/generate_embeddings.py
      hash: md5
      md5: 06f1d956bf4af6164d5bd63bc4f11d17
      size: 1668
    params:
      params.yaml:
        general.embedding_dim: 1024
        general.embedding_model: jinaai/jina-embeddings-v3
    outs:
    - path: results/test/chunk_embeddings.npy
      hash: md5
      md5: d1ba3f2f3b0a3c852e2588aee8833739
      size: 235425920
  embed_chunks@train:
    cmd:
    - mkdir -p results/train/
    - "python src/embed_chunks.py  --chunks results/train/chunks.txt  --out results/train/chunk_embeddings.npy\n"
    deps:
    - path: results/train/chunks.txt
      hash: md5
      md5: f40341ae6608d4233579e6f398834a1a
      size: 100218015
    - path: src/embed_chunks.py
      hash: md5
      md5: 1226d6cbbe506bef5952845f05ade0f0
      size: 646
    - path: src/generate_embeddings.py
      hash: md5
      md5: 06f1d956bf4af6164d5bd63bc4f11d17
      size: 1668
    params:
      params.yaml:
        general.embedding_dim: 1024
        general.embedding_model: jinaai/jina-embeddings-v3
    outs:
    - path: results/train/chunk_embeddings.npy
      hash: md5
      md5: ab9351dbc1aaecc63441049a5557e232
      size: 1411264640
  embed_chunks@validate:
    cmd:
    - mkdir -p results/validate/
    - "python src/embed_chunks.py  --chunks results/validate/chunks.txt  --out results/validate/chunk_embeddings.npy\n"
    deps:
    - path: results/validate/chunks.txt
      hash: md5
      md5: 979fa1434a30120b543a93891fd80676
      size: 27427659
    - path: src/embed_chunks.py
      hash: md5
      md5: 1226d6cbbe506bef5952845f05ade0f0
      size: 646
    - path: src/generate_embeddings.py
      hash: md5
      md5: 06f1d956bf4af6164d5bd63bc4f11d17
      size: 1668
    params:
      params.yaml:
        general.embedding_dim: 1024
        general.embedding_model: jinaai/jina-embeddings-v3
    outs:
    - path: results/validate/chunk_embeddings.npy
      hash: md5
      md5: 78d889f991222975caf2e08096821a7a
      size: 390213760