Trying to stage input files

airflow/dags/docker_cwl_pod.yaml

@@ -8,7 +8,8 @@ spec:
 
   containers:
   - name: cwl-docker
-    image: ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.0.0-beta.2
+    # image: ghcr.io/unity-sds/unity-sps/sps-docker-cwl:2.0.0-beta.2
+    image: ghcr.io/unity-sds/unity-sps/sps-docker-cwl:develop27
     imagePullPolicy: Always
     command: ["/usr/share/cwl/docker_cwl_entrypoint.sh"]
     securityContext:

airflow/dags/sbg_L1_to_L2_e2e_cwl_step_by_step_dag.py

@@ -97,6 +97,7 @@ def setup(ti=None, **context):
 
     preprocess_dict = {
         "input_processing_labels": ["label1", "label2"],
+        # Note: must pass the path RELATIVE to te working directory - not the absolute path on the Pod
         "input_cmr_stac": "cmr-results.json",
         "input_unity_dapa_client": context["params"]["input_unity_dapa_client"],
         "input_unity_dapa_api": context["params"]["input_unity_dapa_api"],
@@ -151,11 +152,13 @@ def setup(ti=None, **context):
             persistent_volume_claim=k8s.V1PersistentVolumeClaimVolumeSource(claim_name="kpo-efs"),
         )
     ],
+    # do_xcom_push=True,
     dag=dag,
 )
 
 # Step: PREPROCESS
-SBG_PREPROCESS_CWL = "https://raw.githubusercontent.com/unity-sds/sbg-workflows/main/preprocess/sbg-preprocess-workflow.cwl"
+# SBG_PREPROCESS_CWL = "https://raw.githubusercontent.com/unity-sds/sbg-workflows/main/preprocess/sbg-preprocess-workflow.cwl"
+SBG_PREPROCESS_CWL = "https://raw.githubusercontent.com/LucaCinquini/sbg-workflows/devel/preprocess/sbg-preprocess-workflow.cwl"
 preprocess_task = KubernetesPodOperator(
     namespace=POD_NAMESPACE,
     name="Preprocess",

airflow/docker/cwl/docker_cwl_entrypoint.sh

@@ -49,6 +49,9 @@ pwd
 ls -lR
 cwl-runner --tmp-outdir-prefix "$PWD"/ --no-read-only "$cwl_workflow" "$job_args"
 ls -lR
+# FIXME
+# mkdir -p /airflow/xcom/
+# cp cmr-results.json /airflow/xcom/return.json
 deactivate
 
 # Stop Docker engine