Adding debug task

airflow/dags/docker_cwl_pod.yaml

@@ -11,7 +11,7 @@ spec:
   serviceAccountName: airflow-worker
   containers:
   - name: cwl-docker
-    image: ghcr.io/unity-sds/unity-sps/sps-docker-cwl:development-tag2
+    image: ghcr.io/unity-sds/unity-sps/sps-docker-cwl:development-tag6
     command: ["/usr/share/cwl/docker_cwl_entrypoint.sh"]
     args: ["https://raw.githubusercontent.com/unity-sds/sbg-workflows/main/preprocess/sbg-preprocess-workflow.cwl", '{
   "input_processing_labels": [

airflow/dags/sbg_preprocess_cwl_dag.py

@@ -12,6 +12,7 @@
 # The Kubernetes Pod that executes the CWL-Docker container
 # Must use elevated privileges to start/stop the Docker engine
 POD_TEMPLATE_FILE = "/opt/airflow/dags/docker_cwl_pod.yaml"
+POD_TEMPLATE_FILE2 = "/opt/airflow/dags/aws_cli_pod.yaml"
 
 # The Kubernetes namespace within which the Pod is run (it must already exist)
 POD_NAMESPACE = "airflow"
@@ -47,6 +48,21 @@
 
           })
 
+awscli_task = KubernetesPodOperator(
+    namespace=POD_NAMESPACE,
+    name="awscli-task",
+    is_delete_operator_pod=True,
+    hostnetwork=False,
+    startup_timeout_seconds=1000,
+    get_logs=True,
+    task_id="awscli-task",
+    full_pod_spec=k8s.V1Pod(
+        metadata=k8s.V1ObjectMeta(
+            name='awscli-pod-' + uuid.uuid4().hex),
+    ),
+    pod_template_file=POD_TEMPLATE_FILE2,
+    dag=dag)
+
 # Task that serializes the job arguments into a JSON string
 def setup(ti=None, **context):
     task_dict = {
@@ -84,4 +100,5 @@ def setup(ti=None, **context):
     # resources={"request_memory": "512Mi", "limit_memory": "1024Mi"},
     dag=dag)
 
-setup_task >> cwl_task
+
+awscli_task >> setup_task >> cwl_task