add s3 destination to harmony job

podaac · Jun 17, 2024 · 80dc7dd · 80dc7dd
1 parent 258e752
commit 80dc7dd
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 34 deletions.
diff --git a/bignbit/submit_harmony_job.py b/bignbit/submit_harmony_job.py
@@ -46,8 +46,15 @@ def create_harmony_job(config):
     variable = current_item.get('id')
     big_config = config.get('big_config')
 
+    files = granule.get('files')
+    bucket_name = None
+    for file in files:
+        if file.get('type') == 'data':
+            file_path = os.path.dirname(file.get('key'))
+            destination_bucket_url = f's3://{file.get('bucket')}/{file_path}'
+
     harmony_client = utils.get_harmony_client(cmr_env)
-    harmony_request = generate_harmony_request(collection_concept_id, granule_concept_id, variable, big_config)
+    harmony_request = generate_harmony_request(collection_concept_id, granule_concept_id, variable, big_config, destination_bucket_url)
 
     job = harmony_client.submit(harmony_request)
     harmony_job = {
@@ -60,7 +67,7 @@ def create_harmony_job(config):
     return harmony_job
 
 
-def generate_harmony_request(collection_concept_id, granule_concept_id, variable, big_config):
+def generate_harmony_request(collection_concept_id, granule_concept_id, variable, big_config, destination_bucket_url):
     """Generate the harmony request to be made and return request object"""
 
     request = Request(
@@ -70,7 +77,8 @@ def generate_harmony_request(collection_concept_id, granule_concept_id, variable
         spatial=BBox(-180, -90, 180, 90),
         width=big_config['config']['width'],
         height=big_config['config']['height'],
-        format="image/png"
+        format="image/png",
+        destination_url=destination_bucket_url
     )
     return request
 

diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl
@@ -268,7 +268,7 @@
               {
                 "Variable":"$.payload.harmony_job_status",
                 "StringMatches":"successful",
-                "Next":"Copy Harmony Results to S3",
+                "Next":"JobSuccessful",
                 "Comment":"Job successful"
               },
               {
@@ -292,36 +292,8 @@
             ],
             "Default":"Wait 20 Seconds"
           },
-          "Copy Harmony Results to S3":{
-            "Type":"Task",
-            "Resource":"${CopyHarmonyOutputToS3Lambda}",
-            "Parameters":{
-              "cma":{
-                "event.$":"$",
-                "task_config":{
-                  "cmr_environment":"{$.meta.cmr.cmrEnvironment}",
-                  "harmony_job":"{$.payload.harmony_job.job}",
-                  "current_item":"{$.current_item}",
-                  "cumulus_message":{
-                    "input":"{$.payload}"
-                  }
-                }
-              }
-            },
-            "Retry":[
-              {
-                "ErrorEquals":[
-                  "Lambda.ServiceException",
-                  "Lambda.AWSLambdaException",
-                  "Lambda.SdkClientException",
-                  "Lambda.TooManyRequestsException"
-                ],
-                "IntervalSeconds":2,
-                "MaxAttempts":6,
-                "BackoffRate":2
-              }
-            ],
-            "End":true
+          "JobSuccessful":{
+            "Type":"Succeed"
           },
           "Fail":{
             "Type":"Fail"