harmony.harmony.ProcessingFailedException: WorkItem failed: podaac/concise:0.10.0: Error in file '/worker/podaac/merger/preprocess_worker.py', line 215, in function '_run_multi_core': Preprocessing failed - exit code: 1
collection_concept_id = 'C3309450672-POCLOUD'
harmony_env = <Environment.PROD: 4>
bearer_token = 'eyJ0eXAiOiJKV1QiLCJvcmlnaW4iOiJFYXJ0aGRhdGEgTG9naW4iLCJzaWciOiJlZGxqd3RwdWJrZXlfb3BzIiwiYWxnIjoiUlMyNTYifQ.eyJ0eXBlIj...NVVKFYPVfHyjSN89lHeGrsiqbnmnLiwCwJTisRjrsC3_l2KufgrgEu5ms0yEcq1Awi3iKCCooIiCJR_93oD_8tIkvUtfHRphaybBjBf9XsWhmo8DXD2y5w'

    @pytest.mark.timeout(600)
    def test_concatenate(collection_concept_id, harmony_env, bearer_token):
    
        max_results = 2
        harmony_client = harmony.Client(env=harmony_env, token=bearer_token)
        collection = harmony.Collection(id=collection_concept_id)
        latest_granule_ids = get_latest_granules(collection_concept_id, max_results, harmony_env, bearer_token)
    
        if latest_granule_ids is None:
            if harmony_env == harmony.config.Environment.UAT:
                pytest.skip(f"No granules found for UAT collection {collection_concept_id}")
            raise Exception('Bad Request', 'Error: No matching granules found.')
    
        request = harmony.Request(
            collection=collection,
            concatenate=True,
            max_results=max_results,
            granule_id=latest_granule_ids,
            skip_preview=True,
            format="application/x-netcdf4",
        )
    
        request.is_valid()
    
        logging.info("Sending harmony request %s", harmony_client.request_as_url(request))
    
        try:
            job1_id = harmony_client.submit(request)
        except Exception as ex:
            if str(ex) == "('Bad Request', 'Error: No matching granules found.')":
                if harmony_env == harmony.config.Environment.UAT:
                    pytest.skip(f"No granules found for UAT collection {collection_concept_id}")
            raise ex
    
        logging.info(f'\n{job1_id}')
    
        logging.info(harmony_client.status(job1_id))
    
        logging.info('\nWaiting for the job to finish')
    
        results = harmony_client.result_json(job1_id)
    
        logging.info('\nDownloading results:')
    
        futures = harmony_client.download_all(job1_id)
>       file_names = [f.result() for f in futures]

verify_collection.py:259: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
verify_collection.py:259: in <listcomp>
    file_names = [f.result() for f in futures]
../../../../.cache/pypoetry/virtualenvs/concise-autotest-tfGktGnJ-py3.10/lib/python3.10/site-packages/harmony/harmony.py:1387: in download_all
    for url in self.result_urls(job_id_or_result_json, show_progress=False) or []:
../../../../.cache/pypoetry/virtualenvs/concise-autotest-tfGktGnJ-py3.10/lib/python3.10/site-packages/harmony/harmony.py:1231: in result_urls
    for page in self._result_pages(job_id, show_progress, link_type):
../../../../.cache/pypoetry/virtualenvs/concise-autotest-tfGktGnJ-py3.10/lib/python3.10/site-packages/harmony/harmony.py:1206: in _result_pages
    self.wait_for_processing(job_id, show_progress)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <harmony.harmony.Client object at 0x7f2fcba162f0>
job_id = 'e0df4d04-5a3f-4588-a2ff-e3023f9f87ca', show_progress = False

    def wait_for_processing(self, job_id: str, show_progress: bool = False) -> None:
        """Retrieve a submitted job's completion status in percent.
    
        Args:
            job_id: UUID string for the job you wish to interrogate.
    
        Returns:
            The job's processing progress as a percentage.
    
        :raises
            Exception: This can happen if an invalid job_id is provided or Harmony services
            can't be reached.
        """
        # How often to refresh the screen for progress updates and animating spinners.
        ui_update_interval = 0.33  # in seconds
        running_w_errors_logged = False
    
        intervals = round(self.check_interval / ui_update_interval)
        if show_progress:
            with progressbar.ProgressBar(max_value=100, widgets=progressbar_widgets) as bar:
                progress = 0
                while progress < 100:
                    progress, status, message = self.progress(job_id)
                    if status == 'failed':
                        raise ProcessingFailedException(job_id, message)
                    if status == 'canceled':
                        print('Job has been canceled.')
                        break
                    if status == 'paused':
                        print('\nJob has been paused. Call `resume()` to resume.', file=sys.stderr)
                        break
                    if (not running_w_errors_logged and status == 'running_with_errors'):
                        print('\nJob is running with errors.', file=sys.stderr)
                        running_w_errors_logged = True
    
                    # This gets around an issue with progressbar. If we update() with 0, the
                    # output shows up as "N/A". If we update with, e.g. 0.1, it rounds down or
                    # truncates to 0 but, importantly, actually displays that.
                    if progress == 0:
                        progress = 0.1
    
                    for _ in range(intervals):
                        bar.update(progress)  # causes spinner to rotate even when no data change
                        sys.stdout.flush()  # ensures correct behavior in Jupyter notebooks
                        if progress >= 100:
                            break
                        else:
                            time.sleep(ui_update_interval)
        else:
            progress = 0
            while progress < 100:
                progress, status, message = self.progress(job_id)
                if status == 'failed':
>                   raise ProcessingFailedException(job_id, message)
E                   harmony.harmony.ProcessingFailedException: WorkItem failed: podaac/concise:0.10.0: Error in file '/worker/podaac/merger/preprocess_worker.py', line 215, in function '_run_multi_core': Preprocessing failed - exit code: 1

../../../../.cache/pypoetry/virtualenvs/concise-autotest-tfGktGnJ-py3.10/lib/python3.10/site-packages/harmony/harmony.py:1146: ProcessingFailedException
--------------------------------- Captured Log ---------------------------------
INFO     root:verify_collection.py:238 Sending harmony request https://harmony.earthdata.nasa.gov/C3309450672-POCLOUD/ogc-api-coverages/1.0.0/collections/parameter_vars/coverage/rangeset?forceAsync=true&granuleId=G3408711413-POCLOUD&granuleId=G3408711560-POCLOUD&format=application%2Fx-netcdf4&maxResults=2&concatenate=true&skipPreview=true&variable=all
INFO     root:verify_collection.py:248 
e0df4d04-5a3f-4588-a2ff-e3023f9f87ca
INFO     root:verify_collection.py:250 {'status': 'running', 'message': 'The job is being processed', 'progress': 0, 'created_at': datetime.datetime(2025, 3, 3, 8, 30, 36, 286000, tzinfo=tzlocal()), 'updated_at': datetime.datetime(2025, 3, 3, 8, 30, 36, 286000, tzinfo=tzlocal()), 'created_at_local': '2025-03-03T08:30:36+00:00', 'updated_at_local': '2025-03-03T08:30:36+00:00', 'request': 'https://harmony.earthdata.nasa.gov/C3309450672-POCLOUD/ogc-api-coverages/1.0.0/collections/parameter_vars/coverage/rangeset?forceAsync=true&granuleId=G3408711413-POCLOUD%2CG3408711560-POCLOUD&format=application%2Fx-netcdf4&maxResults=2&concatenate=true&skipPreview=true&variable=all', 'num_input_granules': 2, 'data_expiration': datetime.datetime(2025, 4, 2, 8, 30, 36, 286000, tzinfo=tzlocal()), 'data_expiration_local': '2025-04-02T08:30:36+00:00'}
INFO     root:verify_collection.py:252 
Waiting for the job to finish
INFO     root:verify_collection.py:256 
Downloading results: