Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow to copy downladed files to alternate location #35

Merged
merged 5 commits into from
Feb 14, 2025
Merged

Allow to copy downladed files to alternate location #35

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
fc1b706
add app_label to job classes
aperrin66 Feb 6, 2025
9de7575
add support for copying files after download
aperrin66 Feb 6, 2025
d0bff70
fix copy_to argument
aperrin66 Feb 6, 2025
cb76ada
update tests
aperrin66 Feb 6, 2025
360f212
update test image dependencies
aperrin66 Feb 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ on:
types: [prereleased, released]
env:
TESTING_IMAGE_NAME: geospaas_rest_api_tests
GEOSPAAS_PROCESSING_VERSION: 3.8.0
GEOSPAAS_PROCESSING_VERSION: '3.8.6.dev0'
GEOSPAAS_HARVESTING_VERSION: 3.8.0
METANORM_VERSION: 4.1.2
jobs:
Expand Down
30 changes: 26 additions & 4 deletions geospaas_rest_api/processing_api/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ class Job(models.Model):
"""Base model that gives access to the status and result of
running one or more Celery tasks.
"""
class Meta:
app_label = 'geospaas_rest_api'

# Database fields
task_id = models.CharField(
unique=True, max_length=255,
Expand Down Expand Up @@ -73,10 +76,14 @@ class DownloadJob(Job):
"""
class Meta:
proxy = True
app_label = 'geospaas_rest_api'

@classmethod
def get_signature(cls, parameters):
tasks = [tasks_core.download.signature()]
tasks = [
tasks_core.download.signature(),
tasks_core.copy.signature(kwargs={'copy_to': parameters.get('copy_to', None)}),
]

# only unarchive if cropping is needed
bounding_box = parameters.get('bounding_box', None)
Expand All @@ -100,8 +107,10 @@ def check_parameters(parameters):
- dataset_id: integer
- bounding_box: 4-elements list
"""
if not set(parameters).issubset(set(('dataset_id', 'bounding_box', 'publish'))):
raise ValidationError("The download action accepts only one parameter: 'dataset_id'")
allowed_parameters = ('dataset_id', 'bounding_box', 'publish', 'copy_to')
if not set(parameters).issubset(set(allowed_parameters)):
raise ValidationError(
f"The download action accepts only the following parameters: {allowed_parameters}")
if not isinstance(parameters['dataset_id'], int):
raise ValidationError("'dataset_id' must be an integer")
if ('bounding_box' in parameters and
Expand All @@ -111,6 +120,8 @@ def check_parameters(parameters):
"west, north, east, south")
if ('publish' in parameters and not isinstance(parameters['publish'], bool)):
raise ValidationError("'publish' must be a boolean")
if 'copy_to' in parameters and not isinstance(parameters['copy_to'], str):
raise ValidationError("'copy_to' must be a string")
return parameters

@staticmethod
Expand All @@ -124,13 +135,15 @@ class ConvertJob(Job): # pylint: disable=abstract-method

class Meta:
proxy = True
app_label = 'geospaas_rest_api'

@classmethod
def get_signature(cls, parameters):
conversion_format = parameters['format']
if conversion_format == 'idf':
return celery.chain(
tasks_core.download.signature(),
tasks_core.copy.signature(kwargs={'copy_to': parameters.get('copy_to', None)}),
tasks_core.unarchive.signature(),
tasks_core.crop.signature(
kwargs={'bounding_box': parameters.get('bounding_box', None)}),
Expand All @@ -140,6 +153,7 @@ def get_signature(cls, parameters):
elif conversion_format == 'syntool':
syntool_tasks = [
tasks_core.download.signature(),
tasks_core.copy.signature(kwargs={'copy_to': parameters.get('copy_to', None)}),
tasks_core.unarchive.signature(),
tasks_core.crop.signature(
kwargs={'bounding_box': parameters.get('bounding_box', None)}),
Expand Down Expand Up @@ -176,7 +190,8 @@ def check_parameters(parameters):
'skip_check',
'converter_options',
'remove_downloaded',
'ttl')
'ttl',
'copy_to')
if not set(parameters).issubset(set(accepted_keys)):
raise ValidationError(
f"The convert action accepts only these parameters: {', '.join(accepted_keys)}")
Expand All @@ -203,6 +218,9 @@ def check_parameters(parameters):
parameters['ttl'] is None or isinstance(parameters['ttl'], dict))):
raise ValidationError("'ttl' should be a dictionary or None")

if 'copy_to' in parameters and not isinstance(parameters['copy_to'], str):
raise ValidationError("'copy_to' must be a string")

return parameters

@staticmethod
Expand All @@ -214,6 +232,7 @@ class SyntoolCleanupJob(Job):
"""Job which cleans up ingested files older than a date"""
class Meta:
proxy = True
app_label = 'geospaas_rest_api'

@classmethod
def get_signature(cls, parameters):
Expand Down Expand Up @@ -246,6 +265,7 @@ class SyntoolCompareJob(Job):
"""
class Meta:
proxy = True
app_label = 'geospaas_rest_api'

@classmethod
def get_signature(cls, parameters):
Expand Down Expand Up @@ -306,6 +326,7 @@ class HarvestJob(Job):
"""Job which harvests metadata into the database"""
class Meta:
proxy = True
app_label = 'geospaas_rest_api'

@classmethod
def get_signature(cls, parameters):
Expand All @@ -328,6 +349,7 @@ class WorkdirCleanupJob(Job):
"""Remove everything in the working directory"""
class Meta:
proxy = True
app_label = 'geospaas_rest_api'

@classmethod
def get_signature(cls, parameters):
Expand Down
19 changes: 14 additions & 5 deletions geospaas_rest_api/tests/test_processing_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,9 +192,11 @@ def test_get_signature_no_cropping(self):
mock_chain.assert_has_calls([
mock.call([
mock_tasks.download.signature.return_value,
mock_tasks.copy.signature.return_value,
]),
mock.call([
mock_tasks.download.signature.return_value,
mock_tasks.copy.signature.return_value,
mock_tasks.archive.signature.return_value,
mock_tasks.publish.signature.return_value,
]),
Expand All @@ -210,11 +212,13 @@ def test_get_signature_cropping(self):
mock_chain.assert_has_calls([
mock.call([
mock_tasks.download.signature.return_value,
mock_tasks.copy.signature.return_value,
mock_tasks.unarchive.signature.return_value,
mock_tasks.crop.signature.return_value,
]),
mock.call([
mock_tasks.download.signature.return_value,
mock_tasks.copy.signature.return_value,
mock_tasks.unarchive.signature.return_value,
mock_tasks.crop.signature.return_value,
mock_tasks.archive.signature.return_value,
Expand All @@ -237,8 +241,10 @@ def test_check_parameters_wrong_key(self):
models.DownloadJob.check_parameters(parameters)
self.assertListEqual(
raised.exception.detail,
[ErrorDetail(string="The download action accepts only one parameter: 'dataset_id'",
code='invalid')])
[ErrorDetail(
string="The download action accepts only the following parameters:"
" ('dataset_id', 'bounding_box', 'publish', 'copy_to')",
code='invalid')])

def test_check_parameters_extra_param(self):
"""`check_parameters()` must raise an exception if an extra parameter is given"""
Expand All @@ -247,7 +253,8 @@ def test_check_parameters_extra_param(self):
models.DownloadJob.check_parameters(parameters)
self.assertListEqual(
raised.exception.detail,
[ErrorDetail(string="The download action accepts only one parameter: 'dataset_id'",
[ErrorDetail(string="The download action accepts only the following parameters:"
" ('dataset_id', 'bounding_box', 'publish', 'copy_to')",
code='invalid')])

def test_check_parameters_wrong_id_type(self):
Expand Down Expand Up @@ -296,7 +303,7 @@ def test_check_parameters_wrong_key(self):
raised.exception.detail,
[ErrorDetail(string="The convert action accepts only these parameters: "
"dataset_id, format, bounding_box, skip_check, converter_options, "
"remove_downloaded, ttl",
"remove_downloaded, ttl, copy_to",
code='invalid')])

def test_check_parameters_wrong_format(self):
Expand All @@ -316,7 +323,7 @@ def test_check_parameters_extra_param(self):
raised.exception.detail,
[ErrorDetail(string="The convert action accepts only these parameters: "
"dataset_id, format, bounding_box, skip_check, converter_options, "
"remove_downloaded, ttl",
"remove_downloaded, ttl, copy_to",
code='invalid')])

def test_check_parameters_wrong_type_for_dataset_id(self):
Expand Down Expand Up @@ -374,6 +381,7 @@ def test_get_signature_syntool(self):
"""Test the right signature is returned"""
base_chain = celery.chain(
tasks_core.download.signature(),
tasks_core.copy.signature(kwargs={'copy_to': None}),
tasks_core.unarchive.signature(),
tasks_core.crop.signature(
kwargs={'bounding_box': [0, 20, 20, 0]}),
Expand Down Expand Up @@ -405,6 +413,7 @@ def test_get_signature_idf(self):
}),
celery.chain(
tasks_core.download.signature(),
tasks_core.copy.signature(kwargs={'copy_to': None}),
tasks_core.unarchive.signature(),
tasks_core.crop.signature(
kwargs={'bounding_box': [0, 20, 20, 0]}),
Expand Down
Loading