Only mirror files <= 10 MB on lower deployments (#7056)

Author: nadove-ucsc

src/azul/azulclient.py

@@ -699,18 +699,28 @@ def mirror_file(self,
         source = self.repository_plugin(catalog).source_ref_cls.from_json(source_json)
         file = self.load_file(catalog, file_json)
         assert file.size is not None, R('File size unknown', file)
-        part_size = FilePart.default_size
-        if file.size > part_size:
-            upload_id = self.mirror_service.begin_mirroring_file(file)
-            parts = list(FilePart.partition(file, part_size))
-            log.info('Mirroring %d parts for file %r in source %r from catalog %r',
-                     len(parts), file.uuid, str(source.spec), catalog)
-            messages = self.mirror_parts_messages(catalog, source, file, parts, upload_id)
-            self.queue_mirror_messages(messages)
+
+        file_is_large = file.size > 10 * 1024 ** 2
+        deployment_is_stable = (config.deployment.is_stable
+                                and not config.deployment.is_unit_test
+                                and catalog not in config.integration_test_catalogs)
+        if file_is_large and not deployment_is_stable:
+            log.info('Not mirroring file %r (%d bytes) to save cost',
+                     file.uuid, file.size)
         else:
-            log.info('Mirroring file %r in source %r from catalog %r',
-                     file.uuid, str(source.spec), catalog)
-            self.mirror_service.mirror_file(catalog, file)
+            # Ensure we test with multiple parts on lower deployments
+            part_size = FilePart.default_size if deployment_is_stable else FilePart.min_size
+            if file.size > part_size:
+                upload_id = self.mirror_service.begin_mirroring_file(file)
+                parts = list(FilePart.partition(file, part_size))
+                log.info('Mirroring %d parts for file %r in source %r from catalog %r',
+                         len(parts), file.uuid, str(source.spec), catalog)
+                messages = self.mirror_parts_messages(catalog, source, file, parts, upload_id)
+                self.queue_mirror_messages(messages)
+            else:
+                log.info('Mirroring file %r in source %r from catalog %r',
+                         file.uuid, str(source.spec), catalog)
+                self.mirror_service.mirror_file(catalog, file)
 
     def load_file(self, catalog: CatalogName, file: JSON) -> File:
         return self.metadata_plugin(catalog).file_class.from_json(file)