CMR-7108: Changed S3 link validation to only allow lowercase s3://

Author: ygliuvt

ingest-app/docs/api.md

@@ -1101,7 +1101,7 @@ supported metadata formats:
   - S3 url in OnlineAccessURLs for ECHO10 format
   - S3 url in RelatedUrls for UMM-G format
 
-The S3 url value provided in the granule bulk update request can be comma-separated urls. Each url must start with s3:// (case-insensitive). During bulk update, the provided S3 urls in the request will overwrite any existing S3 links already in the granule metadata.
+The S3 url value provided in the granule bulk update request can be comma-separated urls. Each url must start with s3:// (case-sensitive). This lowercase s3:// naming convention is to make the s3 links compatible with AWS S3 API. During bulk update, the provided S3 urls in the request will overwrite any existing S3 links already in the granule metadata.
 
 Example: Add/update OPeNDAP url for 3 granules under PROV1.
 

ingest-app/src/cmr/ingest/services/granule_bulk_update/s3/s3_util.clj

@@ -10,13 +10,12 @@
 
 (defn validate-url
   "Validate the given S3 url for granule bulk update. It can be multiple urls
-  separated by comma, and each url must be started with s3:// (case insensitive).
+  separated by comma, and each url must be started with s3:// (case sensitive).
   Returns the parsed urls in a list."
   [input-url]
   (let [urls (map string/trim (string/split input-url #","))]
     (doseq [url urls]
-      (when-not (or (string/starts-with? url "s3://")
-                    (string/starts-with? url "S3://"))
+      (when-not (string/starts-with? url "s3://")
         (errors/throw-service-errors
          :invalid-data
          [(str "Invalid URL value, each S3 url must start with s3://, but was " url)])))

ingest-app/test/cmr/ingest/services/granule_bulk_update/s3/s3_util_test.clj

@@ -6,18 +6,22 @@
 
 (deftest validate-url-test
   (testing "validate url"
-    (is (= ["s3://abc/foo" "S3://abc/bar"]
-           (s3-util/validate-url "s3://abc/foo, S3://abc/bar"))))
+    (is (= ["s3://abc/foo" "s3://abc/bar"]
+           (s3-util/validate-url "s3://abc/foo, s3://abc/bar"))))
 
   (testing "validate url error scenarios"
     (are3 [url-value re]
       (is (thrown-with-msg?
            Exception re (s3-util/validate-url url-value)))
 
       "invalid s3 link"
-      "http://example.com/foo"
-      #"Invalid URL value, each S3 url must start with s3://, but was http://example.com/foo"
+      "S3://abc/foo"
+      #"Invalid URL value, each S3 url must start with s3://, but was S3://abc/foo"
 
       "invalid s3 link in multiple urls"
       "s3://abc/foo,http://example.com/bar,s3://abc/baz"
-      #"Invalid URL value, each S3 url must start with s3://, but was http://example.com/bar")))
+      #"Invalid URL value, each S3 url must start with s3://, but was http://example.com/bar"
+
+      "invalid s3 links in multiple urls, only report the first error"
+      "S3://abc/foo,http://example.com/bar,s3://abc/baz"
+      #"Invalid URL value, each S3 url must start with s3://, but was S3://abc/foo")))

system-int-test/test/cmr/system_int_test/ingest/granule_bulk_update/granule_bulk_update_test.clj

@@ -163,7 +163,7 @@
                            :operation "UPDATE_FIELD"
                            :update-field "S3Link"
                            :updates [["SC:AE_5DSno.002:30500511" "s3://url30500511"]
-                                     ["SC:AE_5DSno.002:30500512" "S3://url1, S3://url2,S3://url3"]
+                                     ["SC:AE_5DSno.002:30500512" "s3://url1, s3://url2,s3://url3"]
                                      ["SC:AE_5DSno.002:30500514" "s3://url30500514"]]}
               response (ingest/bulk-update-granules "PROV1" bulk-update bulk-update-options)
               {:keys [status task-id]} response]
@@ -207,7 +207,7 @@
         (let [bulk-update {:name "add s3 links"
                            :operation "UPDATE_FIELD"
                            :update-field "S3Link"
-                           :updates [["SC:AE_5DSno.002:30500511" "S3://url30500511"]
+                           :updates [["SC:AE_5DSno.002:30500511" "s3://url30500511"]
                                      ["SC:AE_5DSno.002:30500512" "s3://url30500512"]
                                      ["SC:non-existent-ur" "s3://url30500513"]]}
               response (ingest/bulk-update-granules "PROV1" bulk-update bulk-update-options)
@@ -236,7 +236,7 @@
                            :operation "UPDATE_FIELD"
                            :update-field "S3Link"
                            :updates [["SC:AE_5DSno.002:30500511" "https://foo"]
-                                     ["SC:AE_5DSno.002:30500512" "s3://foo,https://bar"]]}
+                                     ["SC:AE_5DSno.002:30500512" "s3://foo,S3://bar"]]}
               response (ingest/bulk-update-granules "PROV1" bulk-update bulk-update-options)
               {:keys [status task-id]} response]
           (index/wait-until-indexed)
@@ -253,7 +253,7 @@
                      :status-message "Invalid URL value, each S3 url must start with s3://, but was https://foo"}
                     {:granule-ur "SC:AE_5DSno.002:30500512"
                      :status "FAILED"
-                     :status-message "Invalid URL value, each S3 url must start with s3://, but was https://bar"}]
+                     :status-message "Invalid URL value, each S3 url must start with s3://, but was S3://bar"}]
                    granule-statuses))))))))
 
 (deftest add-opendap-url

umm-spec-lib/resources/example-data/umm-json/granule/v1.6/GranuleExample.json

@@ -251,7 +251,7 @@
     "Size": 10,
     "SizeUnit": "MB"
   }, {
-    "URL": "S3://aws.com/asdf/asdf/dataproduct.nc",
+    "URL": "s3://aws.com/asdf/asdf/dataproduct.nc",
     "Type": "GET DATA VIA DIRECT ACCESS",
     "Description": "ISLSCP II EARTH RADIATION BUDGET EXPERIMENT (ERBE) MONTHLY ALBEDO, 1986-1990",
     "Format": "NETCDF-4",

umm-spec-lib/src/cmr/umm_spec/test/umm_g/expected_util.clj

@@ -194,7 +194,7 @@
                      :size 10})
                    (umm-c/map->RelatedURL
                     {:type "GET DATA VIA DIRECT ACCESS"
-                     :url "S3://aws.com/asdf/asdf/dataproduct.nc"
+                     :url "s3://aws.com/asdf/asdf/dataproduct.nc"
                      :description "ISLSCP II EARTH RADIATION BUDGET EXPERIMENT (ERBE) MONTHLY ALBEDO, 1986-1990"
                      :mime-type "application/x-netcdf"
                      :title "ISLSCP II EARTH RADIATION BUDGET EXPERIMENT (ERBE) MONTHLY ALBEDO, 1986-1990"

umm-spec-lib/test/cmr/umm_spec/test/migration/version/granule.clj

@@ -595,7 +595,7 @@
                    :Type "GET DATA"
                    :Subtype "OPENDAP DATA"
                    :MimeType "application/x-hdf5"}
-                  {:URL "S3://amazon.something.com/get-data"
+                  {:URL "s3://amazon.something.com/get-data"
                    :Type "GET DATA"
                    :Format "NETCDF-4"
                    :MimeType "application/x-netcdf"}]})
@@ -616,7 +616,7 @@
                    :Type "GET DATA"
                    :Subtype "OPENDAP DATA"
                    :MimeType "application/x-hdf5"}
-                  {:URL "S3://amazon.something.com/get-data"
+                  {:URL "s3://amazon.something.com/get-data"
                    :Type "GET DATA VIA DIRECT ACCESS"
                    :Format "NETCDF-4"
                    :MimeType "application/x-netcdf"}]})