CMR-7220: cloud_hosted search (#1234)

CMR-7220 - adding a cloud_hosted search parameter, updating api doc

Author: jceaser

.gitignore

@@ -14,6 +14,7 @@ browse-scaler/src/node_modules
 .clj-kondo/
 .lsp
 .cider-repl-history
+**/ignore.*
 
 ###############################
 ### Test Files

indexer-app/src/cmr/indexer/data/concepts/collection.clj

@@ -173,6 +173,12 @@
        distinct
        (remove nil?)))
 
+(defn- cloud-hosted?
+  "Test if the collection meets the criteria for being cloud hosted"
+  [collection tags]
+  (or (not (empty? (:DirectDistributionInformation collection)))
+      (tag/has-cloud-s3-tag? tags)))
+
 (defn- get-elastic-doc-for-full-collection
   "Get all the fields for a normal collection index operation."
   [context concept collection]
@@ -388,6 +394,7 @@
             :metadata-format (name (mt/format-key format))
             :related-urls (map json/generate-string opendata-related-urls)
             :has-opendap-url (not (empty? (filter opendap-util/opendap-url? related-urls)))
+            :cloud-hosted (cloud-hosted? collection tags)
             :publication-references opendata-references
             :collection-citations (map json/generate-string opendata-citations)
             :update-time update-time

indexer-app/src/cmr/indexer/data/concepts/tag.clj

@@ -27,3 +27,13 @@
      :originator-id-lowercase  (util/safe-lowercase originator-id)
      :tag-value-lowercase (when (string? data)
                             (str/lower-case data))}))
+
+(def earthdata-cloud-s3-tag
+  "The Earthdata cloud tag for s3 resources"
+  "gov.nasa.earthdata.cloud.s3")
+
+(defn has-cloud-s3-tag?
+  "Looks through a list of tags and returns true if one of them is the
+  gov.nasa.earthdata.cloud.s3 tag"
+  [tags]
+  (some? (some #(= (:tag-key-lowercase %) earthdata-cloud-s3-tag) tags)))

indexer-app/src/cmr/indexer/data/index_set.clj

@@ -375,6 +375,7 @@
           :has-spatial-subsetting m/bool-field-mapping
           :has-temporal-subsetting m/bool-field-mapping
           :has-opendap-url m/bool-field-mapping
+          :cloud-hosted m/bool-field-mapping
 
           :platform-sn                    m/string-field-mapping
           :platform-sn-lowercase          m/string-field-mapping

indexer-app/test/cmr/indexer/test/data/concepts/tag.clj

@@ -0,0 +1,30 @@
+(ns cmr.indexer.test.data.concepts.tag
+  "Code coverage tests for the functions of testing cmr.indexer.data.concepts.tag
+   namespace."
+  (:require
+   [clojure.string :as str]
+   [clojure.test :refer :all]
+   [cmr.common.util :as util]
+   [cmr.indexer.data.concepts.tag :as tag]))
+
+(defn- create-tag
+  "Create a tag map"
+  [tag-key originator data]
+  {:tag-key-lowercase (str/lower-case tag-key)
+   :originator-id-lowercase (util/safe-lowercase originator)
+   :tag-value-lowercase (when (string? data) (str/lower-case data))})
+
+(deftest does-has-cloud-s3-tag-work
+  "The function has-cloud-s3-tag should only return true if there is a
+   gov.nasa.earthdata.cloud.s3 tag in the list"
+  (let [bad1 (create-tag "This-Is-Not-The-Tag-Your-Looking-For" "My-Id" "Value")
+        bad2 (create-tag "This-Is-Also-Not-The-Tag-Your-Looking-For" "My-Id" "Value")
+        good (create-tag tag/earthdata-cloud-s3-tag "My-Id" "Value")
+        will-not-be-found [bad1]
+        these-will-not-be-found [bad1 bad2]
+        will-be-found [good]
+        one-will-be-found [bad1 bad2 good]]
+    (is (false? (tag/has-cloud-s3-tag? will-not-be-found)))
+    (is (false? (tag/has-cloud-s3-tag? these-will-not-be-found)))
+    (is (true? (tag/has-cloud-s3-tag? will-be-found)))
+    (is (true? (tag/has-cloud-s3-tag? one-will-be-found)))))

search-app/docs/api.md

@@ -85,6 +85,7 @@ Join the [CMR Client Developer Forum](https://wiki.earthdata.nasa.gov/display/CM
     * [With/without granules Or Cwic](#c-has-granules-or-cwic)
     * [With/without granules Or OpenSearch](#c-has-granules-or-opensearch)
     * [OPeNDAP service URL](#c-has-opendap-url)
+    * [Cloud Hosted](#c-cloud-hosted)
   * [Sorting Collection Results](#sorting-collection-results)
   * [Retrieving all Revisions of a Collection](#retrieving-all-revisions-of-a-collection)
   * [Granule Search By Parameters](#granule-search-by-parameters)
@@ -1887,6 +1888,10 @@ The `has_granules_or_opensearch` parameter can be set to "true" or "false". When
 
     curl "%CMR-ENDPOINT%/collections?has_opendap_url=true"
 
+#### <a name="c-cloud-hosted"></a> Find collections that have a `DirectDistributionInformation` element or have been tagged with `gov.nasa.earthdata.cloud.s3`.
+
+    curl "%CMR-ENDPOINT%/collections?cloud_hosted=true"
+
 #### <a name="sorting-collection-results"></a> Sorting Collection Results
 
 Collection results are sorted by ascending entry title by default when a search does not result in a score.

search-app/src/cmr/search/services/parameters/conversion.clj

@@ -47,6 +47,7 @@
    :has-granules-created-at :multi-date-range
    :has-granules-revised-at :multi-date-range
    :has-opendap-url :boolean
+   :cloud-hosted :boolean
    :instrument :string
    :instrument-h :humanizer
    :keyword :keyword

search-app/src/cmr/search/services/parameters/parameter_validation.clj

@@ -630,7 +630,7 @@
   (let [bool-params (select-keys params [:downloadable :browsable :include-granule-counts
                                          :include-has-granules :has-granules :hierarchical-facets
                                          :include-highlights :all-revisions :has-opendap-url
-                                         :simplify-shapefile])]
+                                         :simplify-shapefile :cloud-hosted])]
     (mapcat
       (fn [[param value]]
         (when-not (contains? #{"true" "false" "unset"} (when value (s/lower-case value)))

system-int-test/test/cmr/system_int_test/search/collection_cloud_hosted_search_test.clj

@@ -0,0 +1,50 @@
+(ns cmr.system-int-test.search.collection-cloud-hosted-search-test
+  "Integration tests for searching for records that are cloud hosted"
+  (:require
+   [clojure.test :refer :all]
+   [cmr.common.util :as util]
+   [cmr.mock-echo.client.echo-util :as echo]
+   [cmr.indexer.data.concepts.tag :as itag]
+   [cmr.system-int-test.data2.core :as data2]
+   [cmr.system-int-test.data2.umm-spec-collection :as data-umm-c]
+   [cmr.system-int-test.system :as system]
+   [cmr.system-int-test.utils.index-util :as index]
+   [cmr.system-int-test.utils.ingest-util :as ingest]
+   [cmr.system-int-test.utils.search-util :as search]
+   [cmr.system-int-test.utils.tag-util :as tags]))
+
+(use-fixtures :each (join-fixtures
+                      [(ingest/reset-fixture {"provguid1" "PROV1"})
+                       tags/grant-all-tag-fixture]))
+
+(defn- create-direct-dist-info
+  "Create a Direct Distribution"
+  []
+  {:DirectDistributionInformation {:Region "us-east-1"
+                                   :S3CredentialsAPIEndpoint "https://example.org"
+                                   :S3CredentialsAPIDocumentationURL "https://example.org"}})
+
+(deftest search-collections-that-are-cloud-hosted
+  (let [coll1 (data2/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 1 {}))
+        coll2 (data2/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {}))
+        coll3 (data2/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 3 (create-direct-dist-info)))]
+    (index/wait-until-indexed)
+
+    (testing "Search by the cloud_hosted flag only"
+      (util/are3
+       [expected items]
+       (data2/refs-match? items (search/find-refs :collection {:cloud-hosted expected}))
+       "Found using metadata" true [coll3]
+       "Not found" false [coll1 coll2]))
+
+    (testing "Search for collections tagged as cloud hosted"
+      (let [user1-token (echo/login (system/context) "user1")
+            tag1 (tags/make-tag {:tag-key itag/earthdata-cloud-s3-tag})
+            tag_record (tags/save-tag user1-token tag1 [coll1])]
+        (index/wait-until-indexed)
+
+        (util/are3
+         [expected items]
+         (is (data2/refs-match? items (search/find-refs :collection {:cloud-hosted expected})))
+         "Found using metadata or tags" true [coll1 coll3]
+         "Not found" false [coll2])))))