Skip to content

Commit 269bc78

Browse files
daniel-zamoradaniel-zamora
authored
CMR-7379 Adds virtual product support for AST_L1T.031 (#1248)
* CMR-7379 Adds virtual product support for AST_L1T.031
1 parent 015faaf commit 269bc78

File tree

8 files changed

+73
-26
lines changed

8 files changed

+73
-26
lines changed

‎bootstrap-app/src/cmr/bootstrap/data/virtual_products.clj

+2-1
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,8 @@
107107
[system source-collection source-granule virtual-collection]
108108
(let [virtual-granule-ur (svm/generate-granule-ur (:provider-id source-collection)
109109
(:short-name source-collection)
110-
(-> virtual-collection :extra-fields :short-name)
110+
{:short-name (get-in virtual-collection [:extra-fields :short-name])
111+
:version-id (get-in virtual-collection [:extra-fields :version-id])}
111112
(:native-id source-granule))
112113
existing-virtual-granule (find-concept
113114
system

‎system-int-test/src/cmr/system_int_test/utils/virtual_product_util.clj

+7-7
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@
9999
vp-config (get svm/source-to-virtual-product-mapping [provider-id entry-title])]
100100
(for [virtual-coll (:virtual-collections vp-config)]
101101
(svm/generate-granule-ur
102-
provider-id (:short-name vp-config) (:short-name virtual-coll) granule-ur))))
102+
provider-id (:short-name vp-config) virtual-coll granule-ur))))
103103

104104
(defn translate-granule-entries
105105
"Translate the virtual granule entries to the corresponding source entries in the input json"
@@ -236,25 +236,25 @@
236236

237237
{:granule-ur "SC:AST_L1A.003:2006227720"
238238
:product-specific-attributes [psa4]}
239-
["SC:AST_L1B.003:2006227720"]
239+
["SC:AST_L1B.003:2006227720" "SC:AST_L1T.031:2006227720"]
240240

241241
{:granule-ur "SC:AST_L1A.003:2006227721"
242242
:product-specific-attributes [psa1]}
243243
["SC:AST_05.003:2006227721" "SC:AST_08.003:2006227721" "SC:AST_09T.003:2006227721"
244-
"SC:AST_L1B.003:2006227721"]
244+
"SC:AST_L1B.003:2006227721" "SC:AST_L1T.031:2006227721"]
245245

246246
{:granule-ur "SC:AST_L1A.003:2006227722"
247247
:product-specific-attributes [psa1 psa2 psa3 psa4]}
248248
["SC:AST_05.003:2006227722" "SC:AST_08.003:2006227722" "SC:AST_09T.003:2006227722"
249-
"SC:AST_L1B.003:2006227722"]
249+
"SC:AST_L1B.003:2006227722" "SC:AST_L1T.031:2006227722"]
250250

251251
{:granule-ur "SC:AST_L1A.003:2006227724"
252252
:product-specific-attributes [psa3 psa4]
253253
:data-granule (umm-g/map->DataGranule
254254
{:day-night "DAY"
255255
:production-date-time "2014-09-26T11:11:00Z"})}
256256
["SC:AST14DEM.003:2006227724" "SC:AST14OTH.003:2006227724" "SC:AST14DMO.003:2006227724"
257-
"SC:AST_L1B.003:2006227724"]
257+
"SC:AST_L1B.003:2006227724" "SC:AST_L1T.031:2006227724"]
258258

259259
{:granule-ur "SC:AST_L1A.003:2006227725"
260260
:product-specific-attributes [psa2 psa3 psa4]
@@ -264,7 +264,7 @@
264264
["SC:AST14DMO.003:2006227725" "SC:AST_09.003:2006227725"
265265
"SC:AST_09XT.003:2006227725" "SC:AST14DEM.003:2006227725"
266266
"SC:AST_07.003:2006227725" "SC:AST14OTH.003:2006227725"
267-
"SC:AST_07XT.003:2006227725" "SC:AST_L1B.003:2006227725"]
267+
"SC:AST_07XT.003:2006227725" "SC:AST_L1B.003:2006227725" "SC:AST_L1T.031:2006227725"]
268268

269269
{:granule-ur "SC:AST_L1A.003:2006227726"
270270
:product-specific-attributes [psa1 psa2 psa3 psa4]
@@ -274,5 +274,5 @@
274274
["SC:AST_09XT.003:2006227726" "SC:AST14DEM.003:2006227726"
275275
"SC:AST_08.003:2006227726" "SC:AST_05.003:2006227726"
276276
"SC:AST14OTH.003:2006227726" "SC:AST_07.003:2006227726"
277-
"SC:AST_09.003:2006227726" "SC:AST_09T.003:2006227726"
277+
"SC:AST_09.003:2006227726" "SC:AST_09T.003:2006227726" "SC:AST_L1T.031:2006227726"
278278
"SC:AST_07XT.003:2006227726" "SC:AST14DMO.003:2006227726" "SC:AST_L1B.003:2006227726"])))

‎system-int-test/test/cmr/system_int_test/bootstrap/virtual_products_test.clj

+4-2
Original file line numberDiff line numberDiff line change
@@ -139,9 +139,11 @@
139139
(doseq [vp-coll vp-colls
140140
:let [{:keys [provider-id source-collection]} vp-coll
141141
source-short-name (get-in source-collection [:product :short-name])
142-
vp-short-name (get-in vp-coll [:product :short-name])]]
142+
vp-short-name (get-in vp-coll [:product :short-name])
143+
vp-version-id (get-in vp-coll [:product :version-id])]]
143144
(vp/assert-matching-granule-urs
144-
(map #(svm/generate-granule-ur provider-id source-short-name vp-short-name %)
145+
(map #(svm/generate-granule-ur provider-id source-short-name {:short-name vp-short-name
146+
:version-id vp-version-id} %)
145147
(svm/sample-source-granule-urs
146148
[provider-id (:entry-title source-collection)]))
147149
(search/find-refs :granule {:entry-title (:entry-title vp-coll)

‎system-int-test/test/cmr/system_int_test/virtual_product/virtual_product_test.clj

+10-6
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@
4040

4141
(def vpc (vp/ingest-virtual-collections isc)))
4242

43-
4443
(deftest specific-granule-in-virtual-product-test
4544
(let [[ast-coll] (vp/ingest-source-collections
4645
[(assoc
@@ -79,7 +78,8 @@
7978
(doseq [vp-coll vp-colls]
8079
(vp/assert-matching-granule-urs
8180
[(svm/generate-granule-ur
82-
"LPDAAC_ECS" "AST_L1A" (get-in vp-coll [:product :short-name]) granule-ur)]
81+
"LPDAAC_ECS" "AST_L1A" {:short-name (get-in vp-coll [:product :short-name])
82+
:version-id (get-in vp-coll [:product :version-id])} granule-ur)]
8383
(search/find-refs :granule {:entry-title (:entry-title vp-coll)
8484
:page-size 50}))))
8585

@@ -145,9 +145,11 @@
145145
(doseq [vp-coll vp-colls
146146
:let [{:keys [provider-id source-collection]} vp-coll
147147
source-short-name (get-in source-collection [:product :short-name])
148-
vp-short-name (get-in vp-coll [:product :short-name])]]
148+
vp-short-name (get-in vp-coll [:product :short-name])
149+
vp-version-id (get-in vp-coll [:product :version-id])]]
149150
(vp/assert-matching-granule-urs
150-
(map #(svm/generate-granule-ur provider-id source-short-name vp-short-name %)
151+
(map #(svm/generate-granule-ur provider-id source-short-name {:short-name vp-short-name
152+
:version-id vp-version-id} %)
151153
(svm/sample-source-granule-urs
152154
[provider-id (:entry-title source-collection)]))
153155
(search/find-refs :granule {:entry-title (:entry-title vp-coll)
@@ -361,9 +363,11 @@
361363
(doseq [vp-coll enabled-vp-colls
362364
:let [{:keys [provider-id source-collection]} vp-coll
363365
source-short-name (get-in source-collection [:product :short-name])
364-
vp-short-name (get-in vp-coll [:product :short-name])]]
366+
vp-short-name (get-in vp-coll [:product :short-name])
367+
vp-version-id (get-in vp-coll [:product :version-id])]]
365368
(vp/assert-matching-granule-urs
366-
(map #(svm/generate-granule-ur provider-id source-short-name vp-short-name %)
369+
(map #(svm/generate-granule-ur provider-id source-short-name {:short-name vp-short-name
370+
:version-id vp-version-id} %)
367371
(svm/sample-source-granule-urs
368372
[provider-id (:entry-title source-collection)]))
369373
(search/find-refs :granule {:entry-title (:entry-title vp-coll)

‎virtual-product-app/src/cmr/virtual_product/data/ast_l1a.clj

+16-3
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,20 @@
5454
"UpperLeftQuadCloudCoverage" "UpperRightQuadCloudCoverage" "VNIR1_ObservationMode"
5555
"VNIR2_ObservationMode" "source_granule_ur"})
5656

57+
(def additional-attributes-L1T-031
58+
"The set of additional attributes for AST_L1T.031."
59+
#{"ASTERGains" "ASTERMapOrientationAngle" "ASTERMapProjection" "ASTERProcessingCenter"
60+
"ASTERReceivingCenter" "ASTERSWIRPointingAngle" "ASTERTIRPointingAngle"
61+
"ASTERVNIRPointingAngle" "Band10_Available" "Band11_Available" "Band12_Available"
62+
"Band13_Available" "Band14_Available" "Band1_Available" "Band2_Available"
63+
"Band3B_Available" "Band3N_Available" "Band4_Available" "Band5_Available"
64+
"Band6_Available" "Band7_Available" "Band8_Available" "Band9_Available" "DAR_ID"
65+
"GenerationDateandTime" "GeometricDBVersion" "LowerLeftQuadCloudCoverage"
66+
"LowerRightQuadCloudCoverage" "RadiometricDBVersion" "SWIR_ObservationMode"
67+
"SceneCloudCoverage" "Solar_Azimuth_Angle" "Solar_Elevation_Angle" "TIR_ObservationMode"
68+
"UpperLeftQuadCloudCoverage" "UpperRightQuadCloudCoverage" "VNIR1_ObservationMode"
69+
"VNIR2_ObservationMode" "source_granule_ur"})
70+
5771
(def short-name->additional-attributes
5872
"Defines virtual collection short-name to additional-attributes mapping."
5973
{"AST_05" additional-attributes-05-08-09T
@@ -66,6 +80,5 @@
6680
"AST14DEM" additional-attributes-14DEM
6781
"AST14DMO" additional-attributes-14DMO-14OTH
6882
"AST14OTH" additional-attributes-14DMO-14OTH
69-
"AST_L1B" additional-attributes-L1B})
70-
71-
83+
"AST_L1B" additional-attributes-L1B
84+
"AST_L1T" additional-attributes-L1T-031})

‎virtual-product-app/src/cmr/virtual_product/data/source_to_virtual_mapping.clj

+31-6
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,20 @@
4646
(some #(and (= (:name %) psa-name) (some #{value} (:values %)))
4747
(:product-specific-attributes granule))))
4848

49+
(defn- sanitize-version-id
50+
"When version-id is nil, return nil
51+
When version-id is a number, return string version with preceeding zeros
52+
When version-id is a string, parse into number, then return as string with preceeding zeros.
53+
When version-id is a string without a number, return 001. (This should never be the case)"
54+
[version-id]
55+
(when-not (empty? version-id)
56+
(if (number? version-id)
57+
(format "%03d" version-id)
58+
(as-> (re-find #"\d+" version-id) version-id
59+
(or version-id "1")
60+
(Integer/parseInt version-id)
61+
(format "%03d" version-id)))))
62+
4963
(def day-granule? (matches-value [:data-granule :day-night] "DAY"))
5064
(def tir-mode? (matches-on-psa "TIR_ObservationMode" "ON"))
5165
(def swir-mode? (matches-on-psa "SWIR_ObservationMode" "ON"))
@@ -99,7 +113,10 @@
99113
:matcher (match-all vnir1-mode? vnir2-mode? day-granule?)}
100114
{:entry-title "ASTER L1B Registered Radiance at the Sensor V003"
101115
:short-name "AST_L1B"
102-
:version-id "003"}]}
116+
:version-id "003"}
117+
{:entry-title "ASTER Level 1 Precision Terrain Corrected Registered At-Sensor Radiance V031"
118+
:short-name "AST_L1T"
119+
:version-id "031"}]}
103120
["GES_DISC" "OMI/Aura Surface UVB Irradiance and Erythemal Dose Daily L3 Global 1.0x1.0 deg Grid V003 (OMUVBd) at GES DISC"]
104121
{:short-name "OMUVBd"
105122
:virtual-collections [
@@ -231,12 +248,20 @@
231248

232249
(defmulti generate-granule-ur
233250
"Generates a new granule ur for the virtual collection"
234-
(fn [provider-id source-short-name virtual-short-name granule-ur]
235-
[provider-id source-short-name]))
251+
(fn [provider-id source-short-name virtual-granule granule-ur]
252+
source-short-name))
236253

237254
(defmethod generate-granule-ur :default
238-
[provider-id source-short-name virtual-short-name granule-ur]
239-
(str/replace-first granule-ur source-short-name virtual-short-name))
255+
[provider-id source-short-name virtual-granule granule-ur]
256+
(str/replace-first granule-ur source-short-name (:short-name virtual-granule)))
257+
258+
(defmethod generate-granule-ur "AST_L1A"
259+
[provider-id source-short-name virtual-granule granule-ur]
260+
(let [virtual-version-id (sanitize-version-id (:version-id virtual-granule))]
261+
(as-> (str/replace-first granule-ur source-short-name (:short-name virtual-granule)) granule-ur
262+
(if virtual-version-id
263+
(str/replace-first granule-ur #"\.\d\d\d" (str "." virtual-version-id))
264+
granule-ur))))
240265

241266
;; The granule urs of granules in the virtual collection based on AST_L1A is a simple
242267
;; transformation of the granule urs of the corresponding source granules and its inverse is trivial.
@@ -436,7 +461,7 @@
436461
virtual-granule-ur (generate-granule-ur
437462
provider-id
438463
source-short-name
439-
virtual-short-name
464+
virtual-coll
440465
(:granule-ur source-umm))]
441466
(-> source-umm
442467
;; Remove measured parameters from virtual granules

‎virtual-product-app/src/cmr/virtual_product/services/virtual_product_service.clj

+1-1
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,7 @@
239239
(let [new-granule-ur (svm/generate-granule-ur
240240
provider-id
241241
(:short-name vp-config)
242-
(:short-name virtual-coll)
242+
virtual-coll
243243
granule-ur)
244244
resp (ingest/delete-concept
245245
context

‎virtual-product-app/test/cmr/virtual_product/test/data/source_to_virtual_mapping.clj

+2
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,8 @@
118118
"LPDAAC_ECS" ast-l1a "AST_L1B" "003"
119119
{:granule-ur "SC:AST_L1A.003:2006227720"} {:granule-ur "SC:AST_L1B.003:2006227720"}
120120

121+
"LPDAAC_ECS" ast-l1a "AST_L1T" "031"
122+
{:granule-ur "SC:AST_L1A.003:2006227720"} {:granule-ur "SC:AST_L1T.031:2006227720"}
121123

122124
;; OMUVBD
123125
"GES_DISC" omuvbd "OMUVBd_ErythemalUV" "003"

0 commit comments

Comments
 (0)