#238 Improve raster loading progress

Author: Adam Collins

grails-app/conf/application.yml

@@ -408,7 +408,7 @@ environments:
       remote: "https://spatial-test.ala.org.au/ws"
     geoserver:
       # use localhost to play nice with locally running geoserver docker instance.
-      url: 'http://localhost:8079/geoserver'
+      url: 'http://localhost:8080/geoserver'
       #url: 'https://spatial-test.ala.org.au/geoserver'
       username: 'admin'
       password: 'geoserver'

grails-app/conf/logback.xml

@@ -7,8 +7,7 @@
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
             <charset>UTF-8</charset>
-            <pattern>[Spatial-service] %clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(%5p) %clr(%logger{39} [%file:%line]){cyan} %clr(:){faint} %m%n%wex
-            </pattern>
+            <pattern>%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(%5p) %clr(%logger{39} [%file:%line]){cyan} %clr(:){faint} %m%n%wex</pattern>
         </encoder>
     </appender>
 
@@ -21,12 +20,12 @@
     <logger name="org.quartz" level="ERROR"/>
     <logger name="asset.pipeline" level="ERROR"/>
     <logger name="au.org.ala.cas" level="WARN"/>
-    <logger name="au.org.ala.spatial" level="DEBUG"/>
-    <logger name="org.liquibase" level="INFO"/>
-    <logger name="liquibase" level="INFO"/>
+    <logger name="au.org.ala.spatial" level="WARN"/>
+    <logger name="org.liquibase" level="WARN"/>
+    <logger name="liquibase" level="WARN"/>
 
-    <root level="INFO">
+    <root level="WARN">
         <appender-ref ref="STDOUT"/>
     </root>
 
-</configuration>
+</configuration>

grails-app/controllers/au/org/ala/spatial/LoginInterceptor.groovy

@@ -26,6 +26,8 @@ class LoginInterceptor {
     static final String[] USERID_HEADER_NAME = ["X-ALA-userId", "userId", "user_id"]
     static final String[] API_KEY_HEADER_NAME = ["apiKey", "api_key", "api-key"]
 
+    int order = LOWEST_PRECEDENCE
+
     AuthService authService
     SpatialAuthService spatialAuthService
     SpatialConfig spatialConfig

grails-app/init/au/org/ala/spatial/BootStrap.groovy

@@ -131,7 +131,7 @@ class BootStrap {
                 "CREATE INDEX distributions_geom ON distributions USING GIST (the_geom);",
                 "CREATE INDEX tabulation_geom ON tabulation USING GIST (the_geom);",
 
-                // objects.id sequence used in manual sql that creates objects
+                // objects.spid sequence used in manual sql that creates objects
                 "CREATE SEQUENCE objects_id_seq INCREMENT 1 MINVALUE 1 MAXVALUE 9223372036854775807 START 1 CACHE 1;",
 
                 // compound constraint

grails-app/services/au/org/ala/spatial/ManageLayersService.groovy

@@ -15,6 +15,7 @@
 
 package au.org.ala.spatial
 
+import au.org.ala.spatial.grid.Bil2diva
 import au.org.ala.spatial.grid.Diva2bil
 import au.org.ala.spatial.intersect.Grid
 import au.org.ala.spatial.util.UploadSpatialResource
@@ -273,7 +274,7 @@ class ManageLayersService {
                 map.put("error", "no layer files")
             } else {
                 def errors = publishService.layerToGeoserver(new OutputParameter([
-                        file: ([shp.exists() ? shp.getPath() : bil.getPath()] as JSON).toString()
+                        file: ([shp.exists() ? shp.getPath() : tif.getPath()] as JSON).toString()
                 ]), null)
 
                 if (errors) {
@@ -503,7 +504,7 @@ class ManageLayersService {
                 map.put("columns", columns)
                 map.put("type", "Contextual")
             } else if (bil.exists()) {
-                double[] minmax = getMinMax(bil)
+                double[] minmax = Bil2diva.getMinMax(bil, spatialConfig.gdal.dir, spatialConfig.admin.timeout)
                 map.put("environmentalvaluemin", minmax[0])
                 map.put("environmentalvaluemax", minmax[1])
 
@@ -1226,34 +1227,6 @@ class ManageLayersService {
 //        taskDao.addTask(UPDATE_GRID_CACHE, "", 3);
 //    }
 
-    def getMinMax(File bil) {
-        double[] minmax = new double[2]
-
-        //does .grd exist?
-        File grd = new File(bil.getPath().substring(0, bil.getPath().length() - 4) + ".grd")
-        File hdr = new File(bil.getPath().substring(0, bil.getPath().length() - 4) + ".hdr")
-
-        if (!grd.exists() && hdr.exists()) {
-            String n = bil.getPath().substring(0, bil.getPath().length() - 4)
-            Bil2diva.bil2diva(n, n, '')
-        }
-
-        try {
-            String info = grd.text
-            for (String line : info.split("\n")) {
-                if (line.startsWith("MinValue")) {
-                    minmax[0] = Double.parseDouble(line.substring("MinValue=".length()).trim())
-                }
-                if (line.startsWith("MaxValue")) {
-                    minmax[1] = Double.parseDouble(line.substring("MaxValue=".length()).trim())
-                }
-            }
-        } catch (IOException e) {
-            log.error("failed to get min max values in bil file: " + bil.getPath(), e)
-        }
-        return minmax
-    }
-
     def distributionMap(String uploadId) {
         String dir = spatialConfig.data.dir
 

grails-app/views/manageLayers/layer.gsp

@@ -293,24 +293,24 @@
                         </input>
                         </td></tr><tr><td>
 
-                            <label for="minlongitude">Minimum Longitude [readonly]</label></td><td>
-                            <input class="form-control" type="readonly" id="minlongitude" name="minlongitude"
-                                   value="${minlongitude}" maxlength="256" readonly/>
+                            <label for="minlongitude">Minimum Longitude</label></td><td>
+                            <input class="form-control" id="minlongitude" name="minlongitude"
+                                   value="${minlongitude}" maxlength="256"/>
                         </td></tr><tr><td>
 
-                            <label for="maxlongitude">Maximum Longitude [readonly]</label></td><td>
-                            <input class="form-control" type="readonly" id="maxlongitude" name="maxlongitude"
-                                   value="${maxlongitude}" maxlength="256" readonly/>
+                            <label for="maxlongitude">Maximum Longitude</label></td><td>
+                            <input class="form-control" id="maxlongitude" name="maxlongitude"
+                                   value="${maxlongitude}" maxlength="256"/>
                         </td></tr><tr><td>
 
-                            <label for="minlatitude">Minimum Latitude [readonly]</label></td><td>
-                            <input class="form-control" type="readonly" id="minlatitude" name="minlatitude"
-                                   value="${minlatitude}" maxlength="256" readonly/>
+                            <label for="minlatitude">Minimum Latitude</label></td><td>
+                            <input class="form-control" id="minlatitude" name="minlatitude"
+                                   value="${minlatitude}" maxlength="256" />
                         </td></tr><tr><td>
 
-                            <label for="maxlatitude">Maximum Latitude [readonly]</label></td><td>
-                            <input class="form-control" type="readonly" id="maxlatitude" name="maxlatitude"
-                                   value="${maxlatitude}" maxlength="256" readonly/>
+                            <label for="maxlatitude">Maximum Latitude </label></td><td>
+                            <input class="form-control" id="maxlatitude" name="maxlatitude"
+                                   value="${maxlatitude}" maxlength="256" />
                         </td></tr><tr><td>
 
                             <label for="metadatapath">Metadata path (e.g. URL to original metadata, if available)</label>

scripts/bilConvert.c

@@ -0,0 +1,100 @@
+/*
+Exmaple for converting an int16 raster values so that it can be converted to int8, halving the file size.
+
+1. set the .bil file to open.
+2. set "arr", such that it is an ordered list of existing values. The new value will be the index position of the existing value.
+3. set the output.bil file for writing.
+
+*/
+
+#include <stdio.h>
+#include <stdint.h>
+
+#define BUFFER_SIZE 1024*1024  // Adjust this value according to your system's memory capacity
+
+// A recursive binary search function. It returns location of x in
+// given array arr[l..r] is present, otherwise returns -1
+int binarySearch(int arr[], int l, int r, int x){
+    if (r >= l) {
+        int mid = l + (r - l) / 2;
+
+        // If the element is present at the middle itself
+        if (arr[mid] == x)
+            return mid;
+
+        // If element is smaller than mid, then it can only be present
+        // in left subarray
+        if (arr[mid] > x)
+            return binarySearch(arr, l, mid - 1, x);
+
+        // Else the element can only be present in right subarray
+        return binarySearch(arr, mid + 1, r, x);
+    }
+
+    // We reach here when element is not present in array
+    return -1;
+}
+
+int main() {
+    FILE *file = fopen("/data/spatial-data/uploads/land_use/land_use.bil", "rb");
+    if (file == NULL) {
+        printf("Failed to open file.\n");
+        return 1;
+    }
+
+    FILE *outputFile = fopen("output.bil", "wb");
+    if (outputFile == NULL) {
+        printf("Failed to open output file.\n");
+        return 1;
+    }
+
+    // Determine the size of the file
+    fseek(file, 0, SEEK_END);
+    long fileSize = ftell(file);
+    fseek(file, 0, SEEK_SET);
+
+    // Calculate the number of chunks
+    long numChunks = fileSize / (BUFFER_SIZE * sizeof(int16_t));
+    if (fileSize % (BUFFER_SIZE * sizeof(int16_t)) != 0) {
+        numChunks++;  // There is a partial chunk at the end of the file
+    }
+
+    // Buffer to hold each chunk
+    int16_t buffer[BUFFER_SIZE];
+
+    // create an int array to hold [1, 2, 3]
+    int arr[192] = {0,110,111,112,113,114,115,116,117,120,121,122,123,124,125,130,131,132,133,134,210,220,221,222,310,311,312,313,314,320,321,322,323,324,325,330,331,332,333,334,335,336,337,338,340,341,342,343,344,345,346,347,348,349,350,351,352,353,360,361,362,363,364,365,410,411,412,413,414,420,421,422,423,424,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,460,461,462,463,464,465,510,511,512,513,514,515,520,521,522,523,524,525,526,527,528,530,531,532,533,534,535,536,537,538,540,541,542,543,544,545,550,551,552,553,554,555,560,561,562,563,564,565,566,567,570,571,572,573,574,575,580,581,582,583,584,590,591,592,593,594,595,610,611,612,613,614,620,621,622,623,630,631,632,633,640,641,642,643,650,651,652,653,654,660,661,662,663};
+
+    // initialize an array of length 100 with 0's
+    int hist[1000] = {0};
+
+    printf("chunks %ld\n", numChunks);
+
+    // Process each chunk
+    for (long i = 0; i < numChunks; i++) {
+//        printf("%ld,", i);
+//        printf(".");
+        // Read the chunk into the buffer
+        fread(buffer, sizeof(int16_t), BUFFER_SIZE, file);
+
+        // Iterate over the buffer and add 1 to each value
+        for (int j = 0; j < BUFFER_SIZE; j++) {
+            hist[buffer[j]]++;
+//            printf("%d,", buffer[j]);
+            buffer[j] = binarySearch(arr, 0, 191, buffer[j]);
+        }
+
+        // Write the modified data to the output file
+        fwrite(buffer, sizeof(int16_t), BUFFER_SIZE, outputFile);
+    }
+
+    // print the histogram
+    for (int i = 0; i < 1000; i++) {
+        printf("\n%d: %d", i, hist[i]);
+    }
+
+    fclose(file);
+    fclose(outputFile);
+
+    return 0;
+}

scripts/bilStatus.c

@@ -0,0 +1,96 @@
+/*
+
+A work in progress. This will produce the "raster as contextual layer's" *.classes.json instead of using the built in process.
+
+*/
+
+#include <stdio.h>
+#include <stdint.h>
+
+#define BUFFER_SIZE 1024*1024  // Adjust this value according to your system's memory capacity
+
+int main() {
+    FILE *file = fopen("/data/spatial-data/layer/land_use2.bil", "rb");
+    if (file == NULL) {
+        printf("Failed to open file.\n");
+        return 1;
+    }
+
+    FILE *outputFile = fopen("output.bil", "wb");
+    if (outputFile == NULL) {
+        printf("Failed to open output file.\n");
+        return 1;
+    }
+
+    // Determine the size of the file
+    fseek(file, 0, SEEK_END);
+    long fileSize = ftell(file);
+    fseek(file, 0, SEEK_SET);
+
+    // Calculate the number of chunks
+    long numChunks = fileSize / (BUFFER_SIZE * sizeof(unsigned char));
+    if (fileSize % (BUFFER_SIZE * sizeof(unsigned char)) != 0) {
+        numChunks++;  // There is a partial chunk at the end of the file
+    }
+
+    // Buffer to hold each chunk
+    unsigned char buffer[BUFFER_SIZE];
+
+    int nrows = 73752;
+    int ncols = 98262;
+
+    // initialize an array of length 100 with 0's
+    int hist[192*73752] = {0};
+    int minx[192] = {0};
+    int miny[192] = {0};
+    int maxx[192] = {0};
+    int maxy[192] = {0};
+
+    printf("chunks %ld\n", numChunks);
+
+    long pos = 0;
+
+
+    // Process each chunk
+    for (long i = 0; i < numChunks; i++) {
+//        printf("%ld,", i);
+//        printf(".");
+        // Read the chunk into the buffer
+        fread(buffer, sizeof(int16_t), BUFFER_SIZE, file);
+
+        // Iterate over the buffer and add 1 to each value
+        for (int j = 0; j < BUFFER_SIZE; j++) {
+            int row = pos / ncols;
+            int col = pos % ncols;
+
+            hist[buffer[j]]++;
+
+            minx[buffer[j]] = minx[buffer[j]] < col ? minx[buffer[j]] : col;
+            miny[buffer[j]] = miny[buffer[j]] < row ? miny[buffer[j]] : row;
+            maxx[buffer[j]] = maxx[buffer[j]] > col ? maxx[buffer[j]] : col;
+            maxy[buffer[j]] = maxy[buffer[j]] > row ? maxy[buffer[j]] : row;
+
+//            printf("%d,", buffer[j]);
+//            buffer[j] = binarySearch(arr, 0, 191, buffer[j]);
+
+            pos++;
+        }
+
+        // Write the modified data to the output file
+        fwrite(buffer, sizeof(int16_t), BUFFER_SIZE, outputFile);
+    }
+
+    // print the histogram
+//    for (int i = 0; i < 1000; i++) {
+//        printf("\n%d: %d", i, hist[i]);
+//    }
+
+    for (int i = 0; i < 192; i++) {
+        printf("\n%d: %d, %d, %d, %d", i, hist[i], minx[i], miny[i], maxx[i], maxy[i]);
+    }
+
+    fclose(file);
+    fclose(outputFile);
+
+    return 0;
+}

src/main/groovy/au/org/ala/spatial/StreamGobbler.groovy

@@ -26,11 +26,13 @@ class StreamGobbler extends Thread {
     BufferedReader br
     String logPrefix
     TaskWrapper taskWrapper
+    StringBuffer stringBuffer
 
-    StreamGobbler(InputStream is, String logPrefix, TaskWrapper task) {
+    StreamGobbler(InputStream is, String logPrefix, TaskWrapper task, StringBuffer stringBuffer) {
         br = new BufferedReader(new InputStreamReader(is))
         this.logPrefix = logPrefix
         this.taskWrapper = task
+        this.stringBuffer = stringBuffer
     }
 
     @Override
@@ -41,6 +43,9 @@ class StreamGobbler extends Thread {
                 if (taskWrapper != null) {
                     taskWrapper.task.history.put(System.currentTimeMillis() as String, logPrefix + ": " + line)
                 }
+                if (stringBuffer != null) {
+                    stringBuffer.append(line).append('\n');
+                }
                 log.debug logPrefix + ": " + line
             }
         } catch (Exception e) {