Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#250 fix points to grid #262

Merged
merged 2 commits into from
Jan 28, 2025
Merged

#250 fix points to grid #262

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0b64f27
#250 fix points to grid
Jan 28, 2025
c4194bd
fix tests
Jan 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions grails-app/services/au/org/ala/spatial/TaskQueueService.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,13 @@ class TaskQueueService {
taskWrapper.task.message = 'finished'
taskWrapper.task.history.put(System.currentTimeMillis() as String, "finished (id:${taskWrapper.task.id})" as String)

// map output.task to task when null to prevent error when flushing task
taskWrapper.task.output.each {
if (!it.task) {
it.task = taskWrapper.task
}
}

// flush task because it is finished
Task.withTransaction {
if (!taskWrapper.task.save(flush: true)) {
Expand Down
23 changes: 10 additions & 13 deletions src/main/groovy/au/org/ala/spatial/layers/OccurrenceDensity.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@ class OccurrenceDensity {
this.resolution = resolution
this.bbox = bbox

width = (int) ((bbox[2] - bbox[0]) / resolution)
height = (int) ((bbox[3] - bbox[1]) / resolution)
width = (int) Math.round((bbox[2] - bbox[0]) / resolution)
height = (int) Math.round((bbox[3] - bbox[1]) / resolution)
}

/**
Expand All @@ -81,17 +81,17 @@ class OccurrenceDensity {
*/
void setResolution(double resolution) {
this.resolution = resolution
width = (int) ((bbox[2] - bbox[0]) / resolution)
height = (int) ((bbox[3] - bbox[1]) / resolution)
width = (int) Math.round((bbox[2] - bbox[0]) / resolution)
height = (int) Math.round((bbox[3] - bbox[1]) / resolution)
}

/**
* @param bbox
*/
void setBBox(double[] bbox) {
this.bbox = bbox
width = (int) ((bbox[2] - bbox[0]) / resolution)
height = (int) ((bbox[3] - bbox[1]) / resolution)
width = (int) Math.round((bbox[2] - bbox[0]) / resolution)
height = (int) Math.round((bbox[3] - bbox[1]) / resolution)
}

/**
Expand Down Expand Up @@ -143,15 +143,14 @@ class OccurrenceDensity {
boolean worldwrap = (bbox[2] - bbox[0]) == 360
float[] values = new float[width]

int partCount = threadCount * 5
int partCount = threadCount;
int partSize = (int) Math.ceil(width / (double) partCount)
GetValuesOccurrencesThread[] getValues = new GetValuesOccurrencesThread[threadCount]
LinkedBlockingQueue<Integer> lbqGetValues = new LinkedBlockingQueue<Integer>()

int[] rowStarts = records.sortedRowStarts(bbox[1], height, resolution)

for (int row = 0; row < height; row++) {
long start = System.currentTimeMillis()
//get rows
int[] oldRow = cRows[0]
if (row == 0) {
Expand All @@ -167,7 +166,6 @@ class OccurrenceDensity {
}
}
}
long t1 = System.currentTimeMillis()

//operate on current row
int startRow = (row == 0) ? 0 : row + gridSize / 2 //gridSize is odd
Expand Down Expand Up @@ -225,7 +223,6 @@ class OccurrenceDensity {
bw.append("\n")
}
}
long end = System.currentTimeMillis()
}

for (int i = 0; i < threadCount; i++) {
Expand Down Expand Up @@ -260,11 +257,11 @@ class OccurrenceDensity {
}

int len = (row + 1 < rowStarts.length) ? rowStarts[row + 1] : records.getRecordsSize()
for (int i = rowStarts[row]; i < len; i++) {
int y = height - 1 - (int) ((records.getSortedLatitude(i) - bbox[1]) / resolution)
for (int i = (row < rowStarts.length ? rowStarts[row] : len); i < len; i++) {
int y = height - 1 - Math.round((records.getSortedLatitude(i) - bbox[1]) / resolution)

if (y == row) {
int x = (int) ((records.getSortedLongitude(i) - bbox[0]) / resolution)
int x = Math.round((records.getSortedLongitude(i) - bbox[0]) / resolution)

if (x >= 0 && x < width) {
counts[x]++
Expand Down
6 changes: 1 addition & 5 deletions src/main/groovy/au/org/ala/spatial/process/PointsToGrid.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ class PointsToGrid extends SlaveProcess {

// dump the species data to a file
taskLog("getting species data")
Records records = getRecords(speciesArea.bs.toString(), speciesArea.q.join('&fq='), bbox, null, null)
Records records = new Records(speciesArea.bs.toString(), speciesArea.q.join('&fq='), bbox, null, null, "names_and_lsid", false)

//update bbox with spatial extent of records
double minx = 180, miny = 90, maxx = -180, maxy = -90
Expand Down Expand Up @@ -182,10 +182,6 @@ class PointsToGrid extends SlaveProcess {
}
}

def getRecords(String bs, String q, double[] bbox, String filename, SimpleRegion region) {
new Records(bs, q, bbox, filename, region)
}

void writeMetadata(String filename, String title, Records records, double[] bbox, boolean odensity, boolean sdensity, int[] counts, String addAreaSqKm, String speciesName, Double gridCellSize, String movingAverage) throws IOException {
SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy hh:mm:ss")
FileWriter fw = new FileWriter(filename)
Expand Down
190 changes: 94 additions & 96 deletions src/main/groovy/au/org/ala/spatial/util/Records.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import java.util.zip.GZIPInputStream
* @author Adam
*/
@Slf4j
@CompileStatic
class Records {

//private static final Logger logger = log.getLogger(Records.class);
Expand All @@ -36,8 +35,8 @@ class Records {
init(biocache_service_url, q, bbox, filename, region, "names_and_lsid")
}

Records(String biocache_service_url, String q, double[] bbox, String filename, SimpleRegion region, String facetField) throws IOException {
init(biocache_service_url, q, bbox, filename, region, facetField)
Records(String biocache_service_url, String q, double[] bbox, String filename, SimpleRegion region, String facetField, Boolean includeYear = true) throws IOException {
init(biocache_service_url, q, bbox, filename, region, facetField, includeYear)
}

Records(String filename) throws IOException {
Expand Down Expand Up @@ -307,10 +306,12 @@ class Records {
}
}

void init(String biocache_service_url, String q, double[] bbox, String filename, SimpleRegion region, String facetField) throws IOException {
void init(String biocache_service_url, String q, double[] bbox, String filename, SimpleRegion region, String facetField, Boolean includeYear = true) throws IOException {
int speciesEstimate = 250000
int recordsEstimate = 26000000
int pageSize = 50000
int pageSize = 300000000 // Use a large number as a workaround for paging not working. Paging was added
// for an introduced nginx timeout value so this could be considered a revert.
// The biocache-service endpoint is streaming now, except for old sandbox instances.

String bboxTerm = null
if (bbox != null) {
Expand All @@ -336,107 +337,104 @@ class Records {
if (facetField == null) {
facetFieldTerm = ''
}
while (start < 300000000) {
String url = biocache_service_url + "/webportal/occurrences.gz?q=" + q.replace(" ", "%20") + bboxTerm + "&pageSize=" + pageSize + "&fq=year%3A*&start=" + start + "&fl=longitude,latitude" + facetFieldTerm + ",year"

int tryCount = 0
InputStream is = null
CSVReader csv = null
int maxTrys = 4
while (tryCount < maxTrys && csv == null) {
tryCount++
try {
is = getUrlStream(url)
csv = new CSVReader(new InputStreamReader(new GZIPInputStream(is)))
} catch (Exception e) {
log.error("failed try " + tryCount + " of " + maxTrys + ": " + url, e)
}
}

if (csv == null) {
throw new IOException("failed to get records from biocache.")
String yearFq = includeYear ? '&fq=year%3A*' : ''

// no longer using paging
String url = biocache_service_url + "/webportal/occurrences.gz?q=" + q.replace(" ", "%20") + bboxTerm + "&pageSize=" + pageSize + yearFq + "&start=" + start + "&fl=longitude,latitude" + facetFieldTerm + (includeYear ? ",year" : "")

int tryCount = 0
InputStream is = null
CSVReader csv = null
int maxTrys = 4
while (tryCount < maxTrys && csv == null) {
tryCount++
try {
is = getUrlStream(url)
csv = new CSVReader(new InputStreamReader(new GZIPInputStream(is)))
} catch (Exception e) {
log.error("failed try " + tryCount + " of " + maxTrys + ": " + url, e)
}
}

String[] line
int[] header = new int[4] //to contain [0]=lsid, [1]=longitude, [2]=latitude, [3]=year
int row = start
int currentCount = 0
while ((line = csv.readNext()) != null) {
if (raf != null) {
for (int i = 0; i < line.length; i++) {
if (i > 0) {
raf.write(",".bytes)
}
raf.write(line[i].bytes)
if (csv == null) {
throw new IOException("failed to get records from biocache.")
}

String[] line
int[] header = new int[4] //to contain [0]=lsid, [1]=longitude, [2]=latitude, [3]=year
int row = start
int currentCount = 0
while ((line = csv.readNext()) != null) {
if (raf != null) {
for (int i = 0; i < line.length; i++) {
if (i > 0) {
raf.write(",".bytes)
}
raf.write("\n".bytes)
raf.write(line[i].bytes)
}
currentCount++
if (currentCount == 1) {
//determine header
for (int i = 0; i < line.length; i++) {
if (line[i] == facetField) {
header[0] = i
}
if (line[i] == "longitude") {
header[1] = i
}
if (line[i] == "latitude") {
header[2] = i
}
if (line[i] == "year") {
header[3] = i
}
raf.write("\n".bytes)
}
currentCount++
if (currentCount == 1) {
//determine header
for (int i = 0; i < line.length; i++) {
if (line[i] == facetField) {
header[0] = i
}
log.debug("header info:" + header[0] + "," + header[1] + "," + header[2] + "," + header[3])
} else {
if (line.length >= 3) {
try {
double longitude = Double.parseDouble(line[header[1]])
double latitude = Double.parseDouble(line[header[2]])
if (region == null || region.isWithin_EPSG900913(longitude, latitude)) {
points.add(longitude)
points.add(latitude)
String species = facetField == null ? 'species' : line[header[0]]
Integer idx = lsidMap.get(species)
if (idx == null) {
idx = lsidMap.size()
lsidMap.put(species, idx)
}
lsidIdx.add(idx)
years.add(Short.parseShort(line[header[3]]))
if (line[i] == "longitude") {
header[1] = i
}
if (line[i] == "latitude") {
header[2] = i
}
if (line[i] == "year") {
header[3] = i
}
}
log.debug("header info:" + header[0] + "," + header[1] + "," + header[2] + "," + header[3])
} else {
if (line.length >= 3) {
try {
double longitude = Double.parseDouble(line[header[1]])
double latitude = Double.parseDouble(line[header[2]])
if (region == null || region.isWithin_EPSG900913(longitude, latitude)) {
points.add(longitude)
points.add(latitude)
String species = facetField == null ? 'species' : line[header[0]]
Integer idx = lsidMap.get(species)
if (idx == null) {
idx = lsidMap.size()
lsidMap.put(species, idx)
}
} catch (Exception ignored) {

} finally {
if (lsidIdx.size() * 2 < points.size()) {
points.remove(points.size() - 1)
points.remove(points.size() - 1)
} else if (years.size() < lsidIdx.size()) {
years.add((short) 0)
lsidIdx.add(idx)
if (includeYear) {
years.add(Short.parseShort(line[header[3]]))
}
}
} catch (Exception ignored) {

} finally {
if (lsidIdx.size() * 2 < points.size()) {
points.remove(points.size() - 1)
points.remove(points.size() - 1)
} else if (years.size() < lsidIdx.size()) {
years.add((short) 0)
}
}
}
row++
}
if (start == 0) {
start = row - 1 //offset for header
}
row++
}

csv.close()
is.close()
csv.close()
is.close()

if (is != null) {
try {
is.close()
} catch (Exception e) {
log.error(e.getMessage(), e)
}
}

if (currentCount == 0 || currentCount < pageSize) {
break
if (is != null) {
try {
is.close()
} catch (Exception e) {
log.error(e.getMessage(), e)
}
}

Expand Down Expand Up @@ -511,15 +509,15 @@ class Records {

@Override
int compare(Integer o1, Integer o2) {
return (h - 1 - ((int) ((points.get(o1) - mLat) / res)))-(h - 1 - ((int) ((points.get(o2) - mLat) / res)))
return (h - 1 - (Math.round((points.get(o1) - mLat) / res)))-(h - 1 - (Math.round((points.get(o2) - mLat) / res)))
}
})

//get row starts
int[] rowStarts = new int[height]
int row = 0
for (int i = 0; i < sortOrder.length; i++) {
int thisRow = (h - 1 - (int) ((points.get(sortOrder[i]) - mLat) / res))
int thisRow = (h - 1 - (int) Math.round((points.get(sortOrder[i]) - mLat) / res))

//handle overflow
if (thisRow >= height) {
Expand Down Expand Up @@ -578,10 +576,10 @@ class Records {

@Override
int compare(Integer o1, Integer o2) {
int v = ((int) ((points.get(o1) - mLat) / res))-((int) ((points.get(o2) - mLat) / res))
int v = (int) (Math.round((points.get(o1) - mLat) / res) - Math.round((points.get(o2) - mLat) / res))

if (v == 0) {
return ((int) ((points.get(o1 - 1) - mLong) / res))-((int) ((points.get(o2 - 1) - mLong) / res))
return (int) (Math.round((points.get(o1 - 1) - mLong) / res) - Math.round((points.get(o2 - 1) - mLong) / res))
} else {
return v
}
Expand Down