diff --git a/app/opensearch/src/main/java/de/komoot/photon/opensearch/OpenSearchResult.java b/app/opensearch/src/main/java/de/komoot/photon/opensearch/OpenSearchResult.java index a6147fc6..3cbf9506 100644 --- a/app/opensearch/src/main/java/de/komoot/photon/opensearch/OpenSearchResult.java +++ b/app/opensearch/src/main/java/de/komoot/photon/opensearch/OpenSearchResult.java @@ -1,7 +1,6 @@ package de.komoot.photon.opensearch; import de.komoot.photon.searcher.PhotonResult; -import jakarta.json.JsonArray; import org.json.JSONObject; import java.util.Map; diff --git a/src/main/java/de/komoot/photon/App.java b/src/main/java/de/komoot/photon/App.java index b4bf50e0..3f57c9c1 100644 --- a/src/main/java/de/komoot/photon/App.java +++ b/src/main/java/de/komoot/photon/App.java @@ -2,7 +2,8 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.ParameterException; -import de.komoot.photon.nominatim.NominatimConnector; +import de.komoot.photon.nominatim.ImportThread; +import de.komoot.photon.nominatim.NominatimImporter; import de.komoot.photon.nominatim.NominatimUpdater; import de.komoot.photon.searcher.ReverseHandler; import de.komoot.photon.searcher.SearchHandler; @@ -14,7 +15,8 @@ import java.io.FileNotFoundException; import java.io.IOException; -import java.util.Date; +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; import static spark.Spark.*; @@ -107,9 +109,8 @@ private static void startJsonDump(CommandLineArgs args) { try { final String filename = args.getJsonDump(); final JsonDumper jsonDumper = new JsonDumper(filename, args.getLanguages(), args.getExtraTags()); - NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); - nominatimConnector.setImporter(jsonDumper); - nominatimConnector.readEntireDatabase(args.getCountryCodes()); + + importFromDatabase(args, jsonDumper); LOGGER.info("Json dump was created: {}", filename); } catch (FileNotFoundException e) { throw new UsageException("Cannot create dump: " + e.getMessage()); @@ -121,22 +122,95 @@ private static void startJsonDump(CommandLineArgs args) { * Read all data from a Nominatim database and import it into a Photon database. */ private static void startNominatimImport(CommandLineArgs args, Server esServer) { - DatabaseProperties dbProperties; - NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); - Date importDate = nominatimConnector.getLastImportDate(); + final var languages = initDatabase(args, esServer); + + LOGGER.info("Starting import from nominatim to photon with languages: {}", String.join(",", languages)); + importFromDatabase(args, esServer.createImporter(languages, args.getExtraTags())); + + LOGGER.info("Imported data from nominatim to photon with languages: {}", String.join(",", languages)); + } + + private static String[] initDatabase(CommandLineArgs args, Server esServer) { + final var nominatimConnector = new NominatimImporter(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); + final Date importDate = nominatimConnector.getLastImportDate(); + try { - dbProperties = esServer.recreateIndex(args.getLanguages(), importDate, args.getSupportStructuredQueries()); // clear out previous data + // Clear out previous data. + var dbProperties = esServer.recreateIndex(args.getLanguages(), importDate, args.getSupportStructuredQueries()); + return dbProperties.getLanguages(); } catch (IOException e) { throw new UsageException("Cannot setup index, elastic search config files not readable"); } + } + + private static void importFromDatabase(CommandLineArgs args, Importer importer) { + final var connector = new NominatimImporter(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); + connector.prepareDatabase(); + connector.loadCountryNames(); - LOGGER.info("Starting import from nominatim to photon with languages: {}", String.join(",", dbProperties.getLanguages())); - nominatimConnector.setImporter(esServer.createImporter(dbProperties.getLanguages(), args.getExtraTags())); - nominatimConnector.readEntireDatabase(args.getCountryCodes()); + String[] countries = args.getCountryCodes(); + + if (countries == null || countries.length == 0) { + countries = connector.getCountriesFromDatabase(); + } else { + countries = Arrays.stream(countries).map(String::trim).filter(s -> !s.isBlank()).toArray(String[]::new); + } + + final int numThreads = args.getThreads(); + ImportThread importThread = new ImportThread(importer); + + try { + + if (numThreads == 1) { + for (var country : countries) { + connector.readCountry(country, importThread); + } + } else { + final Queue todolist = new ConcurrentLinkedQueue<>(List.of(countries)); + + final List readerThreads = new ArrayList<>(numThreads); + + for (int i = 0; i < numThreads; ++i) { + final NominatimImporter threadConnector; + if (i > 0) { + threadConnector = new NominatimImporter(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); + threadConnector.loadCountryNames(); + } else { + threadConnector = connector; + } + final int threadno = i; + Runnable runner = () -> { + String nextCc = todolist.poll(); + while (nextCc != null) { + LOGGER.info("Thread {}: reading country '{}'", threadno, nextCc); + threadConnector.readCountry(nextCc, importThread); + nextCc = todolist.poll(); + } + }; + Thread thread = new Thread(runner); + thread.start(); + readerThreads.add(thread); + } + readerThreads.forEach(t -> { + while (true) { + try { + t.join(); + break; + } catch (InterruptedException e) { + LOGGER.warn("Thread interrupted:", e); + // Restore interrupted state. + Thread.currentThread().interrupt(); + } + } + }); + } + } finally { + importThread.finish(); + } - LOGGER.info("Imported data from nominatim to photon with languages: {}", String.join(",", dbProperties.getLanguages())); } + private static void startNominatimUpdateInit(CommandLineArgs args) { NominatimUpdater nominatimUpdater = new NominatimUpdater(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); nominatimUpdater.initUpdates(args.getNominatimUpdateInit()); diff --git a/src/main/java/de/komoot/photon/CommandLineArgs.java b/src/main/java/de/komoot/photon/CommandLineArgs.java index 724f4835..cd496a2b 100644 --- a/src/main/java/de/komoot/photon/CommandLineArgs.java +++ b/src/main/java/de/komoot/photon/CommandLineArgs.java @@ -14,6 +14,9 @@ @Parameters(parametersValidators = CorsMutuallyExclusiveValidator.class) public class CommandLineArgs { + @Parameter(names = "-j", description = "Number of threads to use for import.") + private int threads = 1; + @Parameter(names = "-structured", description = "Enable support for structured queries.") private boolean supportStructuredQueries = false; @@ -107,6 +110,10 @@ public String[] getLanguages() { return getLanguages(true); } + public int getThreads() { + return Integer.min(10, Integer.max(0, threads)); + } + public String getCluster() { return this.cluster; } diff --git a/src/main/java/de/komoot/photon/PhotonDoc.java b/src/main/java/de/komoot/photon/PhotonDoc.java index 147cf151..43b90cf4 100644 --- a/src/main/java/de/komoot/photon/PhotonDoc.java +++ b/src/main/java/de/komoot/photon/PhotonDoc.java @@ -1,5 +1,6 @@ package de.komoot.photon; +import de.komoot.photon.nominatim.model.AddressRow; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.Point; @@ -217,17 +218,27 @@ public boolean isUsefulForIndex() { private void extractAddress(Map address, AddressType addressType, String addressFieldName) { String field = address.get(addressFieldName); - if (field != null) { - Map map = addressParts.computeIfAbsent(addressType, k -> new HashMap<>()); + if (field == null) { + return; + } + Map map = addressParts.get(addressType); + if (map == null) { + map = new HashMap<>(); + map.put("name", field); + addressParts.put(addressType, map); + } else { String existingName = map.get("name"); if (!field.equals(existingName)) { + // Make a copy of the original name map because the map is reused for other addresses. + map = new HashMap<>(map); LOGGER.debug("Replacing {} name '{}' with '{}' for osmId #{}", addressFieldName, existingName, field, osmId); // we keep the former name in the context as it might be helpful when looking up typos if (!Objects.isNull(existingName)) { context.add(Collections.singletonMap("formerName", existingName)); } map.put("name", field); + addressParts.put(addressType, map); } } } @@ -241,6 +252,23 @@ public boolean setAddressPartIfNew(AddressType addressType, Map return addressParts.computeIfAbsent(addressType, k -> names) == names; } + /** + * Complete address data from a list of address rows. + */ + public void completePlace(List addresses) { + final AddressType doctype = getAddressType(); + for (AddressRow address : addresses) { + final AddressType atype = address.getAddressType(); + + if (atype != null + && (atype == doctype || !setAddressPartIfNew(atype, address.getName())) + && address.isUsefulForContext()) { + // no specifically handled item, check if useful for context + getContext().add(address.getName()); + } + } + } + public void setCountry(Map names) { addressParts.put(AddressType.COUNTRY, names); } diff --git a/src/main/java/de/komoot/photon/nominatim/DBDataAdapter.java b/src/main/java/de/komoot/photon/nominatim/DBDataAdapter.java index 7608a9ab..6e76426f 100644 --- a/src/main/java/de/komoot/photon/nominatim/DBDataAdapter.java +++ b/src/main/java/de/komoot/photon/nominatim/DBDataAdapter.java @@ -30,4 +30,9 @@ public interface DBDataAdapter { * Wrap a DELETE statement with a RETURNING clause. */ String deleteReturning(String deleteSQL, String columns); + + /** + * Wrap function to create a json array from a SELECT. + */ + String jsonArrayFromSelect(String valueSQL, String fromSQL); } diff --git a/src/main/java/de/komoot/photon/nominatim/ImportThread.java b/src/main/java/de/komoot/photon/nominatim/ImportThread.java index 86e941e5..7d9b8dd4 100644 --- a/src/main/java/de/komoot/photon/nominatim/ImportThread.java +++ b/src/main/java/de/komoot/photon/nominatim/ImportThread.java @@ -11,12 +11,12 @@ /** * Worker thread for bulk importing data from a Nominatim database. */ -class ImportThread { +public class ImportThread { private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(ImportThread.class); private static final int PROGRESS_INTERVAL = 50000; - private static final NominatimResult FINAL_DOCUMENT = new NominatimResult(new PhotonDoc(0, null, 0, null, null)); - private final BlockingQueue documents = new LinkedBlockingDeque<>(20); + private static final NominatimResult FINAL_DOCUMENT = NominatimResult.fromAddress(new PhotonDoc(0, null, 0, null, null), null); + private final BlockingQueue documents = new LinkedBlockingDeque<>(100); private final AtomicLong counter = new AtomicLong(); private final Importer importer; private final Thread thread; @@ -70,7 +70,8 @@ public void finish() { Thread.currentThread().interrupt(); } } - LOGGER.info("Finished import of {} photon documents.", counter.longValue()); + LOGGER.info("Finished import of {} photon documents. (Total processing time: {}s)", + counter.longValue(), (System.currentTimeMillis() - startMillis)/1000); } private class ImportRunnable implements Runnable { diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java index eebe9ab3..e5b28d6e 100644 --- a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java +++ b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java @@ -1,154 +1,29 @@ package de.komoot.photon.nominatim; -import org.locationtech.jts.geom.Geometry; -import de.komoot.photon.Importer; -import de.komoot.photon.PhotonDoc; -import de.komoot.photon.nominatim.model.AddressRow; -import de.komoot.photon.nominatim.model.AddressType; import org.apache.commons.dbcp2.BasicDataSource; -import org.slf4j.Logger; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; /** - * Importer for data from a Mominatim database. + * Base class for workers connecting to a Nominatim database */ public class NominatimConnector { - private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(NominatimConnector.class); + protected final DBDataAdapter dbutils; + protected final JdbcTemplate template; + protected final TransactionTemplate txTemplate; + protected Map> countryNames; + protected final boolean hasNewStyleInterpolation; - private static final String SELECT_COLS_PLACEX = "SELECT place_id, osm_type, osm_id, class, type, name, postcode, address, extratags, ST_Envelope(geometry) AS bbox, parent_place_id, linked_place_id, rank_address, rank_search, importance, country_code, centroid"; - private static final String SELECT_COLS_ADDRESS = "SELECT p.name, p.class, p.type, p.rank_address"; - - private final DBDataAdapter dbutils; - private final JdbcTemplate template; - private Map> countryNames; - - /** - * Map a row from location_property_osmline (address interpolation lines) to a photon doc. - * This may be old-style interpolation (using interpolationtype) or - * new-style interpolation (using step). - */ - private final RowMapper osmlineRowMapper; - private final String selectOsmlineSql; - private Importer importer; - - - /** - * Maps a placex row in nominatim to a photon doc. - * Some attributes are still missing and can be derived by connected address items. - */ - private final RowMapper placeRowMapper = new RowMapper<>() { - @Override - public NominatimResult mapRow(ResultSet rs, int rowNum) throws SQLException { - Map address = dbutils.getMap(rs, "address"); - PhotonDoc doc = new PhotonDoc(rs.getLong("place_id"), - rs.getString("osm_type"), rs.getLong("osm_id"), - rs.getString("class"), rs.getString("type")) - .names(dbutils.getMap(rs, "name")) - .extraTags(dbutils.getMap(rs, "extratags")) - .bbox(dbutils.extractGeometry(rs, "bbox")) - .parentPlaceId(rs.getLong("parent_place_id")) - .countryCode(rs.getString("country_code")) - .centroid(dbutils.extractGeometry(rs, "centroid")) - .linkedPlaceId(rs.getLong("linked_place_id")) - .rankAddress(rs.getInt("rank_address")) - .postcode(rs.getString("postcode")); - - double importance = rs.getDouble("importance"); - doc.importance(rs.wasNull() ? (0.75 - rs.getInt("rank_search") / 40d) : importance); - - completePlace(doc); - // Add address last, so it takes precedence. - doc.address(address); - - doc.setCountry(getCountryNames(rs.getString("country_code"))); - - NominatimResult result = new NominatimResult(doc); - result.addHousenumbersFromAddress(address); - - return result; - } - }; - - /** - * Construct a new importer. - * - * @param host database host - * @param port database port - * @param database database name - * @param username db username - * @param password db username's password - */ - public NominatimConnector(String host, int port, String database, String username, String password) { - this(host, port, database, username, password, new PostgisDataAdapter()); - } - - public NominatimConnector(String host, int port, String database, String username, String password, DBDataAdapter dataAdapter) { - BasicDataSource dataSource = buildDataSource(host, port, database, username, password, false); - - template = new JdbcTemplate(dataSource); - template.setFetchSize(100000); - - dbutils = dataAdapter; - - // Setup handling of interpolation table. There are two different formats depending on the Nominatim version. - if (dbutils.hasColumn(template, "location_property_osmline", "step")) { - // new-style interpolations - selectOsmlineSql = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, step, postcode, country_code, linegeo"; - osmlineRowMapper = (rs, rownum) -> { - Geometry geometry = dbutils.extractGeometry(rs, "linegeo"); - - PhotonDoc doc = new PhotonDoc(rs.getLong("place_id"), "W", rs.getLong("osm_id"), - "place", "house_number") - .parentPlaceId(rs.getLong("parent_place_id")) - .countryCode(rs.getString("country_code")) - .postcode(rs.getString("postcode")); - - completePlace(doc); - - doc.setCountry(getCountryNames(rs.getString("country_code"))); - - NominatimResult result = new NominatimResult(doc); - result.addHouseNumbersFromInterpolation(rs.getLong("startnumber"), rs.getLong("endnumber"), - rs.getLong("step"), geometry); - - return result; - }; - } else { - // old-style interpolations - selectOsmlineSql = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, interpolationtype, postcode, country_code, linegeo"; - osmlineRowMapper = (rs, rownum) -> { - Geometry geometry = dbutils.extractGeometry(rs, "linegeo"); - - PhotonDoc doc = new PhotonDoc(rs.getLong("place_id"), "W", rs.getLong("osm_id"), - "place", "house_number") - .parentPlaceId(rs.getLong("parent_place_id")) - .countryCode(rs.getString("country_code")) - .postcode(rs.getString("postcode")); - - completePlace(doc); - - doc.setCountry(getCountryNames(rs.getString("country_code"))); - - NominatimResult result = new NominatimResult(doc); - result.addHouseNumbersFromInterpolation(rs.getLong("startnumber"), rs.getLong("endnumber"), - rs.getString("interpolationtype"), geometry); - - return result; - }; - } - } - - - static BasicDataSource buildDataSource(String host, int port, String database, String username, String password, boolean autocommit) { + protected NominatimConnector(String host, int port, String database, String username, String password, DBDataAdapter dataAdapter) { BasicDataSource dataSource = new BasicDataSource(); dataSource.setUrl(String.format("jdbc:postgresql://%s:%d/%s", host, port, database)); @@ -156,145 +31,17 @@ static BasicDataSource buildDataSource(String host, int port, String database, S if (password != null) { dataSource.setPassword(password); } - dataSource.setDefaultAutoCommit(autocommit); - return dataSource; - } - - private Map getCountryNames(String countrycode) { - if (countryNames == null) { - countryNames = new HashMap<>(); - template.query("SELECT country_code, name FROM country_name", rs -> { - countryNames.put(rs.getString("country_code"), dbutils.getMap(rs, "name")); - }); - } - - return countryNames.get(countrycode); - } - - public void setImporter(Importer importer) { - this.importer = importer; - } - - public List getByPlaceId(long placeId) { - List result = template.query(SELECT_COLS_PLACEX + " FROM placex WHERE place_id = ? and indexed_status = 0", - placeRowMapper, placeId); - - return result.isEmpty() ? null : result.get(0).getDocsWithHousenumber(); - } - - public List getInterpolationsByPlaceId(long placeId) { - List result = template.query(selectOsmlineSql - + " FROM location_property_osmline WHERE place_id = ? and indexed_status = 0", - osmlineRowMapper, placeId); - - return result.isEmpty() ? null : result.get(0).getDocsWithHousenumber(); - } - - private long parentPlaceId = -1; - private List parentTerms = null; - - List getAddresses(PhotonDoc doc) { - RowMapper rowMapper = (rs, rowNum) -> new AddressRow( - dbutils.getMap(rs, "name"), - rs.getString("class"), - rs.getString("type"), - rs.getInt("rank_address") - ); - - AddressType atype = doc.getAddressType(); - - if (atype == null || atype == AddressType.COUNTRY) { - return Collections.emptyList(); - } - - List terms = null; - if (atype == AddressType.HOUSE) { - long placeId = doc.getParentPlaceId(); - if (placeId != parentPlaceId) { - parentTerms = template.query(SELECT_COLS_ADDRESS - + " FROM placex p, place_addressline pa" - + " WHERE p.place_id = pa.address_place_id and pa.place_id = ?" - + " and pa.cached_rank_address > 4 and pa.address_place_id != ? and pa.isaddress" - + " ORDER BY rank_address desc, fromarea desc, distance asc, rank_search desc", - rowMapper, placeId, placeId); + // Keep disabled or server-side cursors won't work. + dataSource.setDefaultAutoCommit(false); - // need to add the term for the parent place ID itself - parentTerms.addAll(0, template.query(SELECT_COLS_ADDRESS + " FROM placex p WHERE p.place_id = ?", - rowMapper, placeId)); - parentPlaceId = placeId; - } - terms = parentTerms; - - } else { - long placeId = doc.getPlaceId(); - terms = template.query(SELECT_COLS_ADDRESS - + " FROM placex p, place_addressline pa" - + " WHERE p.place_id = pa.address_place_id and pa.place_id = ?" - + " and pa.cached_rank_address > 4 and pa.address_place_id != ? and pa.isaddress" - + " ORDER BY rank_address desc, fromarea desc, distance asc, rank_search desc", - rowMapper, placeId, placeId); - } - - return terms; - } - - static String convertCountryCode(String... countryCodes) { - String countryCodeStr = ""; - for (String cc : countryCodes) { - if (cc.isEmpty()) - continue; - if (cc.length() != 2) - throw new IllegalArgumentException("country code invalid " + cc); - if (!countryCodeStr.isEmpty()) - countryCodeStr += ","; - countryCodeStr += "'" + cc.toLowerCase() + "'"; - } - return countryCodeStr; - } + txTemplate = new TransactionTemplate(new DataSourceTransactionManager(dataSource)); - /** - * Parse every relevant row in placex, create a corresponding document and call the {@link #importer} for each document. - */ - public void readEntireDatabase(String... countryCodes) { - String andCountryCodeStr = ""; - String countryCodeStr = convertCountryCode(countryCodes); - if (!countryCodeStr.isEmpty()) { - andCountryCodeStr = "AND country_code in (" + countryCodeStr + ")"; - } - - LOGGER.info("Start importing documents from nominatim ({})", countryCodeStr.isEmpty() ? "global" : countryCodeStr); - - ImportThread importThread = new ImportThread(importer); - - try { - template.query(SELECT_COLS_PLACEX + " FROM placex " + - " WHERE linked_place_id IS NULL AND centroid IS NOT NULL " + andCountryCodeStr + - " ORDER BY geometry_sector, parent_place_id; ", rs -> { - // turns a placex row into a photon document that gathers all de-normalised information - NominatimResult docs = placeRowMapper.mapRow(rs, 0); - assert (docs != null); - - if (docs.isUsefulForIndex()) { - importThread.addDocument(docs); - } - }); - - template.query(selectOsmlineSql + " FROM location_property_osmline " + - "WHERE startnumber is not null " + - andCountryCodeStr + - " ORDER BY geometry_sector, parent_place_id; ", rs -> { - NominatimResult docs = osmlineRowMapper.mapRow(rs, 0); - assert (docs != null); - - if (docs.isUsefulForIndex()) { - importThread.addDocument(docs); - } - }); + template = new JdbcTemplate(dataSource); + template.setFetchSize(100000); - } finally { - importThread.finish(); - } + dbutils = dataAdapter; + hasNewStyleInterpolation = dbutils.hasColumn(template, "location_property_osmline", "step"); } public Date getLastImportDate() { @@ -310,27 +57,15 @@ public Date mapRow(ResultSet rs, int rowNum) throws SQLException { return importDates.get(0); } - /** - * Query Nominatim's address hierarchy to complete photon doc with missing data (like country, city, street, ...) - * - * @param doc - */ - private void completePlace(PhotonDoc doc) { - final List addresses = getAddresses(doc); - final AddressType doctype = doc.getAddressType(); - for (AddressRow address : addresses) { - AddressType atype = address.getAddressType(); - - if (atype != null - && (atype == doctype || !doc.setAddressPartIfNew(atype, address.getName())) - && address.isUsefulForContext()) { - // no specifically handled item, check if useful for context - doc.getContext().add(address.getName()); - } + public void loadCountryNames() { + if (countryNames == null) { + countryNames = new HashMap<>(); + // Default for places outside any country. + countryNames.put("", new HashMap<>()); + template.query("SELECT country_code, name FROM country_name", rs -> { + countryNames.put(rs.getString("country_code"), dbutils.getMap(rs, "name")); + }); } } - public DBDataAdapter getDataAdaptor() { - return dbutils; - } } diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java b/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java new file mode 100644 index 00000000..8c5ba686 --- /dev/null +++ b/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java @@ -0,0 +1,206 @@ +package de.komoot.photon.nominatim; + +import de.komoot.photon.PhotonDoc; +import de.komoot.photon.nominatim.model.AddressRow; +import de.komoot.photon.nominatim.model.NominatimAddressCache; +import de.komoot.photon.nominatim.model.OsmlineRowMapper; +import de.komoot.photon.nominatim.model.PlaceRowMapper; +import org.locationtech.jts.geom.Geometry; +import org.slf4j.Logger; + +import java.sql.Types; +import java.util.Map; + +/** + * Importer for data from a Nominatim database. + */ +public class NominatimImporter extends NominatimConnector { + private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(NominatimImporter.class); + + public NominatimImporter(String host, int port, String database, String username, String password) { + this(host, port, database, username, password, new PostgisDataAdapter()); + } + + public NominatimImporter(String host, int port, String database, String username, String password, DBDataAdapter dataAdapter) { + super(host, port, database, username, password, dataAdapter); + } + + + /** + * Parse every relevant row in placex and location_osmline + * for the given country. Also imports place from county-less places. + */ + public void readCountry(String countryCode, ImportThread importThread) { + // Make sure, country names are available. + loadCountryNames(); + final var cnames = countryNames.get(countryCode); + if (cnames == null) { + LOGGER.warn("Unknown country code {}. Skipping.", countryCode); + return; + } + + final String countrySQL; + final Object[] sqlArgs; + final int[] sqlArgTypes; + if ("".equals(countryCode)) { + countrySQL = "country_code is null"; + sqlArgs = new Object[0]; + sqlArgTypes = new int[0]; + } else { + countrySQL = "country_code = ?"; + sqlArgs = new Object[]{countryCode}; + sqlArgTypes = new int[]{Types.VARCHAR}; + } + + NominatimAddressCache addressCache = new NominatimAddressCache(); + addressCache.loadCountryAddresses(template, dbutils, countryCode); + + final PlaceRowMapper placeRowMapper = new PlaceRowMapper(dbutils); + // First read ranks below 30, independent places + template.query( + "SELECT place_id, osm_type, osm_id, class, type, name, postcode," + + " address, extratags, ST_Envelope(geometry) AS bbox, parent_place_id," + + " linked_place_id, rank_address, rank_search, importance, country_code, centroid," + + dbutils.jsonArrayFromSelect( + "address_place_id", + "FROM place_addressline pa " + + " WHERE pa.place_id = p.place_id AND isaddress" + + " ORDER BY cached_rank_address DESC") + " as addresslines" + + " FROM placex p" + + " WHERE linked_place_id IS NULL AND centroid IS NOT NULL AND " + countrySQL + + " AND rank_search < 30" + + " ORDER BY geometry_sector, parent_place_id", + sqlArgs, sqlArgTypes, rs -> { + final PhotonDoc doc = placeRowMapper.mapRow(rs, 0); + final Map address = dbutils.getMap(rs, "address"); + + assert (doc != null); + + doc.completePlace(addressCache.getAddressList(rs.getString("addresslines"))); + doc.address(address); // take precedence over computed address + doc.setCountry(cnames); + + var result = NominatimResult.fromAddress(doc, address); + + if (result.isUsefulForIndex()) { + importThread.addDocument(result); + } + }); + + // Next get all POIs/housenumbers. + template.query( + "SELECT p.place_id, p.osm_type, p.osm_id, p.class, p.type, p.name, p.postcode," + + " p.address, p.extratags, ST_Envelope(p.geometry) AS bbox, p.parent_place_id," + + " p.linked_place_id, p.rank_address, p.rank_search, p.importance, p.country_code, p.centroid," + + " parent.class as parent_class, parent.type as parent_type," + + " parent.rank_address as parent_rank_address, parent.name as parent_name, " + + dbutils.jsonArrayFromSelect( + "address_place_id", + "FROM place_addressline pa " + + " WHERE pa.place_id IN (p.place_id, coalesce(p.parent_place_id, p.place_id)) AND isaddress" + + " ORDER BY cached_rank_address DESC, pa.place_id = p.place_id DESC") + " as addresslines" + + " FROM placex p LEFT JOIN placex parent ON p.parent_place_id = parent.place_id" + + " WHERE p.linked_place_id IS NULL AND p.centroid IS NOT NULL AND p." + countrySQL + + " AND p.rank_search = 30 " + + " ORDER BY p.geometry_sector", + sqlArgs, sqlArgTypes, rs -> { + final PhotonDoc doc = placeRowMapper.mapRow(rs, 0); + final Map address = dbutils.getMap(rs, "address"); + + assert (doc != null); + + final var addressPlaces = addressCache.getAddressList(rs.getString("addresslines")); + if (rs.getString("parent_class") != null) { + addressPlaces.add(0, new AddressRow( + dbutils.getMap(rs, "parent_name"), + rs.getString("parent_class"), + rs.getString("parent_type"), + rs.getInt("parent_rank_address"))); + } + doc.completePlace(addressPlaces); + doc.address(address); // take precedence over computed address + doc.setCountry(cnames); + + var result = NominatimResult.fromAddress(doc, address); + + if (result.isUsefulForIndex()) { + importThread.addDocument(result); + } + }); + + final OsmlineRowMapper osmlineRowMapper = new OsmlineRowMapper(); + template.query( + "SELECT p.place_id, p.osm_id, p.parent_place_id, p.startnumber, p.endnumber, p.postcode, p.country_code, p.linegeo," + + (hasNewStyleInterpolation ? " p.step," : " p.interpolationtype,") + + " parent.class as parent_class, parent.type as parent_type," + + " parent.rank_address as parent_rank_address, parent.name as parent_name, " + + dbutils.jsonArrayFromSelect( + "address_place_id", + "FROM place_addressline pa " + + " WHERE pa.place_id IN (p.place_id, coalesce(p.parent_place_id, p.place_id)) AND isaddress" + + " ORDER BY cached_rank_address DESC, pa.place_id = p.place_id DESC") + " as addresslines" + + " FROM location_property_osmline p LEFT JOIN placex parent ON p.parent_place_id = parent.place_id" + + " WHERE startnumber is not null AND p." + countrySQL + + " ORDER BY p.geometry_sector, p.parent_place_id", + sqlArgs, sqlArgTypes, rs -> { + final PhotonDoc doc = osmlineRowMapper.mapRow(rs, 0); + + final var addressPlaces = addressCache.getAddressList(rs.getString("addresslines")); + if (rs.getString("parent_class") != null) { + addressPlaces.add(0, new AddressRow( + dbutils.getMap(rs, "parent_name"), + rs.getString("parent_class"), + rs.getString("parent_type"), + rs.getInt("parent_rank_address"))); + } + doc.completePlace(addressPlaces); + + doc.setCountry(cnames); + + final Geometry geometry = dbutils.extractGeometry(rs, "linegeo"); + final NominatimResult docs; + if (hasNewStyleInterpolation) { + docs = NominatimResult.fromInterpolation( + doc, rs.getLong("startnumber"), rs.getLong("endnumber"), + rs.getLong("step"), geometry); + } else { + docs = NominatimResult.fromInterpolation( + doc, rs.getLong("startnumber"), rs.getLong("endnumber"), + rs.getString("interpolationtype"), geometry); + } + + if (docs.isUsefulForIndex()) { + importThread.addDocument(docs); + } + }); + + } + + + /** + * Prepare the database for export. + * + * This function ensures that the proper index are available and if + * not will create them. This may take a while. + */ + public void prepareDatabase() { + txTemplate.execute(status -> { + Integer indexRowNum = template.queryForObject( + "SELECT count(*) FROM pg_indexes WHERE tablename = 'placex' AND indexdef LIKE '%(country_code)'", + Integer.class); + + if (indexRowNum == null || indexRowNum == 0) { + LOGGER.info("Creating index over countries."); + template.execute("CREATE INDEX ON placex (country_code)"); + } + + return 0; + }); + } + + public String[] getCountriesFromDatabase() { + loadCountryNames(); + + return countryNames.keySet().toArray(new String[0]); + } +} diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimResult.java b/src/main/java/de/komoot/photon/nominatim/NominatimResult.java index 75a13cee..938c6d37 100644 --- a/src/main/java/de/komoot/photon/nominatim/NominatimResult.java +++ b/src/main/java/de/komoot/photon/nominatim/NominatimResult.java @@ -1,10 +1,10 @@ package de.komoot.photon.nominatim; +import de.komoot.photon.PhotonDoc; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.Point; import org.locationtech.jts.linearref.LengthIndexedLine; -import de.komoot.photon.PhotonDoc; import java.util.*; import java.util.regex.Pattern; @@ -20,7 +20,7 @@ class NominatimResult { private static final Pattern HOUSENUMBER_CHECK = Pattern.compile("(\\A|.*,)[^\\d,]{3,}(,.*|\\Z)"); private static final Pattern HOUSENUMBER_SPLIT = Pattern.compile("\\s*[;,]\\s*"); - public NominatimResult(PhotonDoc baseobj) { + private NominatimResult(PhotonDoc baseobj) { doc = baseobj; housenumbers = null; } @@ -58,7 +58,7 @@ List getDocsWithHousenumber() { * * @param str House number string. May be null, in which case nothing is added. */ - public void addHousenumbersFromString(String str) { + private void addHousenumbersFromString(String str) { if (str == null || str.isEmpty()) return; @@ -68,9 +68,6 @@ public void addHousenumbersFromString(String str) { return; } - if (housenumbers == null) - housenumbers = new HashMap<>(); - String[] parts = HOUSENUMBER_SPLIT.split(str); for (String part : parts) { String h = part.trim(); @@ -79,14 +76,17 @@ public void addHousenumbersFromString(String str) { } } - public void addHousenumbersFromAddress(Map address) { - if (address == null) { - return; + public static NominatimResult fromAddress(PhotonDoc doc, Map address) { + NominatimResult result = new NominatimResult(doc); + + if (address != null) { + result.housenumbers = new HashMap<>(); + result.addHousenumbersFromString(address.get("housenumber")); + result.addHousenumbersFromString(address.get("streetnumber")); + result.addHousenumbersFromString(address.get("conscriptionnumber")); } - addHousenumbersFromString(address.get("housenumber")); - addHousenumbersFromString(address.get("streetnumber")); - addHousenumbersFromString(address.get("conscriptionnumber")); + return result; } /** @@ -101,35 +101,36 @@ public void addHousenumbersFromAddress(Map address) { * @param interpoltype Kind of interpolation (odd, even or all). * @param geom Geometry of the interpolation line. */ - public void addHouseNumbersFromInterpolation(long first, long last, String interpoltype, Geometry geom) { - if (last <= first || (last - first) > 1000) - return; + public static NominatimResult fromInterpolation(PhotonDoc doc, long first, long last, String interpoltype, Geometry geom) { + NominatimResult result = new NominatimResult(doc); + if (last > first && (last - first) < 1000) { + result.housenumbers = new HashMap<>(); - if (housenumbers == null) - housenumbers = new HashMap<>(); - - LengthIndexedLine line = new LengthIndexedLine(geom); - double si = line.getStartIndex(); - double ei = line.getEndIndex(); - double lstep = (ei - si) / (last - first); - - // leave out first and last, they have a distinct OSM node that is already indexed - long step = 2; - long num = 1; - if (interpoltype.equals("odd")) { - if (first % 2 == 1) - ++num; - } else if (interpoltype.equals("even")) { - if (first % 2 == 0) - ++num; - } else { - step = 1; - } + LengthIndexedLine line = new LengthIndexedLine(geom); + double si = line.getStartIndex(); + double ei = line.getEndIndex(); + double lstep = (ei - si) / (last - first); - GeometryFactory fac = geom.getFactory(); - for (; first + num < last; num += step) { - housenumbers.put(String.valueOf(num + first), fac.createPoint(line.extractPoint(si + lstep * num))); + // leave out first and last, they have a distinct OSM node that is already indexed + long step = 2; + long num = 1; + if (interpoltype.equals("odd")) { + if (first % 2 == 1) + ++num; + } else if (interpoltype.equals("even")) { + if (first % 2 == 0) + ++num; + } else { + step = 1; + } + + GeometryFactory fac = geom.getFactory(); + for (; first + num < last; num += step) { + result.housenumbers.put(String.valueOf(num + first), fac.createPoint(line.extractPoint(si + lstep * num))); + } } + + return result; } /** @@ -143,25 +144,27 @@ public void addHouseNumbersFromInterpolation(long first, long last, String inter * @param step Gap to leave between each interpolated house number. * @param geom Geometry of the interpolation line. */ - public void addHouseNumbersFromInterpolation(long first, long last, long step, Geometry geom) { - if (last < first || (last - first) > 1000) - return; - - if (housenumbers == null) - housenumbers = new HashMap<>(); - - if (last == first) { - housenumbers.put(String.valueOf(first), geom.getCentroid()); - } else { - LengthIndexedLine line = new LengthIndexedLine(geom); - double si = line.getStartIndex(); - double ei = line.getEndIndex(); - double lstep = (ei - si) / (last - first); - - GeometryFactory fac = geom.getFactory(); - for (long num = 0; first + num <= last; num += step) { - housenumbers.put(String.valueOf(num + first), fac.createPoint(line.extractPoint(si + lstep * num))); + public static NominatimResult fromInterpolation(PhotonDoc doc, long first, long last, long step, Geometry geom) { + NominatimResult result = new NominatimResult(doc); + if (last >= first && (last - first) < 1000) { + result.housenumbers = new HashMap<>(); + + if (last == first) { + result.housenumbers.put(String.valueOf(first), geom.getCentroid()); + } else { + LengthIndexedLine line = new LengthIndexedLine(geom); + double si = line.getStartIndex(); + double ei = line.getEndIndex(); + double lstep = (ei - si) / (last - first); + + GeometryFactory fac = geom.getFactory(); + for (long num = 0; first + num <= last; num += step) { + result.housenumbers.put(String.valueOf(num + first), fac.createPoint(line.extractPoint(si + lstep * num))); + } } + } + + return result; } } diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java b/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java index 3e3637aa..d1afabfc 100644 --- a/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java +++ b/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java @@ -2,22 +2,26 @@ import de.komoot.photon.PhotonDoc; import de.komoot.photon.Updater; -import de.komoot.photon.nominatim.model.UpdateRow; -import org.apache.commons.dbcp2.BasicDataSource; -import org.springframework.jdbc.core.JdbcTemplate; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.Date; -import java.util.List; +import de.komoot.photon.nominatim.model.*; +import org.locationtech.jts.geom.Geometry; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallbackWithoutResult; + +import java.util.*; import java.util.concurrent.locks.ReentrantLock; /** * Importer for updates from a Nominatim database. */ -public class NominatimUpdater { +public class NominatimUpdater extends NominatimConnector { private static final org.slf4j.Logger LOGGER = org.slf4j.LoggerFactory.getLogger(NominatimUpdater.class); + private static final String SELECT_COLS_PLACEX = "SELECT place_id, osm_type, osm_id, class, type, name, postcode, address, extratags, ST_Envelope(geometry) AS bbox, parent_place_id, linked_place_id, rank_address, rank_search, importance, country_code, centroid"; + private static final String SELECT_COLS_ADDRESS = "SELECT p.name, p.class, p.type, p.rank_address"; + private static final String SELECT_OSMLINE_OLD_STYLE = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, interpolationtype, postcode, country_code, linegeo"; + private static final String SELECT_OSMLINE_NEW_STYLE = "SELECT place_id, osm_id, parent_place_id, startnumber, endnumber, step, postcode, country_code, linegeo"; + private static final String TRIGGER_SQL = "DROP TABLE IF EXISTS photon_updates;" + "CREATE TABLE photon_updates (rel TEXT, place_id BIGINT," @@ -45,20 +49,82 @@ public class NominatimUpdater { + " AFTER DELETE ON location_property_osmline FOR EACH ROW" + " EXECUTE FUNCTION photon_update_func()"; - private final JdbcTemplate template; - private final NominatimConnector exporter; - private Updater updater; + /** + * Map a row from location_property_osmline (address interpolation lines) to a photon doc. + * This may be old-style interpolation (using interpolationtype) or + * new-style interpolation (using step). + */ + private final RowMapper osmlineToNominatimResult; + + + /** + * Maps a placex row in nominatim to a photon doc. + * Some attributes are still missing and can be derived by connected address items. + */ + private final RowMapper placeToNominatimResult; + + /** * Lock to prevent thread from updating concurrently. */ private ReentrantLock updateLock = new ReentrantLock(); - public Date getLastImportDate() { - return exporter.getLastImportDate(); + + // One-item cache for address terms. Speeds up processing of rank 30 objects. + private long parentPlaceId = -1; + private List parentTerms = null; + + + public NominatimUpdater(String host, int port, String database, String username, String password) { + this(host, port, database, username, password, new PostgisDataAdapter()); } + public NominatimUpdater(String host, int port, String database, String username, String password, DBDataAdapter dataAdapter) { + super(host, port, database, username, password, dataAdapter); + + final var placeRowMapper = new PlaceRowMapper(dbutils); + placeToNominatimResult = (rs, rowNum) -> { + PhotonDoc doc = placeRowMapper.mapRow(rs, rowNum); + assert (doc != null); + + Map address = dbutils.getMap(rs, "address"); + + doc.completePlace(getAddresses(doc)); + // Add address last, so it takes precedence. + doc.address(address); + + doc.setCountry(countryNames.get(rs.getString("country_code"))); + + return NominatimResult.fromAddress(doc, address); + }; + + // Setup handling of interpolation table. There are two different formats depending on the Nominatim version. + // new-style interpolations + final OsmlineRowMapper osmlineRowMapper = new OsmlineRowMapper(); + osmlineToNominatimResult = (rs, rownum) -> { + PhotonDoc doc = osmlineRowMapper.mapRow(rs, rownum); + + doc.completePlace(getAddresses(doc)); + doc.setCountry(countryNames.get(rs.getString("country_code"))); + + Geometry geometry = dbutils.extractGeometry(rs, "linegeo"); + + if (hasNewStyleInterpolation) { + return NominatimResult.fromInterpolation( + doc, rs.getLong("startnumber"), rs.getLong("endnumber"), + rs.getLong("step"), geometry); + } + + return NominatimResult.fromInterpolation( + doc, rs.getLong("startnumber"), rs.getLong("endnumber"), + rs.getString("interpolationtype"), geometry); + }; + } + + + public boolean isBusy() { return updateLock.isLocked(); } @@ -74,13 +140,19 @@ public void setUpdater(Updater updater) { public void initUpdates(String updateUser) { LOGGER.info("Creating tracking tables"); - template.execute(TRIGGER_SQL); - template.execute("GRANT SELECT, DELETE ON photon_updates TO \"" + updateUser + '"'); + txTemplate.execute(new TransactionCallbackWithoutResult() { + @Override + protected void doInTransactionWithoutResult(TransactionStatus status) { + template.execute(TRIGGER_SQL); + template.execute("GRANT SELECT, DELETE ON photon_updates TO \"" + updateUser + '"'); + } + }); } public void update() { if (updateLock.tryLock()) { try { + loadCountryNames(); updateFromPlacex(); updateFromInterpolations(); updater.finish(); @@ -103,7 +175,7 @@ private void updateFromPlacex() { boolean checkForMultidoc = true; if (!place.isToDelete()) { - final List updatedDocs = exporter.getByPlaceId(placeId); + final List updatedDocs = getByPlaceId(placeId); if (updatedDocs != null && !updatedDocs.isEmpty() && updatedDocs.get(0).isUsefulForIndex()) { checkForMultidoc = updatedDocs.get(0).getRankAddress() == 30; ++updatedPlaces; @@ -143,7 +215,7 @@ private void updateFromInterpolations() { int objectId = -1; if (!place.isToDelete()) { - final List updatedDocs = exporter.getInterpolationsByPlaceId(placeId); + final List updatedDocs = getInterpolationsByPlaceId(placeId); if (updatedDocs != null) { ++updatedInterpolations; for (PhotonDoc updatedDoc : updatedDocs) { @@ -165,48 +237,94 @@ private void updateFromInterpolations() { } private List getPlaces(String table) { - List results = template.query(exporter.getDataAdaptor().deleteReturning( - "DELETE FROM photon_updates WHERE rel = ?", "place_id, operation, indexed_date"), - (rs, rowNum) -> { - boolean isDelete = "DELETE".equals(rs.getString("operation")); - return new UpdateRow(rs.getLong("place_id"), isDelete, rs.getTimestamp("indexed_date")); - }, table); - - // For each place only keep the newest item. - // Order doesn't really matter because updates of each place are independent now. - results.sort(Comparator.comparing(UpdateRow::getPlaceId).thenComparing( - Comparator.comparing(UpdateRow::getUpdateDate).reversed())); - - ArrayList todo = new ArrayList<>(); - long prevId = -1; - for (UpdateRow row: results) { - if (row.getPlaceId() != prevId) { - prevId = row.getPlaceId(); - todo.add(row); + return txTemplate.execute(status -> { + List results = template.query(dbutils.deleteReturning( + "DELETE FROM photon_updates WHERE rel = ?", "place_id, operation, indexed_date"), + (rs, rowNum) -> { + boolean isDelete = "DELETE".equals(rs.getString("operation")); + return new UpdateRow(rs.getLong("place_id"), isDelete, rs.getTimestamp("indexed_date")); + }, table); + + // For each place only keep the newest item. + // Order doesn't really matter because updates of each place are independent now. + results.sort(Comparator.comparing(UpdateRow::getPlaceId).thenComparing( + Comparator.comparing(UpdateRow::getUpdateDate).reversed())); + + ArrayList todo = new ArrayList<>(); + long prevId = -1; + for (UpdateRow row : results) { + if (row.getPlaceId() != prevId) { + prevId = row.getPlaceId(); + todo.add(row); + } } - } - return todo; + return todo; + }); } - /** - * Create a new instance. - * - * @param host Nominatim database host - * @param port Nominatim database port - * @param database Nominatim database name - * @param username Nominatim database username - * @param password Nominatim database password - */ - public NominatimUpdater(String host, int port, String database, String username, String password, DBDataAdapter dataAdapter) { - BasicDataSource dataSource = NominatimConnector.buildDataSource(host, port, database, username, password, true); + public List getByPlaceId(long placeId) { + List result = template.query( + SELECT_COLS_PLACEX + " FROM placex WHERE place_id = ? and indexed_status = 0", + placeToNominatimResult, placeId); - exporter = new NominatimConnector(host, port, database, username, password, dataAdapter); - template = new JdbcTemplate(dataSource); + return result.isEmpty() ? null : result.get(0).getDocsWithHousenumber(); } - public NominatimUpdater(String host, int port, String database, String username, String password) { - this(host, port, database, username, password, new PostgisDataAdapter()); + public List getInterpolationsByPlaceId(long placeId) { + List result = template.query( + (hasNewStyleInterpolation ? SELECT_OSMLINE_NEW_STYLE : SELECT_OSMLINE_OLD_STYLE) + + " FROM location_property_osmline WHERE place_id = ? and indexed_status = 0", + osmlineToNominatimResult, placeId); + + return result.isEmpty() ? null : result.get(0).getDocsWithHousenumber(); + } + + + List getAddresses(PhotonDoc doc) { + RowMapper rowMapper = (rs, rowNum) -> new AddressRow( + dbutils.getMap(rs, "name"), + rs.getString("class"), + rs.getString("type"), + rs.getInt("rank_address") + ); + + AddressType atype = doc.getAddressType(); + + if (atype == null || atype == AddressType.COUNTRY) { + return Collections.emptyList(); + } + + List terms = null; + + if (atype == AddressType.HOUSE) { + long placeId = doc.getParentPlaceId(); + if (placeId != parentPlaceId) { + parentTerms = template.query(SELECT_COLS_ADDRESS + + " FROM placex p, place_addressline pa" + + " WHERE p.place_id = pa.address_place_id and pa.place_id = ?" + + " and pa.cached_rank_address > 4 and pa.address_place_id != ? and pa.isaddress" + + " ORDER BY rank_address desc, fromarea desc, distance asc, rank_search desc", + rowMapper, placeId, placeId); + + // need to add the term for the parent place ID itself + parentTerms.addAll(0, template.query(SELECT_COLS_ADDRESS + " FROM placex p WHERE p.place_id = ?", + rowMapper, placeId)); + parentPlaceId = placeId; + } + terms = parentTerms; + + } else { + long placeId = doc.getPlaceId(); + terms = template.query(SELECT_COLS_ADDRESS + + " FROM placex p, place_addressline pa" + + " WHERE p.place_id = pa.address_place_id and pa.place_id = ?" + + " and pa.cached_rank_address > 4 and pa.address_place_id != ? and pa.isaddress" + + " ORDER BY rank_address desc, fromarea desc, distance asc, rank_search desc", + rowMapper, placeId, placeId); + } + + return terms; } } diff --git a/src/main/java/de/komoot/photon/nominatim/PostgisDataAdapter.java b/src/main/java/de/komoot/photon/nominatim/PostgisDataAdapter.java index bac2f4a9..37e6718f 100644 --- a/src/main/java/de/komoot/photon/nominatim/PostgisDataAdapter.java +++ b/src/main/java/de/komoot/photon/nominatim/PostgisDataAdapter.java @@ -58,4 +58,9 @@ public Boolean mapRow(ResultSet resultSet, int i) throws SQLException { public String deleteReturning(String deleteSQL, String columns) { return deleteSQL + " RETURNING " + columns; } + + @Override + public String jsonArrayFromSelect(String valueSQL, String fromSQL) { + return "(SELECT json_agg(val) FROM (SELECT " + valueSQL + " as val " + fromSQL + ") xxx)"; + } } diff --git a/src/main/java/de/komoot/photon/nominatim/model/AddressRow.java b/src/main/java/de/komoot/photon/nominatim/model/AddressRow.java index 97a02e67..a693b687 100644 --- a/src/main/java/de/komoot/photon/nominatim/model/AddressRow.java +++ b/src/main/java/de/komoot/photon/nominatim/model/AddressRow.java @@ -37,4 +37,14 @@ public boolean isUsefulForContext() { public Map getName() { return this.name; } + + @Override + public String toString() { + return "AddressRow{" + + "name=" + name.getOrDefault("name", "?") + + ", osmKey='" + osmKey + '\'' + + ", osmValue='" + osmValue + '\'' + + ", rankAddress=" + rankAddress + + '}'; + } } diff --git a/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java b/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java new file mode 100644 index 00000000..67399b6c --- /dev/null +++ b/src/main/java/de/komoot/photon/nominatim/model/NominatimAddressCache.java @@ -0,0 +1,67 @@ +package de.komoot.photon.nominatim.model; + +import de.komoot.photon.nominatim.DBDataAdapter; +import org.json.JSONArray; +import org.slf4j.Logger; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowCallbackHandler; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Container for caching information about address parts. + */ +public class NominatimAddressCache { + private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(NominatimAddressCache.class); + + private static final String BASE_COUNTRY_QUERY = + "SELECT place_id, name, class, type, rank_address FROM placex" + + " WHERE rank_address between 5 and 25 AND linked_place_id is null"; + + private final Map addresses = new HashMap<>(); + + public void loadCountryAddresses(JdbcTemplate template, DBDataAdapter dbutils, String countryCode) { + final RowCallbackHandler rowMapper = rs -> + addresses.put( + rs.getLong("place_id"), + new AddressRow( + Map.copyOf(dbutils.getMap(rs, "name")), + rs.getString("class"), + rs.getString("type"), + rs.getInt("rank_address") + )); + + + if ("".equals(countryCode)) { + template.query(BASE_COUNTRY_QUERY + " AND country_code is null", rowMapper); + } else { + template.query(BASE_COUNTRY_QUERY + " AND country_code = ?", rowMapper, countryCode); + } + + if (addresses.size() > 0) { + LOGGER.info("Loaded {} address places for country {}", addresses.size(), countryCode); + } + } + + public List getAddressList(String addressline) { + ArrayList outlist = new ArrayList<>(); + + if (addressline != null && !addressline.isBlank()) { + JSONArray addressPlaces = new JSONArray(addressline); + for (int i = 0; i < addressPlaces.length(); ++i) { + Long placeId = addressPlaces.optLong(i); + if (placeId != null) { + AddressRow row = addresses.get(placeId); + if (row != null) { + outlist.add(row); + } + } + } + } + + return outlist; + } +} diff --git a/src/main/java/de/komoot/photon/nominatim/model/OsmlineRowMapper.java b/src/main/java/de/komoot/photon/nominatim/model/OsmlineRowMapper.java new file mode 100644 index 00000000..5c6a2c6a --- /dev/null +++ b/src/main/java/de/komoot/photon/nominatim/model/OsmlineRowMapper.java @@ -0,0 +1,20 @@ +package de.komoot.photon.nominatim.model; + +import de.komoot.photon.PhotonDoc; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +public class OsmlineRowMapper implements RowMapper { + @Override + public PhotonDoc mapRow(ResultSet rs, int rowNum) throws SQLException { + return new PhotonDoc( + rs.getLong("place_id"), + "W", rs.getLong("osm_id"), + "place", "house_number") + .parentPlaceId(rs.getLong("parent_place_id")) + .countryCode(rs.getString("country_code")) + .postcode(rs.getString("postcode")); + } +} \ No newline at end of file diff --git a/src/main/java/de/komoot/photon/nominatim/model/PlaceRowMapper.java b/src/main/java/de/komoot/photon/nominatim/model/PlaceRowMapper.java new file mode 100644 index 00000000..4944f00a --- /dev/null +++ b/src/main/java/de/komoot/photon/nominatim/model/PlaceRowMapper.java @@ -0,0 +1,44 @@ +package de.komoot.photon.nominatim.model; + +import de.komoot.photon.PhotonDoc; +import de.komoot.photon.nominatim.DBDataAdapter; +import org.springframework.jdbc.core.RowMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; + +/** + * Maps the basic attributes of a placex table row to a PhotonDoc. + * + * This class does not complete address information (neither country information) + * for the place. + */ +public class PlaceRowMapper implements RowMapper { + + private final DBDataAdapter dbutils; + + public PlaceRowMapper(DBDataAdapter dbutils) { + this.dbutils = dbutils; + } + + @Override + public PhotonDoc mapRow(ResultSet rs, int rowNum) throws SQLException { + PhotonDoc doc = new PhotonDoc(rs.getLong("place_id"), + rs.getString("osm_type"), rs.getLong("osm_id"), + rs.getString("class"), rs.getString("type")) + .names(dbutils.getMap(rs, "name")) + .extraTags(dbutils.getMap(rs, "extratags")) + .bbox(dbutils.extractGeometry(rs, "bbox")) + .parentPlaceId(rs.getLong("parent_place_id")) + .countryCode(rs.getString("country_code")) + .centroid(dbutils.extractGeometry(rs, "centroid")) + .linkedPlaceId(rs.getLong("linked_place_id")) + .rankAddress(rs.getInt("rank_address")) + .postcode(rs.getString("postcode")); + + double importance = rs.getDouble("importance"); + doc.importance(rs.wasNull() ? (0.75 - rs.getInt("rank_search") / 40d) : importance); + + return doc; + } +} diff --git a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java b/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java index 7edc325e..f2bd4ddf 100644 --- a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java +++ b/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java @@ -16,15 +16,18 @@ import java.util.Date; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; import org.springframework.jdbc.datasource.embedded.EmbeddedDatabase; import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; +import org.springframework.transaction.support.TransactionTemplate; class NominatimConnectorDBTest { private EmbeddedDatabase db; - private NominatimConnector connector; + private NominatimImporter connector; private CollectingImporter importer; private JdbcTemplate jdbc; + private TransactionTemplate txTemplate; @BeforeEach void setup() { @@ -35,18 +38,31 @@ void setup() { .build(); - connector = new NominatimConnector(null, 0, null, null, null, new H2DataAdapter()); + connector = new NominatimImporter(null, 0, null, null, null, new H2DataAdapter()); importer = new CollectingImporter(); - connector.setImporter(importer); jdbc = new JdbcTemplate(db); - ReflectionTestUtil.setFieldValue(connector, "template", jdbc); + txTemplate = new TransactionTemplate(new DataSourceTransactionManager(db)); + ReflectionTestUtil.setFieldValue(connector, NominatimConnector.class, "template", jdbc); + ReflectionTestUtil.setFieldValue(connector, NominatimConnector.class, "txTemplate", txTemplate); + } + + private void readEntireDatabase() { + ImportThread importThread = new ImportThread(importer); + try { + for (var country: connector.getCountriesFromDatabase()) { + connector.readCountry(country, importThread); + } + } finally { + importThread.finish(); + } + } @Test void testSimpleNodeImport() throws ParseException { PlacexTestRow place = new PlacexTestRow("amenity", "cafe").name("Spot").add(jdbc); - connector.readEntireDatabase(); + readEntireDatabase(); assertEquals(1, importer.size()); importer.assertContains(place); @@ -57,7 +73,15 @@ void testImportForSelectedCountries() throws ParseException { PlacexTestRow place = new PlacexTestRow("amenity", "cafe").name("SpotHU").country("hu").add(jdbc); new PlacexTestRow("amenity", "cafe").name("SpotDE").country("de").add(jdbc); new PlacexTestRow("amenity", "cafe").name("SpotUS").country("us").add(jdbc); - connector.readEntireDatabase("uk", "hu", "nl"); + + ImportThread importThread = new ImportThread(importer); + try { + connector.readCountry("uk", importThread); + connector.readCountry("hu", importThread); + connector.readCountry("nl", importThread); + } finally { + importThread.finish(); + } assertEquals(1, importer.size()); importer.assertContains(place); @@ -68,7 +92,7 @@ void testImportance() { PlacexTestRow place1 = new PlacexTestRow("amenity", "cafe").name("Spot").rankSearch(10).add(jdbc); PlacexTestRow place2 = new PlacexTestRow("amenity", "cafe").name("Spot").importance(0.3).add(jdbc); - connector.readEntireDatabase(); + readEntireDatabase(); assertEquals(0.5, importer.get(place1.getPlaceId()).getImportance(), 0.00001); assertEquals(0.3, importer.get(place2.getPlaceId()).getImportance(), 0.00001); @@ -79,13 +103,13 @@ void testPlaceAddress() throws ParseException { PlacexTestRow place = PlacexTestRow.make_street("Burg").add(jdbc); place.addAddresslines(jdbc, - new PlacexTestRow("place", "neighbourhood").name("Le Coin").rankAddress(24).add(jdbc), - new PlacexTestRow("place", "suburb").name("Crampton").rankAddress(20).add(jdbc), - new PlacexTestRow("place", "city").name("Grand Junction").rankAddress(16).add(jdbc), - new PlacexTestRow("place", "county").name("Lost County").rankAddress(12).add(jdbc), - new PlacexTestRow("place", "state").name("Le Havre").rankAddress(8).add(jdbc)); + new PlacexTestRow("place", "neighbourhood").name("Le Coin").ranks(24).add(jdbc), + new PlacexTestRow("place", "suburb").name("Crampton").ranks(20).add(jdbc), + new PlacexTestRow("place", "city").name("Grand Junction").ranks(16).add(jdbc), + new PlacexTestRow("place", "county").name("Lost County").ranks(12).add(jdbc), + new PlacexTestRow("place", "state").name("Le Havre").ranks(8).add(jdbc)); - connector.readEntireDatabase(); + readEntireDatabase(); assertEquals(6, importer.size()); importer.assertContains(place); @@ -104,10 +128,10 @@ void testPlaceAddressAddressRank0() throws ParseException { PlacexTestRow place = new PlacexTestRow("natural", "water").name("Lake Tee").rankAddress(0).rankSearch(20).add(jdbc); place.addAddresslines(jdbc, - new PlacexTestRow("place", "county").name("Lost County").rankAddress(12).add(jdbc), - new PlacexTestRow("place", "state").name("Le Havre").rankAddress(8).add(jdbc)); + new PlacexTestRow("place", "county").name("Lost County").ranks(12).add(jdbc), + new PlacexTestRow("place", "state").name("Le Havre").ranks(8).add(jdbc)); - connector.readEntireDatabase(); + readEntireDatabase(); assertEquals(3, importer.size()); importer.assertContains(place); @@ -123,11 +147,11 @@ void testPoiAddress() throws ParseException { PlacexTestRow parent = PlacexTestRow.make_street("Burg").add(jdbc); parent.addAddresslines(jdbc, - new PlacexTestRow("place", "city").name("Grand Junction").rankAddress(16).add(jdbc)); + new PlacexTestRow("place", "city").name("Grand Junction").ranks(16).add(jdbc)); PlacexTestRow place = new PlacexTestRow("place", "house").name("House").parent(parent).add(jdbc); - connector.readEntireDatabase(); + readEntireDatabase(); assertEquals(3, importer.size()); importer.assertContains(place); @@ -149,7 +173,7 @@ void testInterpolationPoint() throws ParseException { OsmlineTestRow osmline = new OsmlineTestRow().number(45, 45, 1).parent(street).geom("POINT(45 23)").add(jdbc); - connector.readEntireDatabase(); + readEntireDatabase(); assertEquals(2, importer.size()); @@ -169,7 +193,7 @@ void testInterpolationAny() throws ParseException { OsmlineTestRow osmline = new OsmlineTestRow().number(1, 11, 1).parent(street).geom("LINESTRING(0 0, 0 1)").add(jdbc); - connector.readEntireDatabase(); + readEntireDatabase(); assertEquals(12, importer.size()); @@ -187,7 +211,7 @@ void testInterpolationWithSteps() throws ParseException { OsmlineTestRow osmline = new OsmlineTestRow().number(10, 20, 2).parent(street).geom("LINESTRING(0 0, 0 1)").add(jdbc); - connector.readEntireDatabase(); + readEntireDatabase(); assertEquals(7, importer.size()); @@ -205,13 +229,13 @@ void testInterpolationWithSteps() throws ParseException { @Test void testAddressMappingDuplicate() { PlacexTestRow place = PlacexTestRow.make_street("Main Street").add(jdbc); - PlacexTestRow munip = new PlacexTestRow("place", "municipality").name("Gemeinde").rankAddress(14).add(jdbc); + PlacexTestRow munip = new PlacexTestRow("place", "municipality").name("Gemeinde").ranks(14).add(jdbc); place.addAddresslines(jdbc, munip, - new PlacexTestRow("place", "village").name("Dorf").rankAddress(16).add(jdbc)); + new PlacexTestRow("place", "village").name("Dorf").ranks(16).add(jdbc)); - connector.readEntireDatabase(); + readEntireDatabase(); assertEquals(3, importer.size()); @@ -226,12 +250,12 @@ void testAddressMappingDuplicate() { */ @Test void testAddressMappingAvoidSameTypeAsPlace() { - PlacexTestRow village = new PlacexTestRow("place", "village").name("Dorf").rankAddress(16).add(jdbc); - PlacexTestRow munip = new PlacexTestRow("place", "municipality").name("Gemeinde").rankAddress(14).add(jdbc); + PlacexTestRow village = new PlacexTestRow("place", "village").name("Dorf").ranks(16).add(jdbc); + PlacexTestRow munip = new PlacexTestRow("place", "municipality").name("Gemeinde").ranks(14).add(jdbc); village.addAddresslines(jdbc, munip); - connector.readEntireDatabase(); + readEntireDatabase(); assertEquals(2, importer.size()); @@ -249,7 +273,7 @@ void testUnnamedObjectWithHousenumber() { PlacexTestRow parent = PlacexTestRow.make_street("Main St").add(jdbc); PlacexTestRow place = new PlacexTestRow("building", "yes").addr("housenumber", "123").parent(parent).add(jdbc); - connector.readEntireDatabase(); + readEntireDatabase(); assertEquals(2, importer.size()); @@ -265,7 +289,7 @@ void testObjectWithHousenumberList() throws ParseException { PlacexTestRow parent = PlacexTestRow.make_street("Main St").add(jdbc); PlacexTestRow place = new PlacexTestRow("building", "yes").addr("housenumber", "1;2a;3").parent(parent).add(jdbc); - connector.readEntireDatabase(); + readEntireDatabase(); assertEquals(4, importer.size()); @@ -285,7 +309,7 @@ void testObjectWithconscriptionNumber() throws ParseException { .addr("conscriptionnumber", "99521") .parent(parent).add(jdbc); - connector.readEntireDatabase(); + readEntireDatabase(); assertEquals(3, importer.size()); @@ -300,7 +324,7 @@ void testUnnamedObjectWithOutHousenumber() { PlacexTestRow parent = PlacexTestRow.make_street("Main St").add(jdbc); new PlacexTestRow("building", "yes").parent(parent).add(jdbc); - connector.readEntireDatabase(); + readEntireDatabase(); assertEquals(1, importer.size()); @@ -315,7 +339,7 @@ void testInterpolationLines() { PlacexTestRow parent = PlacexTestRow.make_street("Main St").add(jdbc); new PlacexTestRow("place", "houses").name("something").parent(parent).add(jdbc); - connector.readEntireDatabase(); + readEntireDatabase(); assertEquals(1, importer.size()); @@ -329,7 +353,7 @@ void testInterpolationLines() { void testNoCountry() { PlacexTestRow place = new PlacexTestRow("building", "yes").name("Building").country(null).add(jdbc); - connector.readEntireDatabase(); + readEntireDatabase(); assertEquals(1, importer.size()); diff --git a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorTest.java b/src/test/java/de/komoot/photon/nominatim/NominatimConnectorTest.java deleted file mode 100644 index 5cd8cf08..00000000 --- a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorTest.java +++ /dev/null @@ -1,15 +0,0 @@ -package de.komoot.photon.nominatim; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -class NominatimConnectorTest { - - @Test - void testConvertCountryCode() { - assertEquals("", NominatimConnector.convertCountryCode("".split(","))); - assertEquals("'uk'", NominatimConnector.convertCountryCode("uk".split(","))); - assertEquals("'uk','de'", NominatimConnector.convertCountryCode("uk,de".split(","))); - } -} \ No newline at end of file diff --git a/src/test/java/de/komoot/photon/nominatim/NominatimResultTest.java b/src/test/java/de/komoot/photon/nominatim/NominatimResultTest.java index 3c3eebc4..37de39f7 100644 --- a/src/test/java/de/komoot/photon/nominatim/NominatimResultTest.java +++ b/src/test/java/de/komoot/photon/nominatim/NominatimResultTest.java @@ -47,35 +47,39 @@ private void assertSimpleOnly(List docs) { assertSame(simpleDoc, docs.get(0)); } + private Map housenumberAddress(String housenumber) { + Map address = new HashMap<>(1); + address.put("housenumber", housenumber); + return address; + } + @Test void testIsUsefulForIndex() { assertFalse(simpleDoc.isUsefulForIndex()); - assertFalse(new NominatimResult(simpleDoc).isUsefulForIndex()); + assertFalse(NominatimResult.fromAddress(simpleDoc, null).isUsefulForIndex()); } @Test void testGetDocsWithHousenumber() { - List docs = new NominatimResult(simpleDoc).getDocsWithHousenumber(); + List docs = NominatimResult.fromAddress(simpleDoc, null).getDocsWithHousenumber(); assertSimpleOnly(docs); } @Test void testAddHousenumbersFromStringSimple() { - NominatimResult res = new NominatimResult(simpleDoc); - res.addHousenumbersFromString("34"); + NominatimResult res = NominatimResult.fromAddress(simpleDoc, housenumberAddress("34")); assertDocWithHousenumbers(Arrays.asList("34"), res.getDocsWithHousenumber()); } @Test void testAddHousenumbersFromStringList() { - NominatimResult res = new NominatimResult(simpleDoc); - res.addHousenumbersFromString("34; 50b"); + NominatimResult res = NominatimResult.fromAddress(simpleDoc, housenumberAddress("34; 50b")); assertDocWithHousenumbers(Arrays.asList("34", "50b"), res.getDocsWithHousenumber()); - res.addHousenumbersFromString("4;"); - assertDocWithHousenumbers(Arrays.asList("34", "50b", "4"), res.getDocsWithHousenumber()); + res = NominatimResult.fromAddress(simpleDoc, housenumberAddress("4;")); + assertDocWithHousenumbers(Arrays.asList("4"), res.getDocsWithHousenumber()); } @ParameterizedTest @@ -85,81 +89,69 @@ void testAddHousenumbersFromStringList() { "14, portsmith" }) void testLongHousenumber(String houseNumber) { - NominatimResult res = new NominatimResult(simpleDoc); + NominatimResult res = NominatimResult.fromAddress(simpleDoc, housenumberAddress(houseNumber)); - res.addHousenumbersFromString(houseNumber); assertNoHousenumber(res.getDocsWithHousenumber()); } @Test void testAddHouseNumbersFromInterpolationBad() throws ParseException { - NominatimResult res = new NominatimResult(simpleDoc); - WKTReader reader = new WKTReader(); - res.addHouseNumbersFromInterpolation(34, 33, "odd", + NominatimResult res = NominatimResult.fromInterpolation(simpleDoc, 34, 33, "odd", reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)")); assertSimpleOnly(res.getDocsWithHousenumber()); - res.addHouseNumbersFromInterpolation(1, 10000, "odd", + res = NominatimResult.fromInterpolation(simpleDoc, 1, 10000, "odd", reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)")); assertSimpleOnly(res.getDocsWithHousenumber()); } @Test void testAddHouseNumbersFromInterpolationOdd() throws ParseException { - NominatimResult res = new NominatimResult(simpleDoc); - WKTReader reader = new WKTReader(); - - res.addHouseNumbersFromInterpolation(1, 5, "odd", + NominatimResult res = NominatimResult.fromInterpolation(simpleDoc, 1, 5, "odd", reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)")); assertDocWithHousenumbers(Arrays.asList("3"), res.getDocsWithHousenumber()); - res.addHouseNumbersFromInterpolation(10, 13, "odd", + res = NominatimResult.fromInterpolation(simpleDoc, 10, 13, "odd", reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)")); - assertDocWithHousenumbers(Arrays.asList("3", "11"), res.getDocsWithHousenumber()); + assertDocWithHousenumbers(Arrays.asList("11"), res.getDocsWithHousenumber()); - res.addHouseNumbersFromInterpolation(101, 106, "odd", + res = NominatimResult.fromInterpolation(simpleDoc, 101, 106, "odd", reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)")); - assertDocWithHousenumbers(Arrays.asList("3", "11", "103", "105"), res.getDocsWithHousenumber()); + assertDocWithHousenumbers(Arrays.asList("103", "105"), res.getDocsWithHousenumber()); } @Test void testAddHouseNumbersFromInterpolationEven() throws ParseException { - NominatimResult res = new NominatimResult(simpleDoc); - WKTReader reader = new WKTReader(); - - res.addHouseNumbersFromInterpolation(1, 5, "even", + NominatimResult res = NominatimResult.fromInterpolation(simpleDoc, 1, 5, "even", reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)")); assertDocWithHousenumbers(Arrays.asList("2", "4"), res.getDocsWithHousenumber()); - res.addHouseNumbersFromInterpolation(10, 16, "even", + res= NominatimResult.fromInterpolation(simpleDoc, 10, 16, "even", reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)")); - assertDocWithHousenumbers(Arrays.asList("2", "4", "12", "14"), res.getDocsWithHousenumber()); + assertDocWithHousenumbers(Arrays.asList("12", "14"), res.getDocsWithHousenumber()); - res.addHouseNumbersFromInterpolation(51, 52, "even", + res= NominatimResult.fromInterpolation(simpleDoc, 51, 52, "even", reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)")); - assertDocWithHousenumbers(Arrays.asList("2", "4", "12", "14"), res.getDocsWithHousenumber()); + assertSimpleOnly(res.getDocsWithHousenumber()); } @Test void testAddHouseNumbersFromInterpolationAll() throws ParseException { - NominatimResult res = new NominatimResult(simpleDoc); - WKTReader reader = new WKTReader(); - - res.addHouseNumbersFromInterpolation(1, 3, "", + NominatimResult res = NominatimResult.fromInterpolation(simpleDoc, 1, 3, "", reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)")); assertDocWithHousenumbers(Arrays.asList("2"), res.getDocsWithHousenumber()); - res.addHouseNumbersFromInterpolation(22, 22, null, + res = NominatimResult.fromInterpolation(simpleDoc, 22, 22, null, reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)")); - assertDocWithHousenumbers(Arrays.asList("2"), res.getDocsWithHousenumber()); + assertSimpleOnly(res.getDocsWithHousenumber()); - res.addHouseNumbersFromInterpolation(100, 106, "all", + res = NominatimResult.fromInterpolation(simpleDoc, 100, 106, "all", reader.read("LINESTRING(0.0 0.0 ,0.0 0.1)")); - assertDocWithHousenumbers(Arrays.asList("2", "101", "102", "103", "104", "105"), res.getDocsWithHousenumber()); + assertDocWithHousenumbers(Arrays.asList("101", "102", "103", "104", "105"), res.getDocsWithHousenumber()); } } \ No newline at end of file diff --git a/src/test/java/de/komoot/photon/nominatim/NominatimUpdaterDBTest.java b/src/test/java/de/komoot/photon/nominatim/NominatimUpdaterDBTest.java index cba15bb1..6c30218f 100644 --- a/src/test/java/de/komoot/photon/nominatim/NominatimUpdaterDBTest.java +++ b/src/test/java/de/komoot/photon/nominatim/NominatimUpdaterDBTest.java @@ -5,9 +5,11 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; import org.springframework.jdbc.datasource.embedded.EmbeddedDatabase; import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; +import org.springframework.transaction.support.TransactionTemplate; import static org.junit.jupiter.api.Assertions.*; @@ -16,6 +18,7 @@ class NominatimUpdaterDBTest { private NominatimUpdater connector; private CollectingUpdater updater; private JdbcTemplate jdbc; + private TransactionTemplate txTemplate; @BeforeEach void setup() { @@ -31,8 +34,9 @@ void setup() { connector.setUpdater(updater); jdbc = new JdbcTemplate(db); - ReflectionTestUtil.setFieldValue(connector, "template", jdbc); - ReflectionTestUtil.setFieldValue(connector, "exporter", "template", jdbc); + txTemplate = new TransactionTemplate(new DataSourceTransactionManager(db)); + ReflectionTestUtil.setFieldValue(connector, NominatimConnector.class, "template", jdbc); + ReflectionTestUtil.setFieldValue(connector, NominatimConnector.class, "txTemplate", txTemplate); } @Test diff --git a/src/test/java/de/komoot/photon/nominatim/testdb/H2DataAdapter.java b/src/test/java/de/komoot/photon/nominatim/testdb/H2DataAdapter.java index ec21b4eb..d97c5723 100644 --- a/src/test/java/de/komoot/photon/nominatim/testdb/H2DataAdapter.java +++ b/src/test/java/de/komoot/photon/nominatim/testdb/H2DataAdapter.java @@ -45,4 +45,8 @@ public String deleteReturning(String deleteSQL, String columns) { return "SELECT " + columns + " FROM OLD TABLE (" + deleteSQL + ")"; } + @Override + public String jsonArrayFromSelect(String valueSQL, String fromSQL) { + return "json_array((SELECT " + valueSQL + " " + fromSQL + ") FORMAT JSON)"; + } } diff --git a/src/test/java/de/komoot/photon/nominatim/testdb/PlacexTestRow.java b/src/test/java/de/komoot/photon/nominatim/testdb/PlacexTestRow.java index eb5190e3..85a14553 100644 --- a/src/test/java/de/komoot/photon/nominatim/testdb/PlacexTestRow.java +++ b/src/test/java/de/komoot/photon/nominatim/testdb/PlacexTestRow.java @@ -104,6 +104,12 @@ public PlacexTestRow rankAddress(int rank) { return this; } + public PlacexTestRow ranks(int rank) { + this.rankAddress = rank; + this.rankSearch = rank; + return this; + } + public PlacexTestRow parent(PlacexTestRow row) { this.parentPlaceId = row.getPlaceId(); return this; diff --git a/src/test/resources/test-schema.sql b/src/test/resources/test-schema.sql index 652a9c6a..0da63df8 100644 --- a/src/test/resources/test-schema.sql +++ b/src/test/resources/test-schema.sql @@ -63,6 +63,12 @@ CREATE TABLE country_name ( INSERT INTO country_name VALUES ('de', JSON '{"name" : "Deutschland", "name:en" : "Germany"}', 'de', 2); +INSERT INTO country_name + VALUES ('us', JSON '{"name" : "USA", "name:en" : "United States"}', 'en', 1); +INSERT INTO country_name + VALUES ('hu', JSON '{"name" : "Magyarország", "name:en" : "Hungary"}', 'hu', 12); +INSERT INTO country_name + VALUES ('nl', JSON '{"name" : "Nederland", "name:en" : "Netherlands"}', null, 2); CREATE TABLE photon_updates (