Skip to content
Permalink

Comparing changes

This is a direct comparison between two commits made in this repository or its related repositories. View the default comparison for this range or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: komoot/photon
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: ac46fdf912d5e478b6e3b87ba126877e9686eb2a
Choose a base ref
..
head repository: komoot/photon
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: 4ccb363ebc23741a630ca9b5c81bf0cfb4f98a28
Choose a head ref
Showing with 25 additions and 119 deletions.
  1. +3 −13 src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
  2. +22 −106 src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java
16 changes: 3 additions & 13 deletions src/main/java/de/komoot/photon/nominatim/NominatimConnector.java
View file
Original file line number Diff line number Diff line change
@@ -47,9 +47,8 @@ boolean isUsefulForIndex() {
}

List<PhotonDoc> getDocsWithHousenumber() {
if (housenumbers == null || housenumbers.isEmpty()) {
if (housenumbers == null || housenumbers.isEmpty())
return ImmutableList.of(doc);
}

List<PhotonDoc> results = new ArrayList<PhotonDoc>(housenumbers.size());
for (Map.Entry<String, Point> e : housenumbers.entrySet()) {
@@ -206,7 +205,6 @@ public NominatimResult mapRow(ResultSet rs, int rowNum) throws SQLException {
}
};
private final String selectColsPlaceX = "place_id, osm_type, osm_id, class, type, name, housenumber, postcode, extratags, ST_Envelope(geometry) AS bbox, parent_place_id, linked_place_id, rank_search, importance, country_code, centroid";
private final String selectColsOsmline = "place_id, osm_id, parent_place_id, startnumber, endnumber, interpolationtype, postcode, country_code, linegeo";
private Importer importer;

private Map<String, String> getCountryNames(String countrycode) {
@@ -248,16 +246,8 @@ public void setImporter(Importer importer) {
this.importer = importer;
}

public List<PhotonDoc> getByPlaceId(long placeId) {
NominatimResult result = template.queryForObject("SELECT " + selectColsPlaceX + " FROM placex WHERE place_id = ?", new Object[] { placeId }, placeRowMapper);
completePlace(result.getBaseDoc());
return result.getDocsWithHousenumber();
}

public List<PhotonDoc> getInterpolationsByPlaceId(long placeId) {
NominatimResult result = template.queryForObject("SELECT " + selectColsOsmline + " FROM location_property_osmline WHERE place_id = ?", new Object[] { placeId }, osmlineRowMapper);
completePlace(result.getBaseDoc());
return result.getDocsWithHousenumber();
public PhotonDoc getByPlaceId(long placeId) {
return template.queryForObject("SELECT " + selectColsPlaceX + " FROM placex WHERE place_id = ?", new Object[]{placeId}, placeRowMapper).getBaseDoc();
}

List<AddressRow> getAddresses(PhotonDoc doc) {
128 changes: 22 additions & 106 deletions src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java
View file
Original file line number Diff line number Diff line change
@@ -21,14 +21,8 @@

public class NominatimUpdater {
private static final org.slf4j.Logger LOGGER = org.slf4j.LoggerFactory.getLogger(NominatimUpdater.class);

private static final int CREATE = 1;
private static final int UPDATE = 2;
private static final int DELETE = 100;

private static final int MIN_RANK = 1;
private static final int MAX_RANK = 30;

private final Integer minRank = 1;
private final Integer maxRank = 30;
private final JdbcTemplate template;
private final NominatimConnector exporter;

@@ -39,94 +33,36 @@ public void setUpdater(Updater updater) {
}

public void update() {
int updatedPlaces = 0;
int deletedPlaces = 0;
for (int rank = MIN_RANK; rank <= MAX_RANK; rank++) {
for (Integer rank = this.minRank; rank <= this.maxRank; rank++) {
LOGGER.info(String.format("Starting rank %d", rank));
for (Map<String, Object> sector : getIndexSectors(rank)) {
for (Map<String, Object> sector : getIndexSectors(rank))
for (UpdateRow place : getIndexSectorPlaces(rank, (Integer) sector.get("geometry_sector"))) {
long placeId = place.getPlaceId();
template.update("update placex set indexed_status = 0 where place_id = ?;", placeId);

Integer indexedStatus = place.getIndexdStatus();
if (indexedStatus == DELETE || (indexedStatus == UPDATE && rank == MAX_RANK)) {
updater.delete(placeId);
if (indexedStatus == DELETE) {
deletedPlaces++;
continue;
}
indexedStatus = CREATE; // always create
}
updatedPlaces++;

final List<PhotonDoc> updatedDocs = exporter.getByPlaceId(place.getPlaceId());
boolean wasUseful = false;
for (PhotonDoc updatedDoc : updatedDocs) {
switch (indexedStatus) {
case CREATE:
if (updatedDoc.isUsefulForIndex()) {

template.update("update placex set indexed_status = 0 where place_id = ?", place.getPlaceId());
final PhotonDoc updatedDoc = exporter.getByPlaceId(place.getPlaceId());

switch (place.getIndexdStatus()) {
case 1:
if (updatedDoc.isUsefulForIndex())
updater.create(updatedDoc);
}
break;
case UPDATE:
if (updatedDoc.isUsefulForIndex()) {
updater.updateOrCreate(updatedDoc);
wasUseful = true;
}
case 2:
if (!updatedDoc.isUsefulForIndex())
updater.delete(place.getPlaceId());

updater.updateOrCreate(updatedDoc);
break;
case 100:
updater.delete(place.getPlaceId());
break;
default:
LOGGER.error(String.format("Unknown index status %d", indexedStatus));
LOGGER.error(String.format("Unknown index status %d", place.getIndexdStatus()));
break;
}
}
if (indexedStatus == UPDATE && !wasUseful) {
// only true when rank != 30
// if no documents for the place id exist this will likely cause moaning
updater.delete(placeId);
updatedPlaces--;
}
}
}
}

LOGGER.info(String.format("%d places created or updated, %d deleted", updatedPlaces, deletedPlaces));

// update documents generated from address interpolations
// .isUsefulForIndex() should always return true for documents
// created from interpolations so no need to check them
LOGGER.info("Starting interpolations");
int updatedInterpolations = 0;
int deletedInterpolations = 0;
int interpolationDocuments = 0;
for (Map<String, Object> sector : template.queryForList(
"select geometry_sector,count(*) from location_property_osmline where indexed_status > 0 group by geometry_sector order by geometry_sector;")) {
for (UpdateRow place : getIndexSectorInterpolations((Integer) sector.get("geometry_sector"))) {
long placeId = place.getPlaceId();
template.update("update location_property_osmline set indexed_status = 0 where place_id = ?;", placeId);

Integer indexedStatus = place.getIndexdStatus();
if (indexedStatus != CREATE) {
updater.delete(placeId);
if (indexedStatus == DELETE) {
deletedInterpolations++;
continue;
}
}
updatedInterpolations++;

final List<PhotonDoc> updatedDocs = exporter.getInterpolationsByPlaceId(place.getPlaceId());
for (PhotonDoc updatedDoc : updatedDocs) {
updater.create(updatedDoc);
interpolationDocuments++;
}
}
}
LOGGER.info(String.format("%d interpolations created or updated, %d deleted, %d documents added or updated",
updatedInterpolations, deletedInterpolations, interpolationDocuments));
updater.finish();
template.update("update import_status set indexed=true;"); // indicate that we are finished

LOGGER.info("Finished updating");
}

private List<Map<String, Object>> getIndexSectors(Integer rank) {
@@ -147,27 +83,7 @@ public UpdateRow mapRow(ResultSet rs, int rowNum) throws SQLException {
});
}

private List<UpdateRow> getIndexSectorInterpolations(Integer geometrySector) {
return template.query("select place_id, indexed_status from location_property_osmline where geometry_sector = ? and indexed_status > 0;",
new Object[] { geometrySector }, new RowMapper<UpdateRow>() {
@Override
public UpdateRow mapRow(ResultSet rs, int rowNum) throws SQLException {
UpdateRow updateRow = new UpdateRow();
updateRow.setPlaceId(rs.getLong("place_id"));
updateRow.setIndexdStatus(rs.getInt("indexed_status"));
return updateRow;
}
});
}

/**
* Creates a new instance
*
* @param host Nominatim database host
* @param port Nominatim database port
* @param database Nominatim database name
* @param username Nominatim database username
* @param password Nominatim database password
*/
public NominatimUpdater(String host, int port, String database, String username, String password) {
BasicDataSource dataSource = new BasicDataSource();
@@ -176,8 +92,8 @@ public NominatimUpdater(String host, int port, String database, String username,
dataSource.setUsername(username);
dataSource.setPassword(password);
dataSource.setDriverClassName(JtsWrapper.class.getCanonicalName());
dataSource.setDefaultAutoCommit(true);
dataSource.setDefaultAutoCommit(false);

exporter = new NominatimConnector(host, port, database, username, password);
template = new JdbcTemplate(dataSource);
}