diff --git a/.gitignore b/.gitignore index f977de333e..fd642ad010 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ conf/*.txt conf/*.xml +conf/*.mmdb !conf/nutch-default.xml conf/hadoop-env.sh conf/slaves diff --git a/build.xml b/build.xml index 45395f42a9..42ace3341e 100644 --- a/build.xml +++ b/build.xml @@ -1186,6 +1186,7 @@ + diff --git a/conf/nutch-default.xml b/conf/nutch-default.xml index b41f7ecdab..6e4fd58235 100644 --- a/conf/nutch-default.xml +++ b/conf/nutch-default.xml @@ -2231,33 +2231,98 @@ CAUTION: Set the parser.timeout to -1 or a bigger value than 30, when using this + + + + index.geoip.db.anonymous + + + GeoIP2 Anonymous IP database file (MMDB format). + Identifies anonymous proxies, VPNs, Tor exit nodes, and hosting providers. + + + + + index.geoip.db.asn + + + GeoIP2/GeoLite2 ASN database file (MMDB format). + Provides autonomous system number and organization information. + + + + + index.geoip.db.city + + + GeoIP2/GeoLite2 City database file (MMDB format). + Provides city, subdivision, country, continent, and location data. + + + - index.geoip.usage - insightsService + index.geoip.db.connection + + + GeoIP2 Connection Type database file (MMDB format). + Identifies connection type: Cable/DSL, Cellular, Corporate, or Satellite. + + + + + index.geoip.db.country + + + GeoIP2/GeoLite2 Country database file (MMDB format). + Provides country, continent, and represented country information. + This is a lighter-weight alternative to the City database when only + country-level information is needed. + + + + + index.geoip.db.domain + + + GeoIP2 Domain database file (MMDB format). + Provides the second-level domain associated with the IP address. + + + + + index.geoip.db.isp + - A string representing the information source to be used for GeoIP information - association. Either enter 'cityDatabase', 'connectionTypeDatabase', - 'domainDatabase', 'ispDatabase' or 'insightsService'. If you wish to use any one of the - Database options, you should make one of GeoIP2-City.mmdb, GeoIP2-Connection-Type.mmdb, - GeoIP2-Domain.mmdb or GeoIP2-ISP.mmdb files respectively available on the classpath and - available at runtime. Alternatively, also the GeoLite2 IP databases (GeoLite2-*.mmdb) - can be used. + GeoIP2 ISP database file (MMDB format). + Provides ISP, organization, and autonomous system information. + - index.geoip.userid + index.geoip.insights.userid - The userId associated with the GeoIP2 Precision Services account. + The userId for the MaxMind GeoIP2 Precision Insights web service. + If set along with licensekey, the Insights service will be queried in addition + to any configured local databases. - index.geoip.licensekey + index.geoip.insights.licensekey - The license key associated with the GeoIP2 Precision Services account. + The license key for the MaxMind GeoIP2 Precision Insights web service. diff --git a/src/plugin/build.xml b/src/plugin/build.xml index 498259a950..b7a5a77216 100755 --- a/src/plugin/build.xml +++ b/src/plugin/build.xml @@ -120,7 +120,7 @@ - + diff --git a/src/plugin/index-geoip/build.xml b/src/plugin/index-geoip/build.xml index 92fda8200a..6a7a5eff64 100644 --- a/src/plugin/index-geoip/build.xml +++ b/src/plugin/index-geoip/build.xml @@ -23,5 +23,9 @@ + + + + diff --git a/src/plugin/index-geoip/ivy.xml b/src/plugin/index-geoip/ivy.xml index a6ddc2949a..ce2defbbae 100644 --- a/src/plugin/index-geoip/ivy.xml +++ b/src/plugin/index-geoip/ivy.xml @@ -37,7 +37,7 @@ - + diff --git a/src/plugin/index-geoip/plugin.xml b/src/plugin/index-geoip/plugin.xml index dda1b6a7be..69e196d447 100644 --- a/src/plugin/index-geoip/plugin.xml +++ b/src/plugin/index-geoip/plugin.xml @@ -26,8 +26,9 @@ - - + + + diff --git a/src/plugin/index-geoip/sample/GeoIP2-City-Test.mmdb b/src/plugin/index-geoip/sample/GeoIP2-City-Test.mmdb new file mode 100644 index 0000000000..a8fcd1fd8d Binary files /dev/null and b/src/plugin/index-geoip/sample/GeoIP2-City-Test.mmdb differ diff --git a/src/plugin/index-geoip/sample/GeoIP2-Connection-Type-Test.mmdb b/src/plugin/index-geoip/sample/GeoIP2-Connection-Type-Test.mmdb new file mode 100644 index 0000000000..2dbd6d23af Binary files /dev/null and b/src/plugin/index-geoip/sample/GeoIP2-Connection-Type-Test.mmdb differ diff --git a/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPDocumentCreator.java b/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPDocumentCreator.java index 64b3862be8..e7da55f95a 100644 --- a/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPDocumentCreator.java +++ b/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPDocumentCreator.java @@ -19,21 +19,23 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.net.InetAddress; -import java.net.UnknownHostException; +import java.util.Optional; import org.apache.nutch.indexer.NutchDocument; +import org.apache.nutch.indexer.NutchField; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.maxmind.geoip2.DatabaseReader; import com.maxmind.geoip2.WebServiceClient; -import com.maxmind.geoip2.exception.AddressNotFoundException; import com.maxmind.geoip2.exception.GeoIp2Exception; -import com.maxmind.geoip2.model.InsightsResponse; +import com.maxmind.geoip2.model.AnonymousIpResponse; +import com.maxmind.geoip2.model.AsnResponse; import com.maxmind.geoip2.model.CityResponse; import com.maxmind.geoip2.model.ConnectionTypeResponse; import com.maxmind.geoip2.model.CountryResponse; import com.maxmind.geoip2.model.DomainResponse; +import com.maxmind.geoip2.model.InsightsResponse; import com.maxmind.geoip2.model.IspResponse; import com.maxmind.geoip2.record.City; import com.maxmind.geoip2.record.Continent; @@ -46,44 +48,137 @@ /** *

- * Simple utility class which enables efficient, structured - * {@link org.apache.nutch.indexer.NutchDocument} building based on input from + * Simple utility class which builds a + * {@link org.apache.nutch.indexer.NutchDocument} based on input from * {@link GeoIPIndexingFilter}, where configuration is also read. *

- *

- * Based on the nature of the input, this class wraps factory type - * implementations for populating {@link org.apache.nutch.indexer.NutchDocument} - * 's with the correct {@link org.apache.nutch.indexer.NutchField} information. - * */ public class GeoIPDocumentCreator { private static final Logger LOG = LoggerFactory .getLogger(MethodHandles.lookup().lookupClass()); + // Database-specific network address field names + static final String ANONYMOUS_NETWORK_ADDRESS = "anonymousNetworkAddress"; + static final String ASN_NETWORK_ADDRESS = "asnNetworkAddress"; + static final String CITY_NETWORK_ADDRESS = "cityNetworkAddress"; + static final String CONNECTION_NETWORK_ADDRESS = "connectionNetworkAddress"; + static final String COUNTRY_NETWORK_ADDRESS = "countryNetworkAddress"; + static final String DOMAIN_NETWORK_ADDRESS = "domainNetworkAddress"; + static final String INSIGHTS_NETWORK_ADDRESS = "insightsNetworkAddress"; + + private GeoIPDocumentCreator() {} + /** * Add field to document but only if value isn't null * @param doc the {@link NutchDocument} to augment * @param name the name of the target field * @param value the String value to associate with the target field */ - public static void addIfNotNull(NutchDocument doc, String name, + private static void addIfNotNull(NutchDocument doc, String name, Object value) { if (value != null) { doc.add(name, value); } } - public static NutchDocument createDocFromInsightsService(String serverIp, - NutchDocument doc, WebServiceClient client) throws UnknownHostException, - IOException, GeoIp2Exception { - addIfNotNull(doc, "ip", serverIp); - InsightsResponse response = client - .insights(InetAddress.getByName(serverIp)); + /** + * Add field to document only if the field doesn't already contain this exact value. + * Use for fields like "ip" that should not have duplicate values. + * @param doc the {@link NutchDocument} to augment + * @param name the name of the target field + * @param value the value to associate with the target field + */ + static void addIfNotDuplicate(NutchDocument doc, String name, Object value) { + if (value == null) { + return; + } + NutchField field = doc.getField(name); + if (field == null || !field.getValues().contains(value)) { + doc.add(name, value); + } + } + + /** + * Populate a {@link org.apache.nutch.indexer.NutchDocument} based on lookup + * of IP in Anonymous IP database. + * @param serverIp the server IP address to lookup + * @param doc NutchDocument to populate + * @param reader instantiated DatabaseReader object + * @return populated NutchDocument + * @throws IOException if an error occurs performing the Db lookup + * @throws GeoIp2Exception generic GeoIp2 exception + */ + public static NutchDocument createDocFromAnonymousIpDb(String serverIp, + NutchDocument doc, DatabaseReader reader) throws IOException, GeoIp2Exception { + Optional opt = reader.tryAnonymousIp(InetAddress.getByName(serverIp)); + if (opt.isPresent()) { + AnonymousIpResponse response = opt.get(); + addIfNotDuplicate(doc, "ip", response.getIpAddress()); + addIfNotNull(doc, ANONYMOUS_NETWORK_ADDRESS, response.getNetwork().toString()); + addIfNotNull(doc, "isAnonymous", response.isAnonymous()); + addIfNotNull(doc, "isAnonymousVpn", response.isAnonymousVpn()); + addIfNotNull(doc, "isHostingProxy", response.isHostingProvider()); + addIfNotNull(doc, "isPublicProxy", response.isPublicProxy()); + addIfNotNull(doc, "isResidentialProxy", response.isResidentialProxy()); + addIfNotNull(doc, "isTorExitNode", response.isTorExitNode()); + } else { + LOG.debug("'{}' IP address not found in Anonymous IP DB.", serverIp); + } + return doc; + } + /** + * Populate a {@link org.apache.nutch.indexer.NutchDocument} based on lookup + * of IP in ASN database. + * @param serverIp the server IP address to lookup + * @param doc NutchDocument to populate + * @param reader instantiated DatabaseReader object + * @return populated NutchDocument + * @throws IOException if an error occurs performing the Db lookup + * @throws GeoIp2Exception generic GeoIp2 exception + */ + public static NutchDocument createDocFromAsnDb(String serverIp, + NutchDocument doc, DatabaseReader reader) throws IOException, GeoIp2Exception { + Optional opt = reader.tryAsn(InetAddress.getByName(serverIp)); + if (opt.isPresent()) { + AsnResponse response = opt.get(); + addIfNotDuplicate(doc, "ip", response.getIpAddress()); + addIfNotNull(doc, ASN_NETWORK_ADDRESS, response.getNetwork().toString()); + addIfNotNull(doc, "autonomousSystemNumber", response.getAutonomousSystemNumber()); + addIfNotNull(doc, "autonomousSystemOrganization", response.getAutonomousSystemOrganization()); + } else { + LOG.debug("'{}' IP address not found in ASN DB.", serverIp); + } + return doc; + } + + /** + * Populate a {@link org.apache.nutch.indexer.NutchDocument} based on lookup + * of IP in City database. + * @param serverIp the server IP address to lookup + * @param doc NutchDocument to populate + * @param reader instantiated DatabaseReader object + * @return populated NutchDocument + * @throws IOException if an error occurs performing the Db lookup + * @throws GeoIp2Exception generic GeoIp2 exception + */ + public static NutchDocument createDocFromCityDb(String serverIp, + NutchDocument doc, DatabaseReader reader) throws IOException, GeoIp2Exception { + addIfNotDuplicate(doc, "ip", serverIp); + Optional opt = reader.tryCity(InetAddress.getByName(serverIp)); + if (opt.isPresent()) { + processCityDocument(doc, opt.get()); + } else { + LOG.debug("'{}' IP address not found in City DB.", serverIp); + } + return doc; + } + + private static NutchDocument processCityDocument(NutchDocument doc, CityResponse response) { City city = response.getCity(); - addIfNotNull(doc, "cityName", city.getName()); // 'Minneapolis' - addIfNotNull(doc, "cityConfidence", city.getConfidence()); // 50 + addIfNotNull(doc, "cityName", city.getName()); + addIfNotNull(doc, "cityConfidence", city.getConfidence()); addIfNotNull(doc, "cityGeoNameId", city.getGeoNameId()); Continent continent = response.getContinent(); @@ -91,147 +186,284 @@ public static NutchDocument createDocFromInsightsService(String serverIp, addIfNotNull(doc, "continentGeoNameId", continent.getGeoNameId()); addIfNotNull(doc, "continentName", continent.getName()); - Country country = response.getCountry(); - addIfNotNull(doc, "countryIsoCode", country.getIsoCode()); // 'US' - addIfNotNull(doc, "countryName", country.getName()); // 'United States' - addIfNotNull(doc, "countryConfidence", country.getConfidence()); // 99 + Country country = response.getRegisteredCountry(); + addIfNotNull(doc, "countryIsoCode", country.getIsoCode()); + addIfNotNull(doc, "countryName", country.getName()); + addIfNotNull(doc, "countryConfidence", country.getConfidence()); addIfNotNull(doc, "countryGeoNameId", country.getGeoNameId()); + addIfNotNull(doc, "countryInEuropeanUnion", country.isInEuropeanUnion()); Location location = response.getLocation(); - addIfNotNull(doc, "latLon", location.getLatitude() + "," + location.getLongitude()); // 44.9733, - // -93.2323 - addIfNotNull(doc, "accRadius", location.getAccuracyRadius()); // 3 - addIfNotNull(doc, "timeZone", location.getTimeZone()); // 'America/Chicago' - addIfNotNull(doc, "metroCode", location.getMetroCode()); + if (location.getLatitude() != null && location.getLongitude() != null) { + addIfNotNull(doc, "latLon", location.getLatitude() + "," + location.getLongitude()); + } + addIfNotNull(doc, "accuracyRadius", location.getAccuracyRadius()); + addIfNotNull(doc, "timeZone", location.getTimeZone()); + addIfNotNull(doc, "populationDensity", location.getPopulationDensity()); Postal postal = response.getPostal(); - addIfNotNull(doc, "postalCode", postal.getCode()); // '55455' - addIfNotNull(doc, "postalConfidence", postal.getConfidence()); // 40 + addIfNotNull(doc, "postalCode", postal.getCode()); + addIfNotNull(doc, "postalConfidence", postal.getConfidence()); RepresentedCountry rCountry = response.getRepresentedCountry(); addIfNotNull(doc, "countryType", rCountry.getType()); - Subdivision subdivision = response.getMostSpecificSubdivision(); - addIfNotNull(doc, "subDivName", subdivision.getName()); // 'Minnesota' - addIfNotNull(doc, "subDivIsoCode", subdivision.getIsoCode()); // 'MN' - addIfNotNull(doc, "subDivConfidence", subdivision.getConfidence()); // 90 - addIfNotNull(doc, "subDivGeoNameId", subdivision.getGeoNameId()); + Subdivision mostSubdivision = response.getMostSpecificSubdivision(); + addIfNotNull(doc, "mostSpecificSubDivName", mostSubdivision.getName()); + addIfNotNull(doc, "mostSpecificSubDivIsoCode", mostSubdivision.getIsoCode()); + addIfNotNull(doc, "mostSpecificSubDivConfidence", mostSubdivision.getConfidence()); + addIfNotNull(doc, "mostSpecificSubDivGeoNameId", mostSubdivision.getGeoNameId()); + + Subdivision leastSubdivision = response.getLeastSpecificSubdivision(); + addIfNotNull(doc, "leastSpecificSubDivName", leastSubdivision.getName()); + addIfNotNull(doc, "leastSpecificSubDivIsoCode", leastSubdivision.getIsoCode()); + addIfNotNull(doc, "leastSpecificSubDivConfidence", leastSubdivision.getConfidence()); + addIfNotNull(doc, "leastSpecificSubDivGeoNameId", leastSubdivision.getGeoNameId()); Traits traits = response.getTraits(); - addIfNotNull(doc, "autonSystemNum", traits.getAutonomousSystemNumber()); - addIfNotNull(doc, "autonSystemOrg", traits.getAutonomousSystemOrganization()); + addIfNotNull(doc, "autonomousSystemNumber", traits.getAutonomousSystemNumber()); + addIfNotNull(doc, "autonomousSystemOrganization", traits.getAutonomousSystemOrganization()); + if (traits.getConnectionType() != null) { + addIfNotNull(doc, "connectionType", traits.getConnectionType().toString()); + } addIfNotNull(doc, "domain", traits.getDomain()); addIfNotNull(doc, "isp", traits.getIsp()); - addIfNotNull(doc, "org", traits.getOrganization()); + addIfNotNull(doc, "mobileCountryCode", traits.getMobileCountryCode()); + addIfNotNull(doc, "mobileNetworkCode", traits.getMobileNetworkCode()); + if (traits.getNetwork() != null) { + addIfNotNull(doc, CITY_NETWORK_ADDRESS, traits.getNetwork().toString()); + } + addIfNotNull(doc, "organization", traits.getOrganization()); + addIfNotNull(doc, "staticIpScore", traits.getStaticIpScore()); + addIfNotNull(doc, "userCount", traits.getUserCount()); addIfNotNull(doc, "userType", traits.getUserType()); - //for better results, users should upgrade to - //https://www.maxmind.com/en/solutions/geoip2-enterprise-product-suite/anonymous-ip-database - addIfNotNull(doc, "isAnonProxy", String.valueOf(traits.isAnonymousProxy())); + addIfNotNull(doc, "isAnonymous", traits.isAnonymous()); + addIfNotNull(doc, "isAnonymousVpn", traits.isAnonymousVpn()); + addIfNotNull(doc, "isAnycast", traits.isAnycast()); + addIfNotNull(doc, "isHostingProvider", traits.isHostingProvider()); + addIfNotNull(doc, "isLegitimateProxy", traits.isLegitimateProxy()); + addIfNotNull(doc, "isPublicProxy", traits.isPublicProxy()); + addIfNotNull(doc, "isResidentialProxy", traits.isResidentialProxy()); + addIfNotNull(doc, "isTorExitNode", traits.isTorExitNode()); return doc; } - @SuppressWarnings("unused") - public static NutchDocument createDocFromCityService(String serverIp, - NutchDocument doc, WebServiceClient client) throws UnknownHostException, - IOException, GeoIp2Exception { - CityResponse response = client.city(InetAddress.getByName(serverIp)); + /** + * Populate a {@link org.apache.nutch.indexer.NutchDocument} based on lookup + * of IP in ConnectionDb. + * @param serverIp the server IP + * @param doc NutchDocument to populate + * @param reader instantiated DatabaseReader object + * @return populated NutchDocument + * @throws IOException if an error occurs performing the Db lookup + * @throws GeoIp2Exception generic GeoIp2 exception + */ + public static NutchDocument createDocFromConnectionDb(String serverIp, + NutchDocument doc, DatabaseReader reader) throws IOException, GeoIp2Exception { + Optional opt = reader.tryConnectionType(InetAddress + .getByName(serverIp)); + if (opt.isPresent()) { + ConnectionTypeResponse response = opt.get(); + addIfNotDuplicate(doc, "ip", response.getIpAddress()); + if (response.getConnectionType() != null) { + addIfNotNull(doc, "connectionType", response.getConnectionType().toString()); + } + if (response.getNetwork() != null) { + addIfNotNull(doc, CONNECTION_NETWORK_ADDRESS, response.getNetwork().toString()); + } + } else { + LOG.debug("'{}' IP address not found in Connection DB.", serverIp); + } return doc; } - @SuppressWarnings("unused") - public static NutchDocument createDocFromCountryService(String serverIp, - NutchDocument doc, WebServiceClient client) throws UnknownHostException, - IOException, GeoIp2Exception { - CountryResponse response = client.country(InetAddress.getByName(serverIp)); - return doc; - } + /** + * Populate a {@link org.apache.nutch.indexer.NutchDocument} based on lookup + * of IP in Country database. This is a lighter-weight alternative to the City + * database when only country-level information is needed. + * @param serverIp the server IP address to lookup + * @param doc NutchDocument to populate + * @param reader instantiated DatabaseReader object + * @return populated NutchDocument + * @throws IOException if an error occurs performing the Db lookup + * @throws GeoIp2Exception generic GeoIp2 exception + */ + public static NutchDocument createDocFromCountryDb(String serverIp, + NutchDocument doc, DatabaseReader reader) throws IOException, GeoIp2Exception { + Optional opt = reader.tryCountry(InetAddress.getByName(serverIp)); + if (opt.isPresent()) { + CountryResponse response = opt.get(); + addIfNotDuplicate(doc, "ip", serverIp); - public static NutchDocument createDocFromIspDb(String serverIp, - NutchDocument doc, DatabaseReader reader) throws UnknownHostException, - IOException, GeoIp2Exception { - IspResponse response = reader.isp(InetAddress.getByName(serverIp)); - addIfNotNull(doc, "ip", serverIp); - addIfNotNull(doc, "autonSystemNum", response.getAutonomousSystemNumber()); - addIfNotNull(doc, "autonSystemOrg", response.getAutonomousSystemOrganization()); - addIfNotNull(doc, "isp", response.getIsp()); - addIfNotNull(doc, "org", response.getOrganization()); + Continent continent = response.getContinent(); + addIfNotDuplicate(doc, "continentCode", continent.getCode()); + addIfNotDuplicate(doc, "continentGeoNameId", continent.getGeoNameId()); + addIfNotDuplicate(doc, "continentName", continent.getName()); + + Country country = response.getRegisteredCountry(); + addIfNotDuplicate(doc, "countryIsoCode", country.getIsoCode()); + addIfNotDuplicate(doc, "countryName", country.getName()); + addIfNotDuplicate(doc, "countryConfidence", country.getConfidence()); + addIfNotDuplicate(doc, "countryGeoNameId", country.getGeoNameId()); + addIfNotDuplicate(doc, "countryInEuropeanUnion", country.isInEuropeanUnion()); + + RepresentedCountry rCountry = response.getRepresentedCountry(); + addIfNotDuplicate(doc, "countryType", rCountry.getType()); + + Traits traits = response.getTraits(); + if (traits.getNetwork() != null) { + addIfNotNull(doc, COUNTRY_NETWORK_ADDRESS, traits.getNetwork().toString()); + } + addIfNotDuplicate(doc, "isAnonymous", traits.isAnonymous()); + addIfNotDuplicate(doc, "isAnonymousVpn", traits.isAnonymousVpn()); + addIfNotDuplicate(doc, "isAnycast", traits.isAnycast()); + addIfNotDuplicate(doc, "isHostingProvider", traits.isHostingProvider()); + addIfNotDuplicate(doc, "isLegitimateProxy", traits.isLegitimateProxy()); + addIfNotDuplicate(doc, "isPublicProxy", traits.isPublicProxy()); + addIfNotDuplicate(doc, "isResidentialProxy", traits.isResidentialProxy()); + addIfNotDuplicate(doc, "isTorExitNode", traits.isTorExitNode()); + } else { + LOG.debug("'{}' IP address not found in Country DB.", serverIp); + } return doc; } + /** + * Populate a {@link org.apache.nutch.indexer.NutchDocument} based on lookup + * of IP in Domain database. + * @param serverIp the server IP address to lookup + * @param doc NutchDocument to populate + * @param reader instantiated DatabaseReader object + * @return populated NutchDocument + * @throws IOException if an error occurs performing the Db lookup + * @throws GeoIp2Exception generic GeoIp2 exception + */ public static NutchDocument createDocFromDomainDb(String serverIp, - NutchDocument doc, DatabaseReader reader) throws UnknownHostException, - IOException, GeoIp2Exception { - DomainResponse response; - try { - response = reader.domain(InetAddress.getByName(serverIp)); - } catch (AddressNotFoundException e) { - LOG.debug("IP address not found: {}", serverIp); - return doc; - } - addIfNotNull(doc, "ip", serverIp); - addIfNotNull(doc, "domain", response.getDomain()); + NutchDocument doc, DatabaseReader reader) throws IOException, GeoIp2Exception { + Optional opt = reader.tryDomain(InetAddress.getByName(serverIp)); + if (opt.isPresent()) { + DomainResponse response = opt.get(); + addIfNotDuplicate(doc, "ip", response.getIpAddress()); + addIfNotNull(doc, "domain", response.getDomain()); + addIfNotNull(doc, DOMAIN_NETWORK_ADDRESS, response.getNetwork().toString()); + } else { + LOG.debug("'{}' IP address not found in Domain DB.", serverIp); + } return doc; } - public static NutchDocument createDocFromConnectionDb(String serverIp, - NutchDocument doc, DatabaseReader reader) throws UnknownHostException, - IOException, GeoIp2Exception { - ConnectionTypeResponse response = reader.connectionType(InetAddress - .getByName(serverIp)); - addIfNotNull(doc, "ip", serverIp); - addIfNotNull(doc, "connType", response.getConnectionType().toString()); - return doc; + /** + * Populate a {@link org.apache.nutch.indexer.NutchDocument} based on lookup + * of IP using the MaxMind Insights web service. + * @param serverIp the server IP address to lookup + * @param doc NutchDocument to populate + * @param client WebServiceClient for MaxMind Insights API + * @return populated NutchDocument + * @throws IOException if an error occurs performing the lookup + * @throws GeoIp2Exception generic GeoIp2 exception + */ + public static NutchDocument createDocFromInsightsService(String serverIp, + NutchDocument doc, WebServiceClient client) throws IOException, GeoIp2Exception { + addIfNotDuplicate(doc, "ip", serverIp); + return processInsightsDocument(doc, client.insights(InetAddress.getByName(serverIp))); } - public static NutchDocument createDocFromCityDb(String serverIp, - NutchDocument doc, DatabaseReader reader) throws UnknownHostException, - IOException, GeoIp2Exception { - addIfNotNull(doc, "ip", serverIp); - - CityResponse response; - try { - response = reader.city(InetAddress.getByName(serverIp)); - } catch (AddressNotFoundException e) { - LOG.debug("IP address not found: {}", serverIp); - return doc; - } - + private static NutchDocument processInsightsDocument(NutchDocument doc, InsightsResponse response) { City city = response.getCity(); - addIfNotNull(doc, "cityName", city.getName()); // 'Minneapolis' - addIfNotNull(doc, "cityConfidence", city.getConfidence()); // 50 + addIfNotNull(doc, "cityName", city.getName()); + addIfNotNull(doc, "cityConfidence", city.getConfidence()); addIfNotNull(doc, "cityGeoNameId", city.getGeoNameId()); - Continent continent = response.getContinent(); addIfNotNull(doc, "continentCode", continent.getCode()); addIfNotNull(doc, "continentGeoNameId", continent.getGeoNameId()); addIfNotNull(doc, "continentName", continent.getName()); - Country country = response.getCountry(); - addIfNotNull(doc, "countryIsoCode", country.getIsoCode()); // 'US' - addIfNotNull(doc, "countryName", country.getName()); // 'United States' - addIfNotNull(doc, "countryConfidence", country.getConfidence()); // 99 + Country country = response.getRegisteredCountry(); + addIfNotNull(doc, "countryIsoCode", country.getIsoCode()); + addIfNotNull(doc, "countryName", country.getName()); + addIfNotNull(doc, "countryConfidence", country.getConfidence()); addIfNotNull(doc, "countryGeoNameId", country.getGeoNameId()); + addIfNotNull(doc, "countryInEuropeanUnion", country.isInEuropeanUnion()); Location location = response.getLocation(); - addIfNotNull(doc, "latLon", location.getLatitude() + "," + location.getLongitude()); // 44.9733, - // -93.2323 - addIfNotNull(doc, "accRadius", location.getAccuracyRadius()); // 3 - addIfNotNull(doc, "timeZone", location.getTimeZone()); // 'America/Chicago' - addIfNotNull(doc, "metroCode", location.getMetroCode()); + if (location.getLatitude() != null && location.getLongitude() != null) { + addIfNotNull(doc, "latLon", location.getLatitude() + "," + location.getLongitude()); + } + addIfNotNull(doc, "accuracyRadius", location.getAccuracyRadius()); + addIfNotNull(doc, "timeZone", location.getTimeZone()); + addIfNotNull(doc, "populationDensity", location.getPopulationDensity()); Postal postal = response.getPostal(); - addIfNotNull(doc, "postalCode", postal.getCode()); // '55455' - addIfNotNull(doc, "postalConfidence", postal.getConfidence()); // 40 + addIfNotNull(doc, "postalCode", postal.getCode()); + addIfNotNull(doc, "postalConfidence", postal.getConfidence()); RepresentedCountry rCountry = response.getRepresentedCountry(); addIfNotNull(doc, "countryType", rCountry.getType()); - Subdivision subdivision = response.getMostSpecificSubdivision(); - addIfNotNull(doc, "subDivName", subdivision.getName()); // 'Minnesota' - addIfNotNull(doc, "subDivIsoCode", subdivision.getIsoCode()); // 'MN' - addIfNotNull(doc, "subDivConfidence", subdivision.getConfidence()); // 90 - addIfNotNull(doc, "subDivGeoNameId", subdivision.getGeoNameId()); + Subdivision mostSubdivision = response.getMostSpecificSubdivision(); + addIfNotNull(doc, "mostSpecificSubDivName", mostSubdivision.getName()); + addIfNotNull(doc, "mostSpecificSubDivIsoCode", mostSubdivision.getIsoCode()); + addIfNotNull(doc, "mostSpecificSubDivConfidence", mostSubdivision.getConfidence()); + addIfNotNull(doc, "mostSpecificSubDivGeoNameId", mostSubdivision.getGeoNameId()); + + Subdivision leastSubdivision = response.getLeastSpecificSubdivision(); + addIfNotNull(doc, "leastSpecificSubDivName", leastSubdivision.getName()); + addIfNotNull(doc, "leastSpecificSubDivIsoCode", leastSubdivision.getIsoCode()); + addIfNotNull(doc, "leastSpecificSubDivConfidence", leastSubdivision.getConfidence()); + addIfNotNull(doc, "leastSpecificSubDivGeoNameId", leastSubdivision.getGeoNameId()); + + Traits traits = response.getTraits(); + addIfNotNull(doc, "autonomousSystemNumber", traits.getAutonomousSystemNumber()); + addIfNotNull(doc, "autonomousSystemOrganization", traits.getAutonomousSystemOrganization()); + if (traits.getConnectionType() != null) { + addIfNotNull(doc, "connectionType", traits.getConnectionType().toString()); + } + addIfNotNull(doc, "domain", traits.getDomain()); + addIfNotNull(doc, "isp", traits.getIsp()); + addIfNotNull(doc, "mobileCountryCode", traits.getMobileCountryCode()); + addIfNotNull(doc, "mobileNetworkCode", traits.getMobileNetworkCode()); + if (traits.getNetwork() != null) { + addIfNotNull(doc, INSIGHTS_NETWORK_ADDRESS, traits.getNetwork().toString()); + } + addIfNotNull(doc, "organization", traits.getOrganization()); + addIfNotNull(doc, "staticIpScore", traits.getStaticIpScore()); + addIfNotNull(doc, "userCount", traits.getUserCount()); + addIfNotNull(doc, "userType", traits.getUserType()); + addIfNotNull(doc, "isAnonymous", traits.isAnonymous()); + addIfNotNull(doc, "isAnonymousVpn", traits.isAnonymousVpn()); + addIfNotNull(doc, "isAnycast", traits.isAnycast()); + addIfNotNull(doc, "isHostingProvider", traits.isHostingProvider()); + addIfNotNull(doc, "isLegitimateProxy", traits.isLegitimateProxy()); + addIfNotNull(doc, "isPublicProxy", traits.isPublicProxy()); + addIfNotNull(doc, "isResidentialProxy", traits.isResidentialProxy()); + addIfNotNull(doc, "isTorExitNode", traits.isTorExitNode()); + return doc; + } + + /** + * Populate a {@link org.apache.nutch.indexer.NutchDocument} based on lookup + * of IP in ISP database. + * @param serverIp the server IP address to lookup + * @param doc NutchDocument to populate + * @param reader instantiated DatabaseReader object + * @return populated NutchDocument + * @throws IOException if an error occurs performing the Db lookup + * @throws GeoIp2Exception generic GeoIp2 exception + */ + public static NutchDocument createDocFromIspDb(String serverIp, + NutchDocument doc, DatabaseReader reader) throws IOException, GeoIp2Exception { + Optional opt = reader.tryIsp(InetAddress.getByName(serverIp)); + if (opt.isPresent()) { + IspResponse response = opt.get(); + addIfNotDuplicate(doc, "ip", response.getIpAddress()); + addIfNotNull(doc, "autonSystemNum", response.getAutonomousSystemNumber()); + addIfNotNull(doc, "autonSystemOrg", response.getAutonomousSystemOrganization()); + addIfNotNull(doc, "isp", response.getIsp()); + addIfNotNull(doc, "org", response.getOrganization()); + } else { + LOG.debug("'{}' IP address not found in ISP DB.", serverIp); + } return doc; } diff --git a/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPIndexingFilter.java b/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPIndexingFilter.java index ea30b8c7b2..ce50b71cce 100644 --- a/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPIndexingFilter.java +++ b/src/plugin/index-geoip/src/java/org/apache/nutch/indexer/geoip/GeoIPIndexingFilter.java @@ -16,10 +16,13 @@ */ package org.apache.nutch.indexer.geoip; -import java.lang.invoke.MethodHandles; -import java.net.URL; -import java.io.File; +import java.io.Closeable; import java.io.IOException; +import java.io.InputStream; +import java.lang.invoke.MethodHandles; +import java.util.EnumMap; +import java.util.Map; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.apache.nutch.crawl.CrawlDatum; @@ -32,204 +35,224 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.maxmind.db.CHMCache; import com.maxmind.geoip2.DatabaseReader; import com.maxmind.geoip2.WebServiceClient; +import com.maxmind.geoip2.exception.GeoIp2Exception; /** - * This plugin implements an indexing filter which takes advantage of the GeoIP2-java API. + *

This plugin implements an indexing filter which takes advantage of the GeoIP2-java API.

*

* The third party library distribution provides an API for the GeoIP2 Precision web - * services and databases. The - * API also works with the free GeoLite2 databases. + * href="https://dev.maxmind.com/geoip/geolocate-an-ip/web-services">Precision web + * services, + * GeoLite2 (free) and databases. *

*

- * Depending on the service level agreement, you have with the GeoIP service - * provider, the plugin can add a number of the following fields to the index - * data model: - *

    - *
  1. Continent
    2. - *
  2. Country
    3. - *
  3. Regional Subdivision
    4. - *
  4. City
    5. - *
  5. Postal Code
    6. - *
  6. Latitude/Longitude
    7. - *
  7. ISP/Organization
    8. - *
  8. AS Number
    9. - *
  9. Confidence Factors
    10. - *
  10. Radius
    11. - *
  11. User Type
    12. - *
- * - *

- * Some of the services are documented at the GeoIP2 Precision - * Services webpage where more information can be obtained. + * Multiple databases can be configured simultaneously. Configure each database + * type you want to use by setting its corresponding property: *

- * + *
    + *
  • index.geoip.db.anonymous - Anonymous IP database
    • + *
  • index.geoip.db.asn - ASN database
    • + *
  • index.geoip.db.city - City database
    • + *
  • index.geoip.db.connection - Connection Type database
    • + *
  • index.geoip.db.country - Country database
    • + *
  • index.geoip.db.domain - Domain database
    • + *
  • index.geoip.db.isp - ISP database
    • + *
*

- * You should also consult the following three properties in - * nutch-site.xml + * Alternatively, use the MaxMind Insights web service by setting + * index.geoip.insights.userid and index.geoip.insights.licensekey. *

- * - * 
- *  {@code
- * 
- * 
- *   index.geoip.usage
- *   insightsService
- *   
- *   A string representing the information source to be used for GeoIP information
- *   association. Either enter 'cityDatabase', 'connectionTypeDatabase', 
- *   'domainDatabase', 'ispDatabase' or 'insightsService'. If you wish to use any one of the 
- *   Database options, you should make one of GeoIP2-City.mmdb, GeoIP2-Connection-Type.mmdb, 
- *   GeoIP2-Domain.mmdb or GeoIP2-ISP.mmdb files respectively available on the Hadoop classpath 
- *   and available at runtime. This can be achieved by adding it to `$NUTCH_HOME/conf`.
- *   Alternatively, also the GeoLite2 IP databases (GeoLite2-*.mmdb) can be used.
- *   
- * 
- * 
- * 
- *   index.geoip.userid
- *   
- *   
- *   The userId associated with the GeoIP2 Precision Services account.
- *   
- * 
- * 
- * 
- *   index.geoip.licensekey
- *   
- *   
- *   The license key associated with the GeoIP2 Precision Services account.
- *   
- * 
- * }
- *
- * */ -public class GeoIPIndexingFilter implements IndexingFilter { +public class GeoIPIndexingFilter implements IndexingFilter, Closeable { private static final Logger LOG = LoggerFactory .getLogger(MethodHandles.lookup().lookupClass()); - private Configuration conf; + /** + * Supported GeoIP database types. + */ + public enum DatabaseType { + ANONYMOUS("anonymous"), + ASN("asn"), + CITY("city"), + CONNECTION("connection"), + COUNTRY("country"), + DOMAIN("domain"), + ISP("isp"); - private String usage = null; + private final String propertyName; - WebServiceClient client = null; + DatabaseType(String propertyName) { + this.propertyName = propertyName; + } + + public String getPropertyName() { + return propertyName; + } + } - DatabaseReader reader = null; + private static final String DB_PROPERTY_PREFIX = "index.geoip.db."; + private static final String INSIGHTS_USERID = "index.geoip.insights.userid"; + private static final String INSIGHTS_LICENSEKEY = "index.geoip.insights.licensekey"; - // private AbstractResponse response = null; + private Configuration conf; + private Map readers; + private WebServiceClient insightsClient; /** * Default constructor for this plugin */ public GeoIPIndexingFilter() { + readers = new EnumMap<>(DatabaseType.class); } - /** - * @see org.apache.hadoop.conf.Configurable#getConf() - */ @Override public Configuration getConf() { return this.conf; } - /** - * @see org.apache.hadoop.conf.Configurable#setConf(org.apache.hadoop.conf.Configuration) - */ @Override - public void setConf(Configuration conf) { - this.conf = conf; - usage = conf.get("index.geoip.usage", "insightsService"); - LOG.debug("GeoIP usage medium set to: {}", usage); - if (usage.equalsIgnoreCase("insightsService")) { - client = new WebServiceClient.Builder( - conf.getInt("index.geoip.userid", 12345), - conf.get("index.geoip.licensekey")).build(); - } else { - String dbSuffix = null; - if (usage.equalsIgnoreCase("cityDatabase")) { - dbSuffix = "-City.mmdb"; - } else if (usage.equalsIgnoreCase("connectionTypeDatabase")) { - dbSuffix = "-Connection-Type.mmdb"; - } else if (usage.equalsIgnoreCase("domainDatabase")) { - dbSuffix = "-Domain.mmdb"; - } else if (usage.equalsIgnoreCase("ispDatabase")) { - dbSuffix = "-ISP.mmdb"; - } - String[] dbPrefixes = {"GeoIP2", "GeoLite2"}; - for (String dbPrefix : dbPrefixes) { - String db = dbPrefix + dbSuffix; - URL dbFileUrl = conf.getResource(db); - if (dbFileUrl == null) { - LOG.error("GeoDb file {} not found on classpath", db); - } else { - try { - LOG.info("Reading GeoDb file {}", db); - buildDb(new File(dbFileUrl.getFile())); - } catch (Exception e) { - LOG.error("Failed to read geoDb file {}: ", db, e); - } - } + public void setConf(Configuration config) { + conf = config; + if (!config.getBoolean("store.ip.address", false)) { + LOG.warn("Plugin index-geoip is active but IP address is not stored. " + + "'store.ip.address' must be set to true in nutch-site.xml."); + } + + // Initialize Insights web service if configured + String userId = config.get(INSIGHTS_USERID); + String licenseKey = config.get(INSIGHTS_LICENSEKEY); + if (userId != null && !userId.isEmpty() && licenseKey != null && !licenseKey.isEmpty()) { + insightsClient = new WebServiceClient.Builder( + Integer.parseInt(userId), licenseKey).build(); + LOG.info("Initialized MaxMind Insights web service client."); + } + + // Initialize database readers for each configured database type + for (DatabaseType dbType : DatabaseType.values()) { + String dbFile = config.get(DB_PROPERTY_PREFIX + dbType.getPropertyName()); + if (dbFile != null && !dbFile.isEmpty()) { + loadDatabase(dbType, dbFile, config); } } - if (!conf.getBoolean("store.ip.address", false)) { - LOG.warn("Plugin index-geoip is active but IP address is not stored" - + "(store.ip.address == false)"); + + if (readers.isEmpty() && insightsClient == null) { + LOG.warn("No GeoIP databases or Insights service configured. " + + "Set index.geoip.db. properties or index.geoip.insights.* properties."); + } else { + LOG.debug("GeoIP plugin initialized with {} database(s){}", + readers.size(), + insightsClient != null ? " and Insights service" : ""); } } - private void buildDb(File geoDb) { + /** + * Load a GeoIP database from the classpath. + * @param dbType the database type + * @param dbFile the database file name + * @param config the Hadoop configuration + */ + private void loadDatabase(DatabaseType dbType, String dbFile, Configuration config) { + InputStream db = config.getConfResourceAsInputStream(dbFile); + if (db == null) { + LOG.error("GeoIP {} database file '{}' not found on classpath", dbType, dbFile); + return; + } try { - reader = new DatabaseReader.Builder(geoDb).build(); + DatabaseReader reader = new DatabaseReader.Builder(db) + .withCache(new CHMCache()) + .build(); + readers.put(dbType, reader); + LOG.debug("Loaded GeoIP {} database from file: {}", dbType, dbFile); } catch (IOException e) { - LOG.error("Failed to build geoDb:", e); + LOG.error("Failed to load GeoIP {} database from file: {} - {}", + dbType, dbFile, e.getMessage()); } } - /** - * - * @see org.apache.nutch.indexer.IndexingFilter#filter(org.apache.nutch.indexer.NutchDocument, - * org.apache.nutch.parse.Parse, org.apache.hadoop.io.Text, - * org.apache.nutch.crawl.CrawlDatum, org.apache.nutch.crawl.Inlinks) - */ @Override public NutchDocument filter(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks) throws IndexingException { - return addServerGeo(doc, parse.getData(), url.toString()); + return augmentNutchDocWithIPData(doc, parse.getData()); } - private NutchDocument addServerGeo(NutchDocument doc, ParseData data, - String url) { - + private NutchDocument augmentNutchDocWithIPData(NutchDocument doc, ParseData data) { String serverIp = data.getContentMeta().get("_ip_"); - if (serverIp != null && reader != null) { + if (serverIp == null || serverIp.isEmpty()) { + return doc; + } + + if (readers.isEmpty() && insightsClient == null) { + return doc; + } + + // Query each configured database + for (Map.Entry entry : readers.entrySet()) { + try { + doc = queryDatabase(entry.getKey(), serverIp, doc, entry.getValue()); + } catch (IOException | GeoIp2Exception e) { + LOG.error("Error querying {} database for IP {}: {}", + entry.getKey(), serverIp, e.getMessage()); + } + } + + // Query Insights service if configured + if (insightsClient != null) { try { - if (usage.equalsIgnoreCase("cityDatabase")) { - doc = GeoIPDocumentCreator.createDocFromCityDb(serverIp, doc, reader); - } else if (usage.equalsIgnoreCase("connectionTypeDatabase")) { - doc = GeoIPDocumentCreator.createDocFromConnectionDb(serverIp, doc, - reader); - } else if (usage.equalsIgnoreCase("domainDatabase")) { - doc = GeoIPDocumentCreator.createDocFromDomainDb(serverIp, doc, - reader); - } else if (usage.equalsIgnoreCase("ispDatabase")) { - doc = GeoIPDocumentCreator.createDocFromIspDb(serverIp, doc, reader); - } else if (usage.equalsIgnoreCase("insightsService")) { - doc = GeoIPDocumentCreator.createDocFromInsightsService(serverIp, doc, - client); - } - } catch (Exception e) { - LOG.error("Failed to determine geoip:", e); + doc = GeoIPDocumentCreator.createDocFromInsightsService(serverIp, doc, insightsClient); + } catch (IOException | GeoIp2Exception e) { + LOG.error("Error querying Insights service for IP {}: {}", serverIp, e.getMessage()); } } + return doc; } + /** + * Query the appropriate database based on type. + */ + private NutchDocument queryDatabase(DatabaseType dbType, String serverIp, + NutchDocument doc, DatabaseReader reader) throws IOException, GeoIp2Exception { + switch (dbType) { + case ANONYMOUS: + return GeoIPDocumentCreator.createDocFromAnonymousIpDb(serverIp, doc, reader); + case ASN: + return GeoIPDocumentCreator.createDocFromAsnDb(serverIp, doc, reader); + case CITY: + return GeoIPDocumentCreator.createDocFromCityDb(serverIp, doc, reader); + case CONNECTION: + return GeoIPDocumentCreator.createDocFromConnectionDb(serverIp, doc, reader); + case COUNTRY: + return GeoIPDocumentCreator.createDocFromCountryDb(serverIp, doc, reader); + case DOMAIN: + return GeoIPDocumentCreator.createDocFromDomainDb(serverIp, doc, reader); + case ISP: + return GeoIPDocumentCreator.createDocFromIspDb(serverIp, doc, reader); + default: + LOG.warn("Unknown database type: {}", dbType); + return doc; + } + } + + @Override + public void close() throws IOException { + for (Map.Entry entry : readers.entrySet()) { + try { + entry.getValue().close(); + } catch (IOException e) { + LOG.warn("Error closing {} database reader: {}", entry.getKey(), e.getMessage()); + } + } + readers.clear(); + + // WebServiceClient doesn't implement Closeable in GeoIP2 5.x + insightsClient = null; + } + } diff --git a/src/plugin/index-geoip/src/test/org/apache/nutch/indexer/geoip/TestGeoIPIndexingFilter.java b/src/plugin/index-geoip/src/test/org/apache/nutch/indexer/geoip/TestGeoIPIndexingFilter.java new file mode 100644 index 0000000000..cd75559ee8 --- /dev/null +++ b/src/plugin/index-geoip/src/test/org/apache/nutch/indexer/geoip/TestGeoIPIndexingFilter.java @@ -0,0 +1,430 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nutch.indexer.geoip; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.crawl.Inlinks; +import org.apache.nutch.indexer.NutchDocument; +import org.apache.nutch.indexer.NutchField; +import org.apache.nutch.parse.ParseData; +import org.apache.nutch.parse.ParseImpl; +import org.apache.nutch.util.NutchConfiguration; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Tests for {@link org.apache.nutch.indexer.geoip.GeoIPIndexingFilter} + */ +public class TestGeoIPIndexingFilter { + + private Configuration conf; + private GeoIPIndexingFilter filter; + private NutchDocument doc; + private ParseImpl parseImpl; + private Text text; + private CrawlDatum crawlDatum; + private Inlinks inlinks; + + @BeforeEach + public void setUp() { + conf = NutchConfiguration.create(); + conf.setBoolean("store.ip.address", true); + doc = new NutchDocument(); + parseImpl = new ParseImpl("foo bar", new ParseData()); + inlinks = new Inlinks(); + text = new Text("http://nutch.apache.org/index.html"); + crawlDatum = new CrawlDatum(); + } + + @AfterEach + public void teardown() throws Exception { + if (filter != null) { + filter.close(); + } + } + + /** + * Test method for {@link org.apache.nutch.indexer.geoip.GeoIPIndexingFilter#getConf()}. + */ + @Test + public void testGetConf() { + filter = new GeoIPIndexingFilter(); + filter.setConf(conf); + assertTrue(filter.getConf().getBoolean("store.ip.address", true)); + } + + /** + * Test that City database configuration property works correctly. + */ + @Test + public void testSetConfCityDbProperty() { + conf.set("index.geoip.db.city", "GeoIP2-City-Test.mmdb"); + filter = new GeoIPIndexingFilter(); + filter.setConf(conf); + assertEquals("GeoIP2-City-Test.mmdb", filter.getConf().get("index.geoip.db.city")); + } + + /** + * Test City database with Singapore location data. + */ + @Test + public void testCityDatabaseSingapore() { + conf.set("index.geoip.db.city", "GeoIP2-City-Test.mmdb"); + filter = new GeoIPIndexingFilter(); + filter.setConf(conf); + // IP 214.0.0.1 maps to Singapore in the test database + parseImpl.getData().getContentMeta().add("_ip_", "214.0.0.1"); + try { + filter.filter(doc, parseImpl, text, crawlDatum, inlinks); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + assertNotNull(doc); + // Verify city data for Singapore + assertEquals("Singapore", doc.getFieldValue("cityName")); + assertEquals("AS", doc.getFieldValue("continentCode")); + assertEquals("Asia", doc.getFieldValue("continentName")); + assertEquals("SG", doc.getFieldValue("countryIsoCode")); + assertEquals("Singapore", doc.getFieldValue("countryName")); + assertEquals("Asia/Singapore", doc.getFieldValue("timeZone")); + assertEquals("59", doc.getFieldValue("postalCode")); + // Verify lat/lon is present + assertNotNull(doc.getFieldValue("latLon")); + assertTrue(doc.getFieldValue("latLon").toString().contains("1.336")); + } + + /** + * Test City database with Melbourne, Australia location data. + */ + @Test + public void testCityDatabaseMelbourne() { + conf.set("index.geoip.db.city", "GeoIP2-City-Test.mmdb"); + filter = new GeoIPIndexingFilter(); + filter.setConf(conf); + // IP 214.0.1.1 maps to Melbourne, Australia in the test database + parseImpl.getData().getContentMeta().add("_ip_", "214.0.1.1"); + try { + filter.filter(doc, parseImpl, text, crawlDatum, inlinks); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + assertNotNull(doc); + // Verify city data for Melbourne + assertEquals("Melbourne", doc.getFieldValue("cityName")); + assertEquals("OC", doc.getFieldValue("continentCode")); + assertEquals("Oceania", doc.getFieldValue("continentName")); + assertEquals("AU", doc.getFieldValue("countryIsoCode")); + assertEquals("Australia", doc.getFieldValue("countryName")); + assertEquals("Australia/Melbourne", doc.getFieldValue("timeZone")); + // Verify lat/lon is present + assertNotNull(doc.getFieldValue("latLon")); + assertTrue(doc.getFieldValue("latLon").toString().contains("-37.8159")); + } + + /** + * Test Connection Type database with Cable/DSL connection. + */ + @Test + public void testConnectionTypeCableDsl() { + conf.set("index.geoip.db.connection", "GeoIP2-Connection-Type-Test.mmdb"); + filter = new GeoIPIndexingFilter(); + filter.setConf(conf); + // IP 1.0.0.1 maps to Cable/DSL connection type in the test database + parseImpl.getData().getContentMeta().add("_ip_", "1.0.0.1"); + try { + filter.filter(doc, parseImpl, text, crawlDatum, inlinks); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + assertNotNull(doc); + assertEquals("Cable/DSL", doc.getFieldValue("connectionType")); + assertNotNull(doc.getFieldValue("ip")); + assertNotNull(doc.getFieldValue(GeoIPDocumentCreator.CONNECTION_NETWORK_ADDRESS)); + } + + /** + * Test Connection Type database with Cellular connection. + */ + @Test + public void testConnectionTypeCellular() { + conf.set("index.geoip.db.connection", "GeoIP2-Connection-Type-Test.mmdb"); + filter = new GeoIPIndexingFilter(); + filter.setConf(conf); + // IP 1.0.1.1 maps to Cellular connection type in the test database + parseImpl.getData().getContentMeta().add("_ip_", "1.0.1.1"); + try { + filter.filter(doc, parseImpl, text, crawlDatum, inlinks); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + assertNotNull(doc); + assertEquals("Cellular", doc.getFieldValue("connectionType")); + assertNotNull(doc.getFieldValue("ip")); + } + + /** + * Test Connection Type database with Corporate connection. + */ + @Test + public void testConnectionTypeCorporate() { + conf.set("index.geoip.db.connection", "GeoIP2-Connection-Type-Test.mmdb"); + filter = new GeoIPIndexingFilter(); + filter.setConf(conf); + // IP 201.243.200.1 maps to Corporate connection type in the test database + parseImpl.getData().getContentMeta().add("_ip_", "201.243.200.1"); + try { + filter.filter(doc, parseImpl, text, crawlDatum, inlinks); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + assertNotNull(doc); + assertEquals("Corporate", doc.getFieldValue("connectionType")); + assertNotNull(doc.getFieldValue("ip")); + } + + /** + * Test multiple databases configured simultaneously. + * Both City and Connection Type databases are loaded and queried. + */ + @Test + public void testMultipleDatabases() { + // Configure both City and Connection Type databases + conf.set("index.geoip.db.city", "GeoIP2-City-Test.mmdb"); + conf.set("index.geoip.db.connection", "GeoIP2-Connection-Type-Test.mmdb"); + filter = new GeoIPIndexingFilter(); + filter.setConf(conf); + + // Use IP 1.0.0.1 which exists in Connection Type test database + // Note: This IP may not exist in the City database, so we just verify + // that Connection Type data is returned + parseImpl.getData().getContentMeta().add("_ip_", "1.0.0.1"); + try { + filter.filter(doc, parseImpl, text, crawlDatum, inlinks); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + assertNotNull(doc); + // Verify connection type data is present + assertEquals("Cable/DSL", doc.getFieldValue("connectionType")); + assertNotNull(doc.getFieldValue("ip")); + } + + /** + * Test that filter handles null/empty IP gracefully. + */ + @Test + public void testNullIpAddress() { + conf.set("index.geoip.db.city", "GeoIP2-City-Test.mmdb"); + filter = new GeoIPIndexingFilter(); + filter.setConf(conf); + // Don't set any IP address + try { + NutchDocument result = filter.filter(doc, parseImpl, text, crawlDatum, inlinks); + assertNotNull(result); + // Document should be returned unchanged + assertTrue(result.getFieldNames().isEmpty()); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + /** + * Test that filter handles no configured databases gracefully. + */ + @Test + public void testNoConfiguredDatabases() { + // Don't configure any databases + filter = new GeoIPIndexingFilter(); + filter.setConf(conf); + parseImpl.getData().getContentMeta().add("_ip_", "1.0.0.1"); + try { + NutchDocument result = filter.filter(doc, parseImpl, text, crawlDatum, inlinks); + assertNotNull(result); + // Document should be returned unchanged when no databases configured + assertTrue(result.getFieldNames().isEmpty()); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + /** + * Test addIfNotDuplicate method adds value when field doesn't exist. + */ + @Test + public void testAddIfNotDuplicateNewField() { + NutchDocument testDoc = new NutchDocument(); + GeoIPDocumentCreator.addIfNotDuplicate(testDoc, "ip", "192.168.1.1"); + + assertNotNull(testDoc.getFieldValue("ip")); + assertEquals("192.168.1.1", testDoc.getFieldValue("ip")); + assertEquals(1, testDoc.getField("ip").getValues().size()); + } + + /** + * Test addIfNotDuplicate method doesn't add duplicate values. + */ + @Test + public void testAddIfNotDuplicatePreventsDuplicates() { + NutchDocument testDoc = new NutchDocument(); + + // Add the same IP three times (simulating multiple database lookups) + GeoIPDocumentCreator.addIfNotDuplicate(testDoc, "ip", "192.168.1.1"); + GeoIPDocumentCreator.addIfNotDuplicate(testDoc, "ip", "192.168.1.1"); + GeoIPDocumentCreator.addIfNotDuplicate(testDoc, "ip", "192.168.1.1"); + + // Should only have one value + NutchField field = testDoc.getField("ip"); + assertNotNull(field); + assertEquals(1, field.getValues().size()); + assertEquals("192.168.1.1", field.getValues().get(0)); + } + + /** + * Test addIfNotDuplicate method allows different values. + */ + @Test + public void testAddIfNotDuplicateAllowsDifferentValues() { + NutchDocument testDoc = new NutchDocument(); + + // Add different IPs (this shouldn't happen in practice but tests the logic) + GeoIPDocumentCreator.addIfNotDuplicate(testDoc, "ip", "192.168.1.1"); + GeoIPDocumentCreator.addIfNotDuplicate(testDoc, "ip", "192.168.1.2"); + + // Should have two different values + NutchField field = testDoc.getField("ip"); + assertNotNull(field); + assertEquals(2, field.getValues().size()); + assertTrue(field.getValues().contains("192.168.1.1")); + assertTrue(field.getValues().contains("192.168.1.2")); + } + + /** + * Test addIfNotDuplicate method handles null values gracefully. + */ + @Test + public void testAddIfNotDuplicateNullValue() { + NutchDocument testDoc = new NutchDocument(); + + // Adding null should not create a field + GeoIPDocumentCreator.addIfNotDuplicate(testDoc, "ip", null); + + assertNull(testDoc.getField("ip")); + } + + /** + * Test that database-specific network address field names are used correctly. + * Connection Type database should use connectionNetworkAddress field. + */ + @Test + public void testConnectionDatabaseSpecificNetworkAddressField() { + conf.set("index.geoip.db.connection", "GeoIP2-Connection-Type-Test.mmdb"); + filter = new GeoIPIndexingFilter(); + filter.setConf(conf); + parseImpl.getData().getContentMeta().add("_ip_", "1.0.0.1"); + try { + filter.filter(doc, parseImpl, text, crawlDatum, inlinks); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + assertNotNull(doc); + // Verify that database-specific network address field is used + assertNotNull(doc.getFieldValue(GeoIPDocumentCreator.CONNECTION_NETWORK_ADDRESS)); + // Verify that the old generic field is NOT used + assertNull(doc.getField("networkAddress")); + } + + /** + * Test that multiple databases use their own specific network address fields + * and that IP address is not duplicated. + */ + @Test + public void testMultipleDatabasesUseSpecificNetworkAddressFields() { + // Configure both City and Connection Type databases + conf.set("index.geoip.db.city", "GeoIP2-City-Test.mmdb"); + conf.set("index.geoip.db.connection", "GeoIP2-Connection-Type-Test.mmdb"); + filter = new GeoIPIndexingFilter(); + filter.setConf(conf); + + // Use IP 1.0.0.1 which exists in Connection Type test database + parseImpl.getData().getContentMeta().add("_ip_", "1.0.0.1"); + try { + filter.filter(doc, parseImpl, text, crawlDatum, inlinks); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + assertNotNull(doc); + + // Verify connection type data is present with specific network address field + assertEquals("Cable/DSL", doc.getFieldValue("connectionType")); + assertNotNull(doc.getFieldValue(GeoIPDocumentCreator.CONNECTION_NETWORK_ADDRESS)); + + // Verify the generic networkAddress field is NOT used + assertNull(doc.getField("networkAddress")); + + // Verify IP field exists and has only one value (not duplicated) + NutchField ipField = doc.getField("ip"); + assertNotNull(ipField); + assertEquals(1, ipField.getValues().size(), + "IP field should have exactly one value, not duplicated across databases"); + } + + /** + * Test that Country database configuration property is correctly recognized. + * Note: This test verifies the configuration works. A full test would require + * a GeoIP2-Country-Test.mmdb file which is not currently available in the + * test resources. The City database can be used for country-level lookups + * in production as it contains the same country data. + */ + @Test + public void testCountryDatabaseConfigProperty() { + // Verify the country property is recognized (won't load without actual file) + conf.set("index.geoip.db.country", "nonexistent-country-test.mmdb"); + filter = new GeoIPIndexingFilter(); + filter.setConf(conf); + // Configuration should be accepted without error (file missing is logged as warning) + assertEquals("nonexistent-country-test.mmdb", + filter.getConf().get("index.geoip.db.country")); + } + + /** + * Test Country database specific network address field constant is defined. + */ + @Test + public void testCountryNetworkAddressFieldConstant() { + // Verify the COUNTRY_NETWORK_ADDRESS constant is properly defined + assertEquals("countryNetworkAddress", GeoIPDocumentCreator.COUNTRY_NETWORK_ADDRESS); + } +} diff --git a/src/plugin/indexer-solr/schema.xml b/src/plugin/indexer-solr/schema.xml index 38635e22db..17f77534cb 100644 --- a/src/plugin/indexer-solr/schema.xml +++ b/src/plugin/indexer-solr/schema.xml @@ -353,37 +353,85 @@ - - - - - - - + + + + + + + + + + + + + + + + + + + - - - + + + + - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +