NUTCH-3064 Upgrade index-geoip to GeoIP2 5.0.2

.gitignore

@@ -1,5 +1,6 @@
 conf/*.txt
 conf/*.xml
+conf/*.mmdb
 !conf/nutch-default.xml
 conf/hadoop-env.sh
 conf/slaves

build.xml

@@ -1186,6 +1186,7 @@
         <source path="${plugins.dir}/index-basic/src/java/" />
         <source path="${plugins.dir}/index-basic/src/test/" />
         <source path="${plugins.dir}/index-geoip/src/java/" />
+        <source path="${plugins.dir}/index-geoip/src/test/" />
         <source path="${plugins.dir}/index-jexl-filter/src/java/" />
         <source path="${plugins.dir}/index-jexl-filter/src/test/" />
         <source path="${plugins.dir}/index-links/src/java/" />

conf/nutch-default.xml

@@ -2231,33 +2231,98 @@ CAUTION: Set the parser.timeout to -1 or a bigger value than 30, when using this
 </property>
 
 <!-- index-geoip plugin properties -->
+<!-- 
+  To use the index-geoip plugin, you must set 'store.ip.address' to true.
+
+  Configure one or more GeoIP databases by setting the corresponding property below.
+  Each database type can be enabled independently - set any combination you need.
+  Database files must be in MMDB format and available on the Hadoop classpath
+  (e.g., place them in $NUTCH_HOME/conf).
+
+  More information: https://support.maxmind.com/knowledge-base/articles/maxmind-database-formats
+-->
+
+<property>
+  <name>index.geoip.db.anonymous</name>
+  <value></value>
+  <description>
+  GeoIP2 Anonymous IP database file (MMDB format).
+  Identifies anonymous proxies, VPNs, Tor exit nodes, and hosting providers.
+  </description>
+</property>
+
+<property>
+  <name>index.geoip.db.asn</name>
+  <value></value>
+  <description>
+  GeoIP2/GeoLite2 ASN database file (MMDB format).
+  Provides autonomous system number and organization information.
+  </description>
+</property>
+
+<property>
+  <name>index.geoip.db.city</name>
+  <value></value>
+  <description>
+  GeoIP2/GeoLite2 City database file (MMDB format).
+  Provides city, subdivision, country, continent, and location data.
+  </description>
+</property>
+
 <property>
-  <name>index.geoip.usage</name>
-  <value>insightsService</value>
+  <name>index.geoip.db.connection</name>
+  <value></value>
+  <description>
+  GeoIP2 Connection Type database file (MMDB format).
+  Identifies connection type: Cable/DSL, Cellular, Corporate, or Satellite.
+  </description>
+</property>
+
+<property>
+  <name>index.geoip.db.country</name>
+  <value></value>
+  <description>
+  GeoIP2/GeoLite2 Country database file (MMDB format).
+  Provides country, continent, and represented country information.
+  This is a lighter-weight alternative to the City database when only
+  country-level information is needed.
+  </description>
+</property>
+
+<property>
+  <name>index.geoip.db.domain</name>
+  <value></value>
+  <description>
+  GeoIP2 Domain database file (MMDB format).
+  Provides the second-level domain associated with the IP address.
+  </description>
+</property>
+
+<property>
+  <name>index.geoip.db.isp</name>
+  <value></value>
   <description>
-  A string representing the information source to be used for GeoIP information
-  association. Either enter 'cityDatabase', 'connectionTypeDatabase',
-  'domainDatabase', 'ispDatabase' or 'insightsService'. If you wish to use any one of the
-  Database options, you should make one of GeoIP2-City.mmdb, GeoIP2-Connection-Type.mmdb,
-  GeoIP2-Domain.mmdb or GeoIP2-ISP.mmdb files respectively available on the classpath and
-  available at runtime. Alternatively, also the GeoLite2 IP databases (GeoLite2-*.mmdb)
-  can be used.
+  GeoIP2 ISP database file (MMDB format).
+  Provides ISP, organization, and autonomous system information.
   </description>
 </property>
 
+<!-- MaxMind Insights Web Service (alternative to local databases) -->
 <property>
-  <name>index.geoip.userid</name>
+  <name>index.geoip.insights.userid</name>
   <value></value>
   <description>
-  The userId associated with the GeoIP2 Precision Services account.
+  The userId for the MaxMind GeoIP2 Precision Insights web service.
+  If set along with licensekey, the Insights service will be queried in addition
+  to any configured local databases.
   </description>
 </property>
 
 <property>
-  <name>index.geoip.licensekey</name>
+  <name>index.geoip.insights.licensekey</name>
   <value></value>
   <description>
-  The license key associated with the GeoIP2 Precision Services account.
+  The license key for the MaxMind GeoIP2 Precision Insights web service.
   </description>
 </property>
 

src/plugin/build.xml

@@ -120,7 +120,7 @@
      <ant dir="index-anchor" target="test"/>
      <ant dir="index-arbitrary" target="test"/>
      <ant dir="index-basic" target="test"/>
-     <!--ant dir="index-geoip" target="test"/-->
+     <ant dir="index-geoip" target="test"/>
      <ant dir="index-jexl-filter" target="test"/>
      <ant dir="index-links" target="test"/>
      <ant dir="index-more" target="test"/>

src/plugin/index-geoip/build.xml

@@ -23,5 +23,9 @@
     <copy todir="${build.classes}">
       <fileset dir="${src.dir}" includes="**/*.mmdb" />
     </copy>
+    <!-- Copy sample mmdb files for testing -->
+    <copy todir="${build.classes}">
+      <fileset dir="${root}/sample" includes="**/*.mmdb" erroronmissingdir="false"/>
+    </copy>
   </target>
 </project>

src/plugin/index-geoip/ivy.xml

@@ -37,7 +37,7 @@
   </publications>
 
   <dependencies>
-    <dependency org="com.maxmind.geoip2" name="geoip2" rev="3.0.1">
+    <dependency org="com.maxmind.geoip2" name="geoip2" rev="5.0.2">
       <!-- Exlude libs provided in Nutch core -->
       <exclude org="com.fasterxml.jackson.core" name="jackson-annotations" />
       <exclude org="com.fasterxml.jackson.core" name="jackson-databind" />

src/plugin/index-geoip/plugin.xml

@@ -26,8 +26,9 @@
       <library name="index-geoip.jar">
          <export name="*"/>
       </library>
-      <library name="geoip2-3.0.1.jar"/>
-      <library name="maxmind-db-2.0.0.jar"/>
+      <library name="geoip2-5.0.2.jar"/>
+      <library name="jackson-datatype-jsr310-2.20.1.jar"/>
+      <library name="maxmind-db-4.0.2.jar"/>
    </runtime>
 
    <requires>