From dedd831d4f582c39052d3081b77d7f94ba1c0cf4 Mon Sep 17 00:00:00 2001 From: Isabelle Giguere Date: Thu, 5 Feb 2026 18:20:28 -0500 Subject: [PATCH 1/8] NUTCH-1732: allow deleting non-parsable documents Add CrawlDatum status "parse_failed"/"db_parse_failed", set status in CrawlDbReducer. Handle deletion in IndexerMapReduce. Add unit tests. --- conf/nutch-default.xml | 11 +- .../org/apache/nutch/crawl/CrawlDatum.java | 6 + .../apache/nutch/crawl/CrawlDbReducer.java | 16 +- .../apache/nutch/fetcher/FetcherThread.java | 5 + .../nutch/indexer/IndexerMapReduce.java | 12 + .../apache/nutch/metrics/NutchMetrics.java | 3 + .../org/apache/nutch/parse/ParseSegment.java | 1 + .../org/apache/nutch/parse/ParseStatus.java | 4 + .../apache/nutch/crawl/TestCrawlDbStates.java | 348 ++++++++++++++---- .../crawl/TestCrawlDbStatesExtended.java | 195 ---------- .../nutch/indexer/TestIndexerMapReduce.java | 35 ++ 11 files changed, 362 insertions(+), 274 deletions(-) delete mode 100644 src/test/org/apache/nutch/crawl/TestCrawlDbStatesExtended.java diff --git a/conf/nutch-default.xml b/conf/nutch-default.xml index b41f7ecdab..2efbee9274 100644 --- a/conf/nutch-default.xml +++ b/conf/nutch-default.xml @@ -1794,6 +1794,15 @@ CAUTION: Set the parser.timeout to -1 or a bigger value than 30, when using this + + parser.delete.failed + false + Boolean value for whether we should delete a page from the index when parsing the page fails. + By default this property is deactivated, because it will delete an existing page from the index, where a + previous fetch produced content that was successfully parsed. + + + parser.store.text true @@ -2426,7 +2435,7 @@ CAUTION: Set the parser.timeout to -1 or a bigger value than 30, when using this link.delete.gone false - Whether to delete gone pages from the web graph. + Whether to delete gone pages from the web graph. Gone pages include redirects and duplicates. diff --git a/src/java/org/apache/nutch/crawl/CrawlDatum.java b/src/java/org/apache/nutch/crawl/CrawlDatum.java index c76fc0f336..68b125064d 100644 --- a/src/java/org/apache/nutch/crawl/CrawlDatum.java +++ b/src/java/org/apache/nutch/crawl/CrawlDatum.java @@ -75,6 +75,8 @@ public class CrawlDatum implements WritableComparable, Cloneable { public static final byte STATUS_DB_DUPLICATE = 0x07; /** Page was marked as orphan, e.g. has no inlinks anymore */ public static final byte STATUS_DB_ORPHAN = 0x08; + /** Page parsing failed */ + public static final byte STATUS_DB_PARSE_FAILED = 0x09; /** Maximum value of DB-related status. */ public static final byte STATUS_DB_MAX = 0x1f; @@ -103,6 +105,8 @@ public class CrawlDatum implements WritableComparable, Cloneable { public static final byte STATUS_LINKED = 0x43; /** Page got metadata from a parser */ public static final byte STATUS_PARSE_META = 0x44; + /** Page parse failed */ + public static final byte STATUS_PARSE_FAILED = 0x45; public static final HashMap statNames = new HashMap<>(); static { @@ -114,6 +118,7 @@ public class CrawlDatum implements WritableComparable, Cloneable { statNames.put(STATUS_DB_NOTMODIFIED, "db_notmodified"); statNames.put(STATUS_DB_DUPLICATE, "db_duplicate"); statNames.put(STATUS_DB_ORPHAN, "db_orphan"); + statNames.put(STATUS_DB_PARSE_FAILED, "db_parse_failed"); statNames.put(STATUS_SIGNATURE, "signature"); statNames.put(STATUS_INJECTED, "injected"); statNames.put(STATUS_LINKED, "linked"); @@ -124,6 +129,7 @@ public class CrawlDatum implements WritableComparable, Cloneable { statNames.put(STATUS_FETCH_GONE, "fetch_gone"); statNames.put(STATUS_FETCH_NOTMODIFIED, "fetch_notmodified"); statNames.put(STATUS_PARSE_META, "parse_metadata"); + statNames.put(STATUS_PARSE_FAILED, "parse_failed"); oldToNew.put(OLD_STATUS_DB_UNFETCHED, STATUS_DB_UNFETCHED); oldToNew.put(OLD_STATUS_DB_FETCHED, STATUS_DB_FETCHED); diff --git a/src/java/org/apache/nutch/crawl/CrawlDbReducer.java b/src/java/org/apache/nutch/crawl/CrawlDbReducer.java index e263f8463c..8a6186e64e 100644 --- a/src/java/org/apache/nutch/crawl/CrawlDbReducer.java +++ b/src/java/org/apache/nutch/crawl/CrawlDbReducer.java @@ -96,7 +96,9 @@ public void reduce(Text key, Iterable values, continue; } - if (CrawlDatum.hasFetchStatus(datum)) { + // temporarily handle parse_failed as fetched (it was fetched!) + if (CrawlDatum.hasFetchStatus(datum) + || datum.getStatus() == CrawlDatum.STATUS_PARSE_FAILED) { if (!fetchSet) { if (multiple) { fetch.set(datum); @@ -111,7 +113,7 @@ public void reduce(Text key, Iterable values, } continue; } - + switch (datum.getStatus()) { // collect other info case CrawlDatum.STATUS_LINKED: CrawlDatum link; @@ -213,7 +215,7 @@ public void reduce(Text key, Iterable values, } break; - case CrawlDatum.STATUS_FETCH_SUCCESS: // succesful fetch + case CrawlDatum.STATUS_FETCH_SUCCESS: // successful fetch case CrawlDatum.STATUS_FETCH_REDIR_TEMP: // successful fetch, redirected case CrawlDatum.STATUS_FETCH_REDIR_PERM: case CrawlDatum.STATUS_FETCH_NOTMODIFIED: // successful fetch, notmodified @@ -300,6 +302,14 @@ public void reduce(Text key, Iterable values, } break; + case CrawlDatum.STATUS_PARSE_FAILED: // successful fetch, but parse failed + if (oldSet) + result.setSignature(old.getSignature()); // use old signature + result.setStatus(CrawlDatum.STATUS_DB_PARSE_FAILED); + result = schedule.setPageGoneSchedule(key, result, prevFetchTime, + prevModifiedTime, fetch.getFetchTime()); + break; + case CrawlDatum.STATUS_FETCH_GONE: // permanent failure if (oldSet) result.setSignature(old.getSignature()); // use old signature diff --git a/src/java/org/apache/nutch/fetcher/FetcherThread.java b/src/java/org/apache/nutch/fetcher/FetcherThread.java index 2abcfe9f68..820159ec91 100644 --- a/src/java/org/apache/nutch/fetcher/FetcherThread.java +++ b/src/java/org/apache/nutch/fetcher/FetcherThread.java @@ -112,6 +112,7 @@ public class FetcherThread extends Thread { URLNormalizers normalizersForOutlinks; private boolean skipTruncated; + private boolean deleteFailedParse; private boolean halted = false; @@ -182,6 +183,7 @@ public FetcherThread(Configuration conf, AtomicInteger activeThreads, FetchItemQ this.scfilters = new ScoringFilters(conf); this.parseUtil = new ParseUtil(conf); this.skipTruncated = conf.getBoolean(ParseSegment.SKIP_TRUNCATED, true); + this.deleteFailedParse = conf.getBoolean(ParseSegment.DELETE_FAILED_PARSE, false); this.signatureWithoutParsing = conf.getBoolean("fetcher.signature", false); this.protocolFactory = new ProtocolFactory(conf); this.normalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_FETCHER); @@ -469,6 +471,9 @@ public void run() { fit = queueRedirect(redirUrl, fit); } } + if (pstatus != null && pstatus.isFailed() && deleteFailedParse) { + output(fit.url, fit.datum, null, status, CrawlDatum.STATUS_PARSE_FAILED); + } break; case ProtocolStatus.MOVED: // redirect diff --git a/src/java/org/apache/nutch/indexer/IndexerMapReduce.java b/src/java/org/apache/nutch/indexer/IndexerMapReduce.java index b61a7f99cd..0eb8141a15 100644 --- a/src/java/org/apache/nutch/indexer/IndexerMapReduce.java +++ b/src/java/org/apache/nutch/indexer/IndexerMapReduce.java @@ -225,6 +225,7 @@ public static class IndexerReducer extends private Counter deletedGoneCounter; private Counter deletedRedirectsCounter; private Counter deletedDuplicatesCounter; + private Counter deletedFailedParseCounter; private Counter skippedNotModifiedCounter; private Counter errorsScoringFilterCounter; private Counter errorsIndexingFilterCounter; @@ -277,6 +278,8 @@ private void initCounters(Reducer.C NutchMetrics.GROUP_INDEXER, NutchMetrics.INDEXER_DELETED_REDIRECTS_TOTAL); deletedDuplicatesCounter = context.getCounter( NutchMetrics.GROUP_INDEXER, NutchMetrics.INDEXER_DELETED_DUPLICATES_TOTAL); + deletedFailedParseCounter = context.getCounter( + NutchMetrics.GROUP_INDEXER, NutchMetrics.INDEXER_DELETED_FAILED_PARSE_TOTAL); skippedNotModifiedCounter = context.getCounter( NutchMetrics.GROUP_INDEXER, NutchMetrics.INDEXER_SKIPPED_NOT_MODIFIED_TOTAL); errorsScoringFilterCounter = context.getCounter( @@ -354,6 +357,15 @@ public void reduce(Text key, Iterable values, } } + // Whether to delete pages where parsing failed + if (delete && fetchDatum != null) { + if (fetchDatum.getStatus() == CrawlDatum.STATUS_PARSE_FAILED + || dbDatum != null && dbDatum.getStatus() == CrawlDatum.STATUS_DB_PARSE_FAILED) { + deletedFailedParseCounter.increment(1); + context.write(key, DELETE_ACTION); + return; + } + } // Whether to delete GONE or REDIRECTS if (delete && fetchDatum != null) { if (fetchDatum.getStatus() == CrawlDatum.STATUS_FETCH_GONE diff --git a/src/java/org/apache/nutch/metrics/NutchMetrics.java b/src/java/org/apache/nutch/metrics/NutchMetrics.java index dea34be7f6..95e2a348ec 100644 --- a/src/java/org/apache/nutch/metrics/NutchMetrics.java +++ b/src/java/org/apache/nutch/metrics/NutchMetrics.java @@ -191,6 +191,9 @@ private NutchMetrics() { /** Documents deleted as duplicates. */ public static final String INDEXER_DELETED_DUPLICATES_TOTAL = "deleted_duplicates_total"; + /** Documents deleted because parsing failed. */ + public static final String INDEXER_DELETED_FAILED_PARSE_TOTAL = "deleted_failed_parse_total"; + /** Documents deleted by indexing filter. */ public static final String INDEXER_DELETED_BY_INDEXING_FILTER_TOTAL = "deleted_by_indexing_filter_total"; diff --git a/src/java/org/apache/nutch/parse/ParseSegment.java b/src/java/org/apache/nutch/parse/ParseSegment.java index a7fbe066ce..43fd85a423 100644 --- a/src/java/org/apache/nutch/parse/ParseSegment.java +++ b/src/java/org/apache/nutch/parse/ParseSegment.java @@ -66,6 +66,7 @@ public class ParseSegment extends NutchTool implements Tool { .getLogger(MethodHandles.lookup().lookupClass()); public static final String SKIP_TRUNCATED = "parser.skip.truncated"; + public static final String DELETE_FAILED_PARSE = "parser.delete.failed"; public ParseSegment() { this(null); diff --git a/src/java/org/apache/nutch/parse/ParseStatus.java b/src/java/org/apache/nutch/parse/ParseStatus.java index 052a342247..e1f32df28c 100644 --- a/src/java/org/apache/nutch/parse/ParseStatus.java +++ b/src/java/org/apache/nutch/parse/ParseStatus.java @@ -188,6 +188,10 @@ public void write(DataOutput out) throws IOException { public boolean isSuccess() { return majorCode == SUCCESS; } + + public boolean isFailed() { + return majorCode == FAILED; + } /** * @return a String representation of the first argument, diff --git a/src/test/org/apache/nutch/crawl/TestCrawlDbStates.java b/src/test/org/apache/nutch/crawl/TestCrawlDbStates.java index 34743811a1..636401390b 100644 --- a/src/test/org/apache/nutch/crawl/TestCrawlDbStates.java +++ b/src/test/org/apache/nutch/crawl/TestCrawlDbStates.java @@ -21,9 +21,11 @@ import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.Reducer.Context; import org.apache.hadoop.util.StringUtils; +import org.apache.nutch.parse.ParseSegment; import org.apache.nutch.scoring.ScoringFilterException; import org.apache.nutch.scoring.ScoringFilters; import org.apache.nutch.util.ReducerContextWrapper; +import org.apache.nutch.util.TimingUtil; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -323,7 +325,150 @@ public void testCrawlDbReducerNotModified() { } } - protected class CrawlTestFetchNotModified extends ContinuousCrawlTestUtil { + + /** + * NUTCH-1245: a fetch_gone should always result in a db_gone. + *

+ * Even in a long-running continuous crawl, when a gone page is re-fetched + * several times over time. + *

+ */ + @Test + public void testCrawlDbReducerPageGoneSchedule1() { + LOG.info("NUTCH-1245: test long running continuous crawl"); + ContinuousCrawlTestUtil crawlUtil = new ContinuousCrawlTestUtil( + STATUS_FETCH_GONE, STATUS_DB_GONE); + try { + if (!crawlUtil.run(20)) { + fail("fetch_gone did not result in a db_gone (NUTCH-1245)"); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * NUTCH-1245: a fetch_gone should always result in a db_gone. + *

+ * As some kind of misconfiguration set db.fetch.interval.default to a value + * > (fetchIntervalMax * 1.5). + *

+ */ + @Test + public void testCrawlDbReducerPageGoneSchedule2() { + LOG.info("NUTCH-1245 (misconfiguration): test with db.fetch.interval.default > (1.5 * db.fetch.interval.max)"); + Context context = CrawlDBTestUtil.createContext(); + Configuration conf = context.getConfiguration(); + int fetchIntervalMax = conf.getInt("db.fetch.interval.max", 0); + conf.setInt("db.fetch.interval.default", 3 + (int) (fetchIntervalMax * 1.5)); + ContinuousCrawlTestUtil crawlUtil = new ContinuousCrawlTestUtil(context, + STATUS_FETCH_GONE, STATUS_DB_GONE); + try { + if (!crawlUtil.run(0)) { + fail("fetch_gone did not result in a db_gone (NUTCH-1245)"); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + + @Test + public void testCrawlDbReducerParseFailed() { + LOG.info("NUTCH-1732: allow deleting un-parsable documents"); + Context context = CrawlDBTestUtil.createContext(); + Configuration conf = context.getConfiguration(); + conf.setBoolean(ParseSegment.DELETE_FAILED_PARSE, true); + CrawlTestParserFailure crawlUtil = new CrawlTestParserFailure(context, + CrawlDatum.STATUS_PARSE_FAILED, CrawlDatum.STATUS_DB_PARSE_FAILED); + try { + if (!crawlUtil.run(20)) { + fail("parse failure did not result in a parse_fail (NUTCH-1732)"); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * Test whether signatures are reset for "content-less" states (gone, + * redirect, etc.): otherwise, if this state is temporary and the document + * appears again with the old content, it may get marked as not_modified in + * CrawlDb just after the redirect state. In this case we cannot expect + * content in segments. Cf. NUTCH-1422: reset signature for redirects. + */ + // TODO: can only test if solution is done in CrawlDbReducer + @Test + public void testSignatureReset() { + LOG.info("NUTCH-1422 must reset signature for redirects and similar states"); + Context context = CrawlDBTestUtil.createContext(); + Configuration conf = context.getConfiguration(); + for (String sched : schedules) { + LOG.info("Testing reset signature with {}", sched); + conf.set("db.fetch.schedule.class", "org.apache.nutch.crawl." + sched); + ContinuousCrawlTestUtil crawlUtil = new CrawlTestSignatureReset(context); + try { + if (!crawlUtil.run(20)) { + fail("failed: signature not reset"); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + /** + * NUTCH-578: a fetch_retry should result in a db_gone if db.fetch.retry.max + * is reached. Retry counter has to be reset appropriately. + */ + @Test + public void testCrawlDbReducerPageRetrySchedule() { + LOG.info("NUTCH-578: test long running continuous crawl with fetch_retry"); + ContinuousCrawlTestUtil crawlUtil = new ContinuousCrawlTestFetchRetry(); + // keep going for long, to "provoke" a retry counter overflow + try { + if (!crawlUtil.run(150)) { + fail("fetch_retry did not result in a db_gone if retry counter > maxRetries (NUTCH-578)"); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * NUTCH-1564 AdaptiveFetchSchedule: sync_delta forces immediate re-fetch for + * documents not modified + *

+ * Problem: documents not modified for a longer time are fetched in every + * cycle because of an error in the SYNC_DELTA calculation of + * {@link AdaptiveFetchSchedule}.
+ * The next fetch time should always be in the future, never in the past. + *

+ */ + @Test + public void testAdaptiveFetchScheduleSyncDelta() { + LOG.info("NUTCH-1564 test SYNC_DELTA calculation of AdaptiveFetchSchedule"); + Context context = CrawlDBTestUtil.createContext(); + Configuration conf = context.getConfiguration(); + conf.setLong("db.fetch.interval.default", 172800); // 2 days + conf.setLong("db.fetch.schedule.adaptive.min_interval", 86400); // 1 day + conf.setLong("db.fetch.schedule.adaptive.max_interval", 604800); // 7 days + conf.setLong("db.fetch.interval.max", 604800); // 7 days + conf.set("db.fetch.schedule.class", + "org.apache.nutch.crawl.AdaptiveFetchSchedule"); + ContinuousCrawlTestUtil crawlUtil = new CrawlTestFetchScheduleNotModifiedFetchTime( + context); + crawlUtil.setInterval(FetchSchedule.SECONDS_PER_DAY / 3); + try { + if (!crawlUtil.run(100)) { + fail("failed: sync_delta calculation with AdaptiveFetchSchedule"); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + private class CrawlTestFetchNotModified extends ContinuousCrawlTestUtil { /** time of the current fetch */ protected long currFetchTime; @@ -451,7 +596,7 @@ protected CrawlDatum fetch(CrawlDatum datum, long currentTime) { } } - protected class CrawlTestFetchNotModifiedHttp304 extends + private class CrawlTestFetchNotModifiedHttp304 extends CrawlTestFetchNotModified { CrawlTestFetchNotModifiedHttp304(Context context) { @@ -487,79 +632,6 @@ protected CrawlDatum fetch(CrawlDatum datum, long currentTime) { } } - /** - * NUTCH-1245: a fetch_gone should always result in a db_gone. - *

- * Even in a long-running continuous crawl, when a gone page is re-fetched - * several times over time. - *

- */ - @Test - public void testCrawlDbReducerPageGoneSchedule1() { - LOG.info("NUTCH-1245: test long running continuous crawl"); - ContinuousCrawlTestUtil crawlUtil = new ContinuousCrawlTestUtil( - STATUS_FETCH_GONE, STATUS_DB_GONE); - try { - if (!crawlUtil.run(20)) { - fail("fetch_gone did not result in a db_gone (NUTCH-1245)"); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - - /** - * NUTCH-1245: a fetch_gone should always result in a db_gone. - *

- * As some kind of misconfiguration set db.fetch.interval.default to a value - * > (fetchIntervalMax * 1.5). - *

- */ - @Test - public void testCrawlDbReducerPageGoneSchedule2() { - LOG.info("NUTCH-1245 (misconfiguration): test with db.fetch.interval.default > (1.5 * db.fetch.interval.max)"); - Context context = CrawlDBTestUtil.createContext(); - Configuration conf = context.getConfiguration(); - int fetchIntervalMax = conf.getInt("db.fetch.interval.max", 0); - conf.setInt("db.fetch.interval.default", 3 + (int) (fetchIntervalMax * 1.5)); - ContinuousCrawlTestUtil crawlUtil = new ContinuousCrawlTestUtil(context, - STATUS_FETCH_GONE, STATUS_DB_GONE); - try { - if (!crawlUtil.run(0)) { - fail("fetch_gone did not result in a db_gone (NUTCH-1245)"); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - - /** - * Test whether signatures are reset for "content-less" states (gone, - * redirect, etc.): otherwise, if this state is temporary and the document - * appears again with the old content, it may get marked as not_modified in - * CrawlDb just after the redirect state. In this case we cannot expect - * content in segments. Cf. NUTCH-1422: reset signature for redirects. - */ - // TODO: can only test if solution is done in CrawlDbReducer - @Test - public void testSignatureReset() { - LOG.info("NUTCH-1422 must reset signature for redirects and similar states"); - Context context = CrawlDBTestUtil.createContext(); - Configuration conf = context.getConfiguration(); - for (String sched : schedules) { - LOG.info("Testing reset signature with {}", sched); - conf.set("db.fetch.schedule.class", "org.apache.nutch.crawl." + sched); - ContinuousCrawlTestUtil crawlUtil = new CrawlTestSignatureReset(context); - try { - if (!crawlUtil.run(20)) { - fail("failed: signature not reset"); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - } - private class CrawlTestSignatureReset extends ContinuousCrawlTestUtil { byte[][] noContentStates = { { STATUS_FETCH_GONE, STATUS_DB_GONE }, @@ -607,4 +679,130 @@ protected boolean check(CrawlDatum result) { } + + private class ContinuousCrawlTestFetchRetry extends ContinuousCrawlTestUtil { + + private int retryMax = 3; + private int totalRetries = 0; + + ContinuousCrawlTestFetchRetry() { + super(); + fetchStatus = STATUS_FETCH_RETRY; + retryMax = context.getConfiguration().getInt("db.fetch.retry.max", retryMax); + } + + @Override + protected CrawlDatum fetch(CrawlDatum datum, long currentTime) { + datum.setStatus(fetchStatus); + datum.setFetchTime(currentTime); + totalRetries++; + return datum; + } + + @Override + protected boolean check(CrawlDatum result) { + if (result.getRetriesSinceFetch() > retryMax) { + LOG.warn("Retry counter > db.fetch.retry.max: {}", result); + } else if (result.getRetriesSinceFetch() == Byte.MAX_VALUE) { + LOG.warn("Retry counter max. value reached (overflow imminent): {}", result); + } else if (result.getRetriesSinceFetch() < 0) { + LOG.error("Retry counter overflow: {}", result); + return false; + } + // use retry counter bound to this class (totalRetries) + // instead of result.getRetriesSinceFetch() because the retry counter + // in CrawlDatum could be reset (eg. NUTCH-578_v5.patch) + if (totalRetries < retryMax) { + if (result.getStatus() == STATUS_DB_UNFETCHED) { + LOG.info("ok: {}", result); + result.getRetriesSinceFetch(); + return true; + } + } else { + if (result.getStatus() == STATUS_DB_GONE) { + LOG.info("ok: {}", result); + return true; + } + } + LOG.warn("wrong: {}", result); + return false; + } + + } + + private class CrawlTestFetchScheduleNotModifiedFetchTime extends + CrawlTestFetchNotModified { + + // time of current fetch + private long fetchTime; + + private long minInterval; + private long maxInterval; + + CrawlTestFetchScheduleNotModifiedFetchTime(Context context) { + super(context); + Configuration conf = context.getConfiguration(); + minInterval = conf.getLong("db.fetch.schedule.adaptive.min_interval", + 86400); // 1 day + maxInterval = conf.getLong("db.fetch.schedule.adaptive.max_interval", + 604800); // 7 days + if (conf.getLong("db.fetch.interval.max", 604800) < maxInterval) { + maxInterval = conf.getLong("db.fetch.interval.max", 604800); + } + } + + @Override + protected CrawlDatum fetch(CrawlDatum datum, long currentTime) { + // remember time of fetching + fetchTime = currentTime; + return super.fetch(datum, currentTime); + } + + @Override + protected boolean check(CrawlDatum result) { + if (result.getStatus() == STATUS_DB_NOTMODIFIED) { + // check only status notmodified here + long secondsUntilNextFetch = (result.getFetchTime() - fetchTime) / 1000L; + if (secondsUntilNextFetch < -1) { + // next fetch time is in the past (more than one second) + LOG.error("Next fetch time is in the past: {}", result); + return false; + } + if (secondsUntilNextFetch < 60) { + // next fetch time is in less than one minute + // (critical: Nutch can hardly be so fast) + LOG.error("Less then one minute until next fetch: {}", result); + } + // Next fetch time should be within min. and max. (tolerance: 60 sec.) + if (secondsUntilNextFetch + 60 < minInterval + || secondsUntilNextFetch - 60 > maxInterval) { + LOG.error( + "Interval until next fetch time ({}) is not within min. and max. interval: {}", + TimingUtil.elapsedTime(fetchTime, result.getFetchTime()), result); + // TODO: is this a failure? + } + } + return true; + } + + } + + private class CrawlTestParserFailure extends ContinuousCrawlTestUtil { + + public CrawlTestParserFailure(Context context, byte fetchStatus, + byte expectedDbStatus) { + super(context, fetchStatus, expectedDbStatus); + } + + @Override + protected List parse(CrawlDatum fetchDatum) { + List parseDatums = new ArrayList(0); + if (fetchDatum.getStatus() == CrawlDatum.STATUS_FETCH_SUCCESS) { + // fail parsing everything + parseDatums.add(new CrawlDatum(STATUS_PARSE_FAILED, 0)); + } + return parseDatums; + } + } + } diff --git a/src/test/org/apache/nutch/crawl/TestCrawlDbStatesExtended.java b/src/test/org/apache/nutch/crawl/TestCrawlDbStatesExtended.java deleted file mode 100644 index 2e6ea55af1..0000000000 --- a/src/test/org/apache/nutch/crawl/TestCrawlDbStatesExtended.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nutch.crawl; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapreduce.Reducer.Context; -import org.apache.nutch.util.TimingUtil; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; - -import static org.apache.nutch.crawl.CrawlDatum.*; -import static org.junit.jupiter.api.Assertions.fail; - -public class TestCrawlDbStatesExtended extends TestCrawlDbStates { - - private static final Logger LOG = LoggerFactory - .getLogger(MethodHandles.lookup().lookupClass()); - - /** - * NUTCH-578: a fetch_retry should result in a db_gone if db.fetch.retry.max - * is reached. Retry counter has to be reset appropriately. - */ - @Test - public void testCrawlDbReducerPageRetrySchedule() { - LOG.info("NUTCH-578: test long running continuous crawl with fetch_retry"); - ContinuousCrawlTestUtil crawlUtil = new ContinuousCrawlTestFetchRetry(); - // keep going for long, to "provoke" a retry counter overflow - try { - if (!crawlUtil.run(150)) { - fail("fetch_retry did not result in a db_gone if retry counter > maxRetries (NUTCH-578)"); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - - private class ContinuousCrawlTestFetchRetry extends ContinuousCrawlTestUtil { - - private int retryMax = 3; - private int totalRetries = 0; - - ContinuousCrawlTestFetchRetry() { - super(); - fetchStatus = STATUS_FETCH_RETRY; - retryMax = context.getConfiguration().getInt("db.fetch.retry.max", retryMax); - } - - @Override - protected CrawlDatum fetch(CrawlDatum datum, long currentTime) { - datum.setStatus(fetchStatus); - datum.setFetchTime(currentTime); - totalRetries++; - return datum; - } - - @Override - protected boolean check(CrawlDatum result) { - if (result.getRetriesSinceFetch() > retryMax) { - LOG.warn("Retry counter > db.fetch.retry.max: {}", result); - } else if (result.getRetriesSinceFetch() == Byte.MAX_VALUE) { - LOG.warn("Retry counter max. value reached (overflow imminent): {}", result); - } else if (result.getRetriesSinceFetch() < 0) { - LOG.error("Retry counter overflow: {}", result); - return false; - } - // use retry counter bound to this class (totalRetries) - // instead of result.getRetriesSinceFetch() because the retry counter - // in CrawlDatum could be reset (eg. NUTCH-578_v5.patch) - if (totalRetries < retryMax) { - if (result.getStatus() == STATUS_DB_UNFETCHED) { - LOG.info("ok: {}", result); - result.getRetriesSinceFetch(); - return true; - } - } else { - if (result.getStatus() == STATUS_DB_GONE) { - LOG.info("ok: {}", result); - return true; - } - } - LOG.warn("wrong: {}", result); - return false; - } - - } - - /** - * NUTCH-1564 AdaptiveFetchSchedule: sync_delta forces immediate re-fetch for - * documents not modified - *

- * Problem: documents not modified for a longer time are fetched in every - * cycle because of an error in the SYNC_DELTA calculation of - * {@link AdaptiveFetchSchedule}.
- * The next fetch time should always be in the future, never in the past. - *

- */ - @Test - public void testAdaptiveFetchScheduleSyncDelta() { - LOG.info("NUTCH-1564 test SYNC_DELTA calculation of AdaptiveFetchSchedule"); - Context context = CrawlDBTestUtil.createContext(); - Configuration conf = context.getConfiguration(); - conf.setLong("db.fetch.interval.default", 172800); // 2 days - conf.setLong("db.fetch.schedule.adaptive.min_interval", 86400); // 1 day - conf.setLong("db.fetch.schedule.adaptive.max_interval", 604800); // 7 days - conf.setLong("db.fetch.interval.max", 604800); // 7 days - conf.set("db.fetch.schedule.class", - "org.apache.nutch.crawl.AdaptiveFetchSchedule"); - ContinuousCrawlTestUtil crawlUtil = new CrawlTestFetchScheduleNotModifiedFetchTime( - context); - crawlUtil.setInterval(FetchSchedule.SECONDS_PER_DAY / 3); - try { - if (!crawlUtil.run(100)) { - fail("failed: sync_delta calculation with AdaptiveFetchSchedule"); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - - private class CrawlTestFetchScheduleNotModifiedFetchTime extends - CrawlTestFetchNotModified { - - // time of current fetch - private long fetchTime; - - private long minInterval; - private long maxInterval; - - CrawlTestFetchScheduleNotModifiedFetchTime(Context context) { - super(context); - Configuration conf = context.getConfiguration(); - minInterval = conf.getLong("db.fetch.schedule.adaptive.min_interval", - 86400); // 1 day - maxInterval = conf.getLong("db.fetch.schedule.adaptive.max_interval", - 604800); // 7 days - if (conf.getLong("db.fetch.interval.max", 604800) < maxInterval) { - maxInterval = conf.getLong("db.fetch.interval.max", 604800); - } - } - - @Override - protected CrawlDatum fetch(CrawlDatum datum, long currentTime) { - // remember time of fetching - fetchTime = currentTime; - return super.fetch(datum, currentTime); - } - - @Override - protected boolean check(CrawlDatum result) { - if (result.getStatus() == STATUS_DB_NOTMODIFIED) { - // check only status notmodified here - long secondsUntilNextFetch = (result.getFetchTime() - fetchTime) / 1000L; - if (secondsUntilNextFetch < -1) { - // next fetch time is in the past (more than one second) - LOG.error("Next fetch time is in the past: {}", result); - return false; - } - if (secondsUntilNextFetch < 60) { - // next fetch time is in less than one minute - // (critical: Nutch can hardly be so fast) - LOG.error("Less then one minute until next fetch: {}", result); - } - // Next fetch time should be within min. and max. (tolerance: 60 sec.) - if (secondsUntilNextFetch + 60 < minInterval - || secondsUntilNextFetch - 60 > maxInterval) { - LOG.error( - "Interval until next fetch time ({}) is not within min. and max. interval: {}", - TimingUtil.elapsedTime(fetchTime, result.getFetchTime()), result); - // TODO: is this a failure? - } - } - return true; - } - - } - -} diff --git a/src/test/org/apache/nutch/indexer/TestIndexerMapReduce.java b/src/test/org/apache/nutch/indexer/TestIndexerMapReduce.java index 81dd192063..9b21e9083c 100644 --- a/src/test/org/apache/nutch/indexer/TestIndexerMapReduce.java +++ b/src/test/org/apache/nutch/indexer/TestIndexerMapReduce.java @@ -26,6 +26,7 @@ import org.apache.nutch.metadata.Nutch; import org.apache.nutch.parse.Outlink; import org.apache.nutch.parse.ParseData; +import org.apache.nutch.parse.ParseSegment; import org.apache.nutch.parse.ParseStatus; import org.apache.nutch.parse.ParseText; import org.apache.nutch.protocol.Content; @@ -46,6 +47,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; /** Test {@link IndexerMapReduce} */ public class TestIndexerMapReduce { @@ -84,6 +86,8 @@ public class TestIndexerMapReduce { public static ParseText parseText = new ParseText("Test"); public static ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS, "Test", new Outlink[] {}, htmlMeta); + public static ParseData failedParseData = new ParseData(ParseStatus.STATUS_FAILURE, + "Test", new Outlink[] {}, htmlMeta); public static CrawlDatum crawlDatumDbFetched = new CrawlDatum( CrawlDatum.STATUS_DB_FETCHED, 60 * 60 * 24); public static CrawlDatum crawlDatumFetchSuccess = new CrawlDatum( @@ -93,6 +97,37 @@ public class TestIndexerMapReduce { private Configuration configuration; + @Test + public void testDeleteParseFailure() { + configuration = NutchConfiguration.create(); + configuration.setBoolean(ParseSegment.DELETE_FAILED_PARSE, true); + + // unrelated issue with "index.jexl.filter", don't use all plugins. Ref: src/test/nutch-site.xml + configuration.set("plugin.includes", "protocol-http|urlfilter-regex|parse-(html|tika)|index-(basic|anchor)|indexer-csv|scoring-opic|urlnormalizer-(pass|regex|basic)"); + + for(int i=0; i<5; i++) { + boolean failParse = i % 2 == 0; + ParseData data = null; + + Content content = new Content(testUrl, testUrl, + testHtmlDoc.getBytes(StandardCharsets.UTF_8), htmlContentType, htmlMeta, + configuration); + + if(failParse) { + data = failedParseData; + } else { + data = parseData; + } + + NutchDocument doc = runIndexer(crawlDatumDbFetched, + crawlDatumFetchSuccess, parseText, data, content); + if(failParse) { + assertNull(doc, "NutchDocument should not be indexed"); + } else { + assertNotNull(doc, "No NutchDocument indexed"); + } + } + } /** * Test indexing of base64-encoded binary content. From 5b5fdced52eb163dd7416d02e0e306666f657ef7 Mon Sep 17 00:00:00 2001 From: Isabelle Giguere Date: Tue, 24 Feb 2026 16:35:32 -0500 Subject: [PATCH 2/8] NUTCH-1732: allow deleting non-parseable documents Fix IndexerMapReduce. Improve test for failed parsing in TestCrawlDbStates and TestIndexerMapReduce. TestFetchWithPArseFailure "should" test the fetcher, but it strangely does not fully run most of the time. Disabled while thinking of a solution. --- conf/nutch-default.xml | 10 +- .../org/apache/nutch/crawl/CrawlDatum.java | 12 +- .../apache/nutch/crawl/CrawlDbReducer.java | 4 +- .../org/apache/nutch/fetcher/Fetcher.java | 8 +- .../apache/nutch/fetcher/FetcherThread.java | 13 +- .../nutch/indexer/IndexerMapReduce.java | 4 +- src/java/org/apache/nutch/metadata/Nutch.java | 147 +++++----- .../org/apache/nutch/parse/ParseSegment.java | 1 - .../apache/nutch/crawl/TestCrawlDbStates.java | 53 +++- .../fetcher/TestFetchWithParseFailures.java | 252 ++++++++++++++++++ .../nutch/indexer/TestIndexerMapReduce.java | 32 ++- .../fetch-parse-failure/index.html | 25 ++ .../fetch-parse-failure/robots.txt | 16 ++ .../fetch-parse-failure/test.html | 25 ++ 14 files changed, 498 insertions(+), 104 deletions(-) create mode 100644 src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java create mode 100644 src/testresources/fetch-parse-failure/index.html create mode 100644 src/testresources/fetch-parse-failure/robots.txt create mode 100644 src/testresources/fetch-parse-failure/test.html diff --git a/conf/nutch-default.xml b/conf/nutch-default.xml index 1d442a9bc7..42fc2ad6b7 100644 --- a/conf/nutch-default.xml +++ b/conf/nutch-default.xml @@ -1503,6 +1503,14 @@ + + indexer.delete + false + Whether the indexer will delete documents that are gone. Gone pages include redirects and duplicates. + See also: 'link.delete.gone'. + + + indexer.indexwriters.file index-writers.xml @@ -1795,7 +1803,7 @@ CAUTION: Set the parser.timeout to -1 or a bigger value than 30, when using this - parser.delete.failed + parser.delete.failed.parse false Boolean value for whether we should delete a page from the index when parsing the page fails. By default this property is deactivated, because it will delete an existing page from the index, where a diff --git a/src/java/org/apache/nutch/crawl/CrawlDatum.java b/src/java/org/apache/nutch/crawl/CrawlDatum.java index 68b125064d..14f134089e 100644 --- a/src/java/org/apache/nutch/crawl/CrawlDatum.java +++ b/src/java/org/apache/nutch/crawl/CrawlDatum.java @@ -150,16 +150,14 @@ public class CrawlDatum implements WritableComparable, Cloneable { private long modifiedTime; private org.apache.hadoop.io.MapWritable metaData; + /** Validate DB Status (ref.: CrawlDbReducer, IndexerReducer) */ public static boolean hasDbStatus(CrawlDatum datum) { - if (datum.status <= STATUS_DB_MAX) - return true; - return false; + return (datum.status <= STATUS_DB_MAX) || CrawlDatum.STATUS_DB_PARSE_FAILED == datum.getStatus(); } - + /** Validate Fetch Status (ref.: CrawlDbReducer, IndexerReducer, SegmentMergerReducer) */ public static boolean hasFetchStatus(CrawlDatum datum) { - if (datum.status > STATUS_DB_MAX && datum.status <= STATUS_FETCH_MAX) - return true; - return false; + return (datum.status > STATUS_DB_MAX && datum.status <= STATUS_FETCH_MAX) + || CrawlDatum.STATUS_PARSE_FAILED == datum.getStatus(); } public CrawlDatum() { diff --git a/src/java/org/apache/nutch/crawl/CrawlDbReducer.java b/src/java/org/apache/nutch/crawl/CrawlDbReducer.java index d2b3dfbead..cd553af174 100644 --- a/src/java/org/apache/nutch/crawl/CrawlDbReducer.java +++ b/src/java/org/apache/nutch/crawl/CrawlDbReducer.java @@ -115,9 +115,7 @@ public void reduce(Text key, Iterable values, continue; } - // temporarily handle parse_failed as fetched (it was fetched!) - if (CrawlDatum.hasFetchStatus(datum) - || datum.getStatus() == CrawlDatum.STATUS_PARSE_FAILED) { + if (CrawlDatum.hasFetchStatus(datum)) { if (!fetchSet) { if (multiple) { fetch.set(datum); diff --git a/src/java/org/apache/nutch/fetcher/Fetcher.java b/src/java/org/apache/nutch/fetcher/Fetcher.java index 029d95ff7c..712c7a711e 100644 --- a/src/java/org/apache/nutch/fetcher/Fetcher.java +++ b/src/java/org/apache/nutch/fetcher/Fetcher.java @@ -577,11 +577,11 @@ public void fetch(Path segment, int threads) throws IOException, } catch (InterruptedException | ClassNotFoundException e) { LOG.error(StringUtils.stringifyException(e)); throw e; + } finally { + stopWatch.stop(); + LOG.info("Fetcher: finished, elapsed: {} ms", stopWatch.getTime( + TimeUnit.MILLISECONDS)); } - - stopWatch.stop(); - LOG.info("Fetcher: finished, elapsed: {} ms", stopWatch.getTime( - TimeUnit.MILLISECONDS)); } /** diff --git a/src/java/org/apache/nutch/fetcher/FetcherThread.java b/src/java/org/apache/nutch/fetcher/FetcherThread.java index c0462ed848..8595f7e767 100644 --- a/src/java/org/apache/nutch/fetcher/FetcherThread.java +++ b/src/java/org/apache/nutch/fetcher/FetcherThread.java @@ -187,7 +187,7 @@ public FetcherThread(Configuration conf, AtomicInteger activeThreads, FetchItemQ this.scfilters = new ScoringFilters(conf); this.parseUtil = new ParseUtil(conf); this.skipTruncated = conf.getBoolean(ParseSegment.SKIP_TRUNCATED, true); - this.deleteFailedParse = conf.getBoolean(ParseSegment.DELETE_FAILED_PARSE, false); + this.deleteFailedParse = conf.getBoolean(Nutch.DELETE_FAILED_PARSE, false); this.signatureWithoutParsing = conf.getBoolean("fetcher.signature", false); this.protocolFactory = new ProtocolFactory(conf); this.normalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_FETCHER); @@ -228,7 +228,7 @@ public FetcherThread(Configuration conf, AtomicInteger activeThreads, FetchItemQ .getInt("http.robots.503.defer.visits.retries", 3); } - if((activatePublisher=conf.getBoolean("fetcher.publisher", false))) + if ((activatePublisher = conf.getBoolean("fetcher.publisher", false))) this.publisher = new FetcherThreadPublisher(conf); queueMode = conf.get("fetcher.queue.mode", @@ -466,7 +466,7 @@ public void run() { case ProtocolStatus.SUCCESS: // got a page pstatus = output(fit.url, fit.datum, content, status, CrawlDatum.STATUS_FETCH_SUCCESS, fit.outlinkDepth); - updateStatus(content.getContent().length); + updateStatus(content.getContent() != null ? content.getContent().length : 0); if (pstatus != null && pstatus.isSuccess() && pstatus.getMinorCode() == ParseStatus.SUCCESS_REDIRECT) { String newUrl = pstatus.getMessage(); @@ -478,6 +478,8 @@ public void run() { fit = queueRedirect(redirUrl, fit); } } + + // Got a ParseStatus, but isFailed, so call output() again to set set CrawlDatum.STATUS_PARSE_FAILED if (pstatus != null && pstatus.isFailed() && deleteFailedParse) { output(fit.url, fit.datum, null, status, CrawlDatum.STATUS_PARSE_FAILED); } @@ -758,6 +760,11 @@ private ParseStatus output(Text key, CrawlDatum datum, Content content, .calculate(content, new ParseStatus().getEmptyParse(conf)); datum.setSignature(signature); } + + if (parseResult == null && parsing && deleteFailedParse) { + datum.setStatus(CrawlDatum.STATUS_PARSE_FAILED); + status = CrawlDatum.STATUS_PARSE_FAILED; + } } /* diff --git a/src/java/org/apache/nutch/indexer/IndexerMapReduce.java b/src/java/org/apache/nutch/indexer/IndexerMapReduce.java index 7e032a8dd8..2d34d863dc 100644 --- a/src/java/org/apache/nutch/indexer/IndexerMapReduce.java +++ b/src/java/org/apache/nutch/indexer/IndexerMapReduce.java @@ -206,6 +206,7 @@ public static class IndexerReducer extends private boolean delete = false; private boolean deleteRobotsNoIndex = false; private boolean deleteSkippedByIndexingFilter = false; + private boolean deleteFailedParse = false; private boolean base64 = false; private IndexingFilters filters; private ScoringFilters scfilters; @@ -245,6 +246,7 @@ public void setup(Reducer.Context c false); deleteSkippedByIndexingFilter = conf.getBoolean(INDEXER_DELETE_SKIPPED, false); + deleteFailedParse = conf.getBoolean(Nutch.DELETE_FAILED_PARSE, false); skip = conf.getBoolean(INDEXER_SKIP_NOTMODIFIED, false); base64 = conf.getBoolean(INDEXER_BINARY_AS_BASE64, false); @@ -358,7 +360,7 @@ public void reduce(Text key, Iterable values, } // Whether to delete pages where parsing failed - if (delete && fetchDatum != null) { + if (deleteFailedParse && fetchDatum != null) { if (fetchDatum.getStatus() == CrawlDatum.STATUS_PARSE_FAILED || dbDatum != null && dbDatum.getStatus() == CrawlDatum.STATUS_DB_PARSE_FAILED) { deletedFailedParseCounter.increment(1); diff --git a/src/java/org/apache/nutch/metadata/Nutch.java b/src/java/org/apache/nutch/metadata/Nutch.java index 0cfb26369b..8724ccf7ac 100644 --- a/src/java/org/apache/nutch/metadata/Nutch.java +++ b/src/java/org/apache/nutch/metadata/Nutch.java @@ -26,31 +26,31 @@ */ public interface Nutch { - public static final String ORIGINAL_CHAR_ENCODING = "OriginalCharEncoding"; + public static final String ORIGINAL_CHAR_ENCODING = "OriginalCharEncoding"; - public static final String CHAR_ENCODING_FOR_CONVERSION = "CharEncodingForConversion"; + public static final String CHAR_ENCODING_FOR_CONVERSION = "CharEncodingForConversion"; - public static final String SIGNATURE_KEY = "nutch.content.digest"; + public static final String SIGNATURE_KEY = "nutch.content.digest"; - public static final String SEGMENT_NAME_KEY = "nutch.segment.name"; + public static final String SEGMENT_NAME_KEY = "nutch.segment.name"; - public static final String SCORE_KEY = "nutch.crawl.score"; + public static final String SCORE_KEY = "nutch.crawl.score"; - public static final String GENERATE_TIME_KEY = "_ngt_"; + public static final String GENERATE_TIME_KEY = "_ngt_"; - public static final Text WRITABLE_GENERATE_TIME_KEY = new Text( - GENERATE_TIME_KEY); + public static final Text WRITABLE_GENERATE_TIME_KEY = new Text( + GENERATE_TIME_KEY); - public static final Text PROTOCOL_STATUS_CODE_KEY = new Text("nutch.protocol.code"); + public static final Text PROTOCOL_STATUS_CODE_KEY = new Text("nutch.protocol.code"); - public static final String PROTO_STATUS_KEY = "_pst_"; + public static final String PROTO_STATUS_KEY = "_pst_"; - public static final Text WRITABLE_PROTO_STATUS_KEY = new Text( - PROTO_STATUS_KEY); + public static final Text WRITABLE_PROTO_STATUS_KEY = new Text( + PROTO_STATUS_KEY); - public static final String FETCH_TIME_KEY = "_ftk_"; + public static final String FETCH_TIME_KEY = "_ftk_"; - public static final String FETCH_STATUS_KEY = "_fst_"; + public static final String FETCH_STATUS_KEY = "_fst_"; /** * Name to store the robots @@ -58,60 +58,67 @@ public interface Nutch { */ public static final String ROBOTS_METATAG = "robots"; - /** - * Sites may request that search engines don't provide access to cached - * documents. - */ - public static final String CACHING_FORBIDDEN_KEY = "caching.forbidden"; - - /** Show both original forbidden content and summaries (default). */ - public static final String CACHING_FORBIDDEN_NONE = "none"; - - /** Don't show either original forbidden content or summaries. */ - public static final String CACHING_FORBIDDEN_ALL = "all"; - - /** Don't show original forbidden content, but show summaries. */ - public static final String CACHING_FORBIDDEN_CONTENT = "content"; - - public static final String REPR_URL_KEY = "_repr_"; - - public static final Text WRITABLE_REPR_URL_KEY = new Text(REPR_URL_KEY); - - /** Used by AdaptiveFetchSchedule to maintain custom fetch interval */ - public static final String FIXED_INTERVAL_KEY = "fixedInterval"; - - public static final Text WRITABLE_FIXED_INTERVAL_KEY = new Text( - FIXED_INTERVAL_KEY); - - /** For progress of job. Used by the Nutch REST service */ - public static final String STAT_PROGRESS = "progress"; - /**Used by Nutch REST service */ - public static final String CRAWL_ID_KEY = "storage.crawl.id"; - /** Argument key to specify location of the seed url dir for the REST endpoints **/ - public static final String ARG_SEEDDIR = "url_dir"; - /** Argument key to specify name of a seed list for the REST endpoints **/ - public static final String ARG_SEEDNAME = "seedName"; - /** Argument key to specify the location of crawldb for the REST endpoints **/ - public static final String ARG_CRAWLDB = "crawldb"; - /** Argument key to specify the location of linkdb for the REST endpoints **/ - public static final String ARG_LINKDB = "linkdb"; - /** Name of the key used in the Result Map sent back by the REST endpoint **/ - public static final String VAL_RESULT = "result"; - /** Argument key to specify the location of a directory of segments for the REST endpoints. - * Similar to the -dir command in the bin/nutch script **/ - public static final String ARG_SEGMENTDIR = "segment_dir"; - /** Argument key to specify the location of individual segment or list of segments for the REST endpoints. The behavior differs for diffirent endpoints: CrawlDb, LinkDb and Indexing Jobs take list of segments, Fetcher and Parse segment take one segment **/ - public static final String ARG_SEGMENTS = "segment"; - /** Argument key to specify the location of hostdb for the REST endpoints **/ - public static final String ARG_HOSTDB = "hostdb"; - /** Title key in the Pub/Sub event metadata for the title of the parsed page*/ - public static final String FETCH_EVENT_TITLE = "title"; - /** Content-type key in the Pub/Sub event metadata for the content-type of the parsed page*/ - public static final String FETCH_EVENT_CONTENTTYPE = "content-type"; - /** Score key in the Pub/Sub event metadata for the score of the parsed page*/ - public static final String FETCH_EVENT_SCORE = "score"; - /** Fetch time key in the Pub/Sub event metadata for the fetch time of the parsed page*/ - public static final String FETCH_EVENT_FETCHTIME = "fetchTime"; - /** Content-lanueage key in the Pub/Sub event metadata for the content-language of the parsed page*/ - public static final String FETCH_EVENT_CONTENTLANG = "content-language"; + /** + * Sites may request that search engines don't provide access to cached + * documents. + */ + public static final String CACHING_FORBIDDEN_KEY = "caching.forbidden"; + + /** Show both original forbidden content and summaries (default). */ + public static final String CACHING_FORBIDDEN_NONE = "none"; + + /** Don't show either original forbidden content or summaries. */ + public static final String CACHING_FORBIDDEN_ALL = "all"; + + /** Don't show original forbidden content, but show summaries. */ + public static final String CACHING_FORBIDDEN_CONTENT = "content"; + + public static final String REPR_URL_KEY = "_repr_"; + + public static final Text WRITABLE_REPR_URL_KEY = new Text(REPR_URL_KEY); + + /** Used by AdaptiveFetchSchedule to maintain custom fetch interval */ + public static final String FIXED_INTERVAL_KEY = "fixedInterval"; + + public static final Text WRITABLE_FIXED_INTERVAL_KEY = new Text( + FIXED_INTERVAL_KEY); + + /** For progress of job. Used by the Nutch REST service */ + public static final String STAT_PROGRESS = "progress"; + /**Used by Nutch REST service */ + public static final String CRAWL_ID_KEY = "storage.crawl.id"; + /** Argument key to specify location of the seed url dir for the REST endpoints **/ + public static final String ARG_SEEDDIR = "url_dir"; + /** Argument key to specify name of a seed list for the REST endpoints **/ + public static final String ARG_SEEDNAME = "seedName"; + /** Argument key to specify the location of crawldb for the REST endpoints **/ + public static final String ARG_CRAWLDB = "crawldb"; + /** Argument key to specify the location of linkdb for the REST endpoints **/ + public static final String ARG_LINKDB = "linkdb"; + /** Name of the key used in the Result Map sent back by the REST endpoint **/ + public static final String VAL_RESULT = "result"; + /** Argument key to specify the location of a directory of segments for the REST endpoints. + * Similar to the -dir command in the bin/nutch script **/ + public static final String ARG_SEGMENTDIR = "segment_dir"; + /** Argument key to specify the location of individual segment or list of segments for the REST endpoints. The behavior differs for diffirent endpoints: CrawlDb, LinkDb and Indexing Jobs take list of segments, Fetcher and Parse segment take one segment **/ + public static final String ARG_SEGMENTS = "segment"; + /** Argument key to specify the location of hostdb for the REST endpoints **/ + public static final String ARG_HOSTDB = "hostdb"; + /** Title key in the Pub/Sub event metadata for the title of the parsed page*/ + public static final String FETCH_EVENT_TITLE = "title"; + /** Content-type key in the Pub/Sub event metadata for the content-type of the parsed page*/ + public static final String FETCH_EVENT_CONTENTTYPE = "content-type"; + /** Score key in the Pub/Sub event metadata for the score of the parsed page*/ + public static final String FETCH_EVENT_SCORE = "score"; + /** Fetch time key in the Pub/Sub event metadata for the fetch time of the parsed page*/ + public static final String FETCH_EVENT_FETCHTIME = "fetchTime"; + /** Content-lanueage key in the Pub/Sub event metadata for the content-language of the parsed page*/ + public static final String FETCH_EVENT_CONTENTLANG = "content-language"; + + + /** + * Configuration property to delete documents that failed to be parsed. Used + * in Fetcher and Indexer. + */ + public static final String DELETE_FAILED_PARSE = "parser.delete.failed.parse"; } diff --git a/src/java/org/apache/nutch/parse/ParseSegment.java b/src/java/org/apache/nutch/parse/ParseSegment.java index 7d77b19ff1..0b2a6f2290 100644 --- a/src/java/org/apache/nutch/parse/ParseSegment.java +++ b/src/java/org/apache/nutch/parse/ParseSegment.java @@ -67,7 +67,6 @@ public class ParseSegment extends NutchTool implements Tool { .getLogger(MethodHandles.lookup().lookupClass()); public static final String SKIP_TRUNCATED = "parser.skip.truncated"; - public static final String DELETE_FAILED_PARSE = "parser.delete.failed"; public ParseSegment() { this(null); diff --git a/src/test/org/apache/nutch/crawl/TestCrawlDbStates.java b/src/test/org/apache/nutch/crawl/TestCrawlDbStates.java index 636401390b..b69c6efedc 100644 --- a/src/test/org/apache/nutch/crawl/TestCrawlDbStates.java +++ b/src/test/org/apache/nutch/crawl/TestCrawlDbStates.java @@ -21,7 +21,7 @@ import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.Reducer.Context; import org.apache.hadoop.util.StringUtils; -import org.apache.nutch.parse.ParseSegment; +import org.apache.nutch.metadata.Nutch; import org.apache.nutch.scoring.ScoringFilterException; import org.apache.nutch.scoring.ScoringFilters; import org.apache.nutch.util.ReducerContextWrapper; @@ -378,9 +378,8 @@ public void testCrawlDbReducerParseFailed() { LOG.info("NUTCH-1732: allow deleting un-parsable documents"); Context context = CrawlDBTestUtil.createContext(); Configuration conf = context.getConfiguration(); - conf.setBoolean(ParseSegment.DELETE_FAILED_PARSE, true); - CrawlTestParserFailure crawlUtil = new CrawlTestParserFailure(context, - CrawlDatum.STATUS_PARSE_FAILED, CrawlDatum.STATUS_DB_PARSE_FAILED); + conf.setBoolean(Nutch.DELETE_FAILED_PARSE, true); + CrawlTestParserFailure crawlUtil = new CrawlTestParserFailure(context); try { if (!crawlUtil.run(20)) { fail("parse failure did not result in a parse_fail (NUTCH-1732)"); @@ -788,21 +787,57 @@ protected boolean check(CrawlDatum result) { } private class CrawlTestParserFailure extends ContinuousCrawlTestUtil { + + int counter = 0; + boolean failing = false; + + public CrawlTestParserFailure(Context context) { + super(context); + } - public CrawlTestParserFailure(Context context, byte fetchStatus, - byte expectedDbStatus) { - super(context, fetchStatus, expectedDbStatus); + @Override + protected CrawlDatum fetch(CrawlDatum datum, long currentTime) { + counter++; + if(counter % 2 == 0) { + failing = true; + // STATUS_PARSE_FAILED is normally set in FetcherThread.run from a fetch success + // This test is for CrawlDbReducer, after fetch + datum.setStatus(STATUS_PARSE_FAILED); + LOG.info("expect parse failed"); + } else { + failing = false; + datum.setStatus(STATUS_FETCH_SUCCESS); + LOG.info("expect fetch success"); + } + datum.setFetchTime(currentTime); + return datum; } @Override protected List parse(CrawlDatum fetchDatum) { List parseDatums = new ArrayList(0); - if (fetchDatum.getStatus() == CrawlDatum.STATUS_FETCH_SUCCESS) { - // fail parsing everything + if (failing){ + LOG.info("set parse failed"); parseDatums.add(new CrawlDatum(STATUS_PARSE_FAILED, 0)); + } else { + LOG.info("set signature"); + CrawlDatum signed = new CrawlDatum(STATUS_SIGNATURE, 0); + signed.setSignature(getSignature()); + parseDatums.add(signed); } return parseDatums; } + + @Override + protected boolean check(CrawlDatum result) { + if (failing) { + LOG.info("check parse failed"); + return result.getStatus() == STATUS_DB_PARSE_FAILED; + } else { + LOG.info("check fetched"); + return result.getStatus() == STATUS_DB_FETCHED || result.getStatus() == STATUS_DB_NOTMODIFIED; + } + } } } diff --git a/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java b/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java new file mode 100644 index 0000000000..c0c76dd483 --- /dev/null +++ b/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java @@ -0,0 +1,252 @@ +package org.apache.nutch.fetcher; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.BufferedWriter; +import java.lang.invoke.MethodHandles; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.io.Text; +import org.apache.nutch.crawl.AbstractFetchSchedule; +import org.apache.nutch.crawl.CrawlDBTestUtil; +import org.apache.nutch.crawl.Generator; +import org.apache.nutch.crawl.Injector; +import org.apache.nutch.metadata.Nutch; +import org.apache.nutch.protocol.Content; +import org.eclipse.jetty.server.Server; +import org.apache.nutch.crawl.CrawlDatum; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class TestFetchWithParseFailures { + + private static final Logger LOG = LoggerFactory + .getLogger(MethodHandles.lookup().lookupClass()); + + private static final Path TEST_DIR = new Path("build/test/test-fail-parse"); + + private static final String BASE_FOLDER = "build/test/data/fetch-parse-failure"; + private static final String TEST_FILE = "test.html"; + private static final String BKP_FILE = "test.html.bkp"; + private static final String HTML_BAD = "This should not parse as files; + private static java.nio.file.Path baseFolderPath; + + private static ExecutorService executor = Executors.newFixedThreadPool(1); + + + @BeforeEach + public void setUp() throws Exception { + baseFolderPath = java.nio.file.Paths.get(BASE_FOLDER); + java.nio.file.Files.copy(baseFolderPath.resolve(TEST_FILE), baseFolderPath.getParent().resolve(BKP_FILE)); + + files = java.nio.file.Files.list(baseFolderPath).map(p -> p.getFileName().toString()).filter(n -> !n.equals("robots.txt")).collect(Collectors.toList()); + Collections.sort(files); + LOG.info("TEST FILES : " + files); + + conf = CrawlDBTestUtil.createContext().getConfiguration(); + conf.set("plugin.includes", "protocol-http|urlfilter-regex|parse-(html|tika)|index-(basic|anchor)|indexer-csv|scoring-opic|urlnormalizer-(pass|regex|basic)"); + //conf.setLong("db.fetch.interval.default", 1); + //conf.setLong("db.fetch.interval.max", 1); + //conf.setLong("fetcher.timelimit.mins", 2); + //conf.setInt("fetcher.threads.fetch", 1); + //conf.setInt("fetcher.threads.start.delay", 1); + conf.setBoolean("fetcher.parse", true); + conf.setBoolean("fetcher.store.content", true); + conf.setBoolean(Nutch.DELETE_FAILED_PARSE, true); + + fs = FileSystem.get(conf); + fs.delete(TEST_DIR, true); + crawldbPath = new Path(TEST_DIR, "crawldb"); + segmentsPath = new Path(TEST_DIR, "segments"); + urlPath = new Path(TEST_DIR, "urls"); + server = CrawlDBTestUtil.getServer( + conf.getInt("content.server.port", 1234), + BASE_FOLDER); + server.start(); + } + + @AfterEach + public void tearDown() throws Exception { + server.stop(); + for (int i = 0; i < 5; i++) { + if (!server.isStopped()) { + Thread.sleep(1000); + } + } + fs.delete(TEST_DIR, true); + java.nio.file.Files.copy(baseFolderPath.getParent().resolve(BKP_FILE), baseFolderPath.resolve(TEST_FILE), StandardCopyOption.REPLACE_EXISTING); + java.nio.file.Files.delete(baseFolderPath.getParent().resolve(BKP_FILE)); + } + + + @Test + @Disabled("This test does not fully run 99 times out of 100, but still fails at line 246, instead of printing log message or failing on intermediate assertions.") + public void testFetchWithParseFailure() throws Exception { + AtomicInteger fetchCount= new AtomicInteger(0); + AbstractFetchSchedule schedule = new AbstractFetchSchedule(conf) {}; + + // generate seedlist + ArrayList urls = new ArrayList(); + files.forEach(f -> urls.add("http://127.0.0.1:" + server.getURI().getPort() + "/" + f)); + CrawlDBTestUtil.generateSeedList(fs, urlPath, urls); + + // inject + Injector injector = new Injector(conf); + injector.inject(crawldbPath, urlPath); + + // generate + Generator g1 = new Generator(conf); + Path[] generatedSegment1 = g1.generate(crawldbPath, segmentsPath, 1, + Long.MAX_VALUE, Long.MAX_VALUE, false, false, true, 1, null); + Assertions.assertNotNull(generatedSegment1); + + Map args1 = Map.of("segment", generatedSegment1[0]); + // fetch once + LOG.info("1ST FETCH"); + Fetcher fetcher1 = new Fetcher(conf); + + Map result1 = executor.submit(new Callable>(){ + + @Override + public Map call() throws Exception { + try { + return fetcher1.run(args1, "1"); + } catch (Throwable t) { + LOG.error("ERROR!", t); + } finally { + fetchCount.incrementAndGet(); + } + return null; + }}).get(); + Assertions.assertFalse(result1.isEmpty()); + Assertions.assertEquals("0", result1.get(Nutch.VAL_RESULT)); + Assertions.assertEquals(1, fetchCount.get()); + + // verify content + LOG.info("1ST VERIFY CONTENT"); + Map fetchedUrls = new HashMap<>(); + Path data = new Path( + new Path(generatedSegment1[0], Content.DIR_NAME), "part-r-00000/data"); + try(SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(data))){ + do { + Text key = new Text(); + Content value = new Content(); + if (!reader.next(key, value)) + break; + // status is stored in Nutch.FETCH_STATUS_KEY: ref. FetcherThread + String status = value.getMetadata().get(Nutch.FETCH_STATUS_KEY); + fetchedUrls.put(key.toString(), status); + } while (true); + } + assertEquals(urls.size(), fetchedUrls.size()); + LOG.info("1ST FETCHED URLS: {}", fetchedUrls); + + // change content to un-parseable + try(BufferedWriter writer = + java.nio.file.Files.newBufferedWriter(baseFolderPath.resolve(TEST_FILE), StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)){ + writer.write(HTML_BAD); + //writer.write(HTML_GOOD); + writer.flush(); + } + + // force re-fetch and wait a bit + urls.forEach(i -> + schedule.forceRefetch(new Text(i), new CrawlDatum(CrawlDatum.STATUS_DB_UNFETCHED, 1), true)); + long start = System.currentTimeMillis(); + while(System.currentTimeMillis() < start + 1000L); + + // inject and generate again + injector.inject(crawldbPath, urlPath, true, true); + Generator g2 = new Generator(conf); + Path[] generatedSegment2 = g2.generate(crawldbPath, segmentsPath, 1, + Long.MAX_VALUE, Long.MAX_VALUE, false, false, true, 1, null); + Assertions.assertNotNull(generatedSegment2); + + + Map args2 = Map.of("segment", generatedSegment2[0]); + // next fetch should generate parse failure + LOG.info("2ND FETCH"); + Fetcher fetcher2 = new Fetcher(conf); + + Map result2 = executor.submit(new Callable>(){ + + @Override + public Map call() throws Exception { + try { + return fetcher2.run(args2, "2"); + } catch (Throwable t) { + System.out.println("ERROR! " + t.getMessage() + " StackTrace: " + t.getStackTrace()); + LOG.error("ERROR!", t); + } finally { + fetchCount.incrementAndGet(); + } + return null; + }}).get(); + Assertions.assertFalse(result2.isEmpty()); + Assertions.assertEquals("0", result2.get(Nutch.VAL_RESULT)); + Assertions.assertEquals(2, fetchCount.get()); + + /* Even if execution comes to this point, it does not print the next 2 System.out messages, + * but it continues, and fails the validation below. + * Log for the 2nd fetch ends with the QueueFeeder queuing status" */ + + // verify parsed data + LOG.info("2ND VERIFY CONTENT"); + Path newData = new Path( + new Path(generatedSegment2[0], Content.DIR_NAME), "part-r-00000/data"); + try(SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(newData))){ + do { + Text key = new Text(); + Content value = new Content(); + if (!reader.next(key, value)) + break; + // status is stored in Nutch.FETCH_STATUS_KEY: ref. FetcherThread + String status = value.getMetadata().get(Nutch.FETCH_STATUS_KEY); + fetchedUrls.put(key.toString(), status); + } while (true); + } + assertEquals(urls.size(), fetchedUrls.size()); + LOG.info("2ND FETCHED STATUS: {}", fetchedUrls); + + for (Map.Entry entry : fetchedUrls.entrySet()) { + if(entry.getKey().endsWith(TEST_FILE)) { + Assertions.assertEquals(entry.getValue(), ""+CrawlDatum.STATUS_PARSE_FAILED); + } else { + Assertions.assertEquals(entry.getValue(), ""+CrawlDatum.STATUS_FETCH_SUCCESS); + } + } + } +} diff --git a/src/test/org/apache/nutch/indexer/TestIndexerMapReduce.java b/src/test/org/apache/nutch/indexer/TestIndexerMapReduce.java index 9b21e9083c..a5001515d3 100644 --- a/src/test/org/apache/nutch/indexer/TestIndexerMapReduce.java +++ b/src/test/org/apache/nutch/indexer/TestIndexerMapReduce.java @@ -26,7 +26,6 @@ import org.apache.nutch.metadata.Nutch; import org.apache.nutch.parse.Outlink; import org.apache.nutch.parse.ParseData; -import org.apache.nutch.parse.ParseSegment; import org.apache.nutch.parse.ParseStatus; import org.apache.nutch.parse.ParseText; import org.apache.nutch.protocol.Content; @@ -84,14 +83,19 @@ public class TestIndexerMapReduce { htmlMeta.add(Nutch.SIGNATURE_KEY, "123"); } public static ParseText parseText = new ParseText("Test"); + public static ParseText failedParseText = new ParseText("Failed Parse"); public static ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS, "Test", new Outlink[] {}, htmlMeta); public static ParseData failedParseData = new ParseData(ParseStatus.STATUS_FAILURE, - "Test", new Outlink[] {}, htmlMeta); + "Failed Parse", new Outlink[] {}, htmlMeta); public static CrawlDatum crawlDatumDbFetched = new CrawlDatum( CrawlDatum.STATUS_DB_FETCHED, 60 * 60 * 24); public static CrawlDatum crawlDatumFetchSuccess = new CrawlDatum( CrawlDatum.STATUS_FETCH_SUCCESS, 60 * 60 * 24); + public static CrawlDatum crawlDatumDbParseFailed = new CrawlDatum( + CrawlDatum.STATUS_DB_PARSE_FAILED, 60 * 60 * 24); + public static CrawlDatum crawlDatumParseFailed = new CrawlDatum( + CrawlDatum.STATUS_PARSE_FAILED, 60 * 60 * 24); private IndexerMapReduce.IndexerReducer reducer = new IndexerMapReduce.IndexerReducer(); @@ -100,14 +104,21 @@ public class TestIndexerMapReduce { @Test public void testDeleteParseFailure() { configuration = NutchConfiguration.create(); - configuration.setBoolean(ParseSegment.DELETE_FAILED_PARSE, true); + configuration.setBoolean(Nutch.DELETE_FAILED_PARSE, true); // unrelated issue with "index.jexl.filter", don't use all plugins. Ref: src/test/nutch-site.xml configuration.set("plugin.includes", "protocol-http|urlfilter-regex|parse-(html|tika)|index-(basic|anchor)|indexer-csv|scoring-opic|urlnormalizer-(pass|regex|basic)"); + int docToDelete = 0; + int docsToIndex = 0; + int deletedDocs = 0; + int indexedDocs = 0; for(int i=0; i<5; i++) { boolean failParse = i % 2 == 0; ParseData data = null; + ParseText text = null; + CrawlDatum dbStatus = null; + CrawlDatum status = null; Content content = new Content(testUrl, testUrl, testHtmlDoc.getBytes(StandardCharsets.UTF_8), htmlContentType, htmlMeta, @@ -115,17 +126,28 @@ public void testDeleteParseFailure() { if(failParse) { data = failedParseData; + text = failedParseText; + dbStatus = crawlDatumDbParseFailed; + status = crawlDatumParseFailed; + docToDelete++; } else { data = parseData; + text = parseText; + dbStatus = crawlDatumDbFetched; + status = crawlDatumFetchSuccess; + docsToIndex++; } - NutchDocument doc = runIndexer(crawlDatumDbFetched, - crawlDatumFetchSuccess, parseText, data, content); + NutchDocument doc = runIndexer(dbStatus, status, text, data, content); if(failParse) { + deletedDocs++; assertNull(doc, "NutchDocument should not be indexed"); } else { + indexedDocs++; assertNotNull(doc, "No NutchDocument indexed"); } + assertEquals(docToDelete, deletedDocs); + assertEquals(docsToIndex, indexedDocs); } } diff --git a/src/testresources/fetch-parse-failure/index.html b/src/testresources/fetch-parse-failure/index.html new file mode 100644 index 0000000000..8dd11bf16d --- /dev/null +++ b/src/testresources/fetch-parse-failure/index.html @@ -0,0 +1,25 @@ + + + + front page + + +This is the front page. +Test + + \ No newline at end of file diff --git a/src/testresources/fetch-parse-failure/robots.txt b/src/testresources/fetch-parse-failure/robots.txt new file mode 100644 index 0000000000..796ccf7131 --- /dev/null +++ b/src/testresources/fetch-parse-failure/robots.txt @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#User-agent: * +#Allow: / \ No newline at end of file diff --git a/src/testresources/fetch-parse-failure/test.html b/src/testresources/fetch-parse-failure/test.html new file mode 100644 index 0000000000..2151060d30 --- /dev/null +++ b/src/testresources/fetch-parse-failure/test.html @@ -0,0 +1,25 @@ + + + + Test + + +This is a test +home + + \ No newline at end of file From 9cb56a710a4f1d68945ac783ac6d0da125c10695 Mon Sep 17 00:00:00 2001 From: Isabelle Giguere Date: Tue, 24 Feb 2026 17:27:06 -0500 Subject: [PATCH 3/8] NUTCH-1732: allow deleting non-parseable documents Change the type of ExecutorService, to no avail. Fix test conf to have just 1 fetcher thread. --- .../apache/nutch/fetcher/TestFetchWithParseFailures.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java b/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java index c0c76dd483..ca66b53d1c 100644 --- a/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java +++ b/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java @@ -64,7 +64,7 @@ public class TestFetchWithParseFailures { private static List files; private static java.nio.file.Path baseFolderPath; - private static ExecutorService executor = Executors.newFixedThreadPool(1); + private static ExecutorService executor = Executors.newCachedThreadPool(); @BeforeEach @@ -81,8 +81,8 @@ public void setUp() throws Exception { //conf.setLong("db.fetch.interval.default", 1); //conf.setLong("db.fetch.interval.max", 1); //conf.setLong("fetcher.timelimit.mins", 2); - //conf.setInt("fetcher.threads.fetch", 1); //conf.setInt("fetcher.threads.start.delay", 1); + conf.setInt("fetcher.threads.fetch", 1); conf.setBoolean("fetcher.parse", true); conf.setBoolean("fetcher.store.content", true); conf.setBoolean(Nutch.DELETE_FAILED_PARSE, true); @@ -100,6 +100,7 @@ public void setUp() throws Exception { @AfterEach public void tearDown() throws Exception { + executor.shutdown(); server.stop(); for (int i = 0; i < 5; i++) { if (!server.isStopped()) { @@ -113,7 +114,7 @@ public void tearDown() throws Exception { @Test - @Disabled("This test does not fully run 99 times out of 100, but still fails at line 246, instead of printing log message or failing on intermediate assertions.") + //@Disabled("This test does not fully run 99 times out of 100, but still fails at line 246, instead of printing log message or failing on intermediate assertions.") public void testFetchWithParseFailure() throws Exception { AtomicInteger fetchCount= new AtomicInteger(0); AbstractFetchSchedule schedule = new AbstractFetchSchedule(conf) {}; @@ -137,7 +138,6 @@ public void testFetchWithParseFailure() throws Exception { // fetch once LOG.info("1ST FETCH"); Fetcher fetcher1 = new Fetcher(conf); - Map result1 = executor.submit(new Callable>(){ @Override From 60882c8e122d4f67d303cc150441788b1986dec6 Mon Sep 17 00:00:00 2001 From: Isabelle Giguere Date: Mon, 2 Mar 2026 17:40:24 -0500 Subject: [PATCH 4/8] NUTCH-1732: allow deleting non-parseable docs Investigate the faling test, fix the disabled message --- .../fetcher/TestFetchWithParseFailures.java | 51 +++++++++++++------ .../fetch-parse-failure/robots.txt | 4 +- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java b/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java index ca66b53d1c..60f2286cb6 100644 --- a/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java +++ b/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java @@ -2,8 +2,11 @@ import static org.junit.jupiter.api.Assertions.assertEquals; +import java.io.BufferedReader; import java.io.BufferedWriter; +import java.io.InputStream; import java.lang.invoke.MethodHandles; +import java.nio.charset.StandardCharsets; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.util.ArrayList; @@ -11,6 +14,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -29,8 +33,11 @@ import org.apache.nutch.crawl.Injector; import org.apache.nutch.metadata.Nutch; import org.apache.nutch.protocol.Content; +import org.apache.nutch.protocol.Protocol; +import org.apache.nutch.protocol.ProtocolFactory; import org.eclipse.jetty.server.Server; import org.apache.nutch.crawl.CrawlDatum; +import org.apache.nutch.crawl.CrawlDb; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -50,9 +57,7 @@ public class TestFetchWithParseFailures { private static final String BASE_FOLDER = "build/test/data/fetch-parse-failure"; private static final String TEST_FILE = "test.html"; private static final String BKP_FILE = "test.html.bkp"; - private static final String HTML_BAD = "This should not parse as call() throws Exception { Assertions.assertEquals("0", result1.get(Nutch.VAL_RESULT)); Assertions.assertEquals(1, fetchCount.get()); + CrawlDb crawlDb = new CrawlDb(conf); + crawlDb.update(crawldbPath, generatedSegment1, true, true, true, true); + // verify content LOG.info("1ST VERIFY CONTENT"); Map fetchedUrls = new HashMap<>(); @@ -173,14 +182,28 @@ public Map call() throws Exception { } assertEquals(urls.size(), fetchedUrls.size()); LOG.info("1ST FETCHED URLS: {}", fetchedUrls); - + + LOG.info("CHANGE FILE CONTENT: {}", baseFolderPath.resolve(TEST_FILE)); // change content to un-parseable try(BufferedWriter writer = java.nio.file.Files.newBufferedWriter(baseFolderPath.resolve(TEST_FILE), StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)){ - writer.write(HTML_BAD); - //writer.write(HTML_GOOD); + writer.write(badContent); writer.flush(); } +// try(InputStream input = java.nio.file.Files.newInputStream(baseFolderPath.resolve(TEST_FILE), StandardOpenOption.READ)){ +// String content = new String(input.readAllBytes(), StandardCharsets.UTF_8); +// LOG.info("TEST CONTENT IN FILE: {}", content); // makes the log file unreadable +// } + // refresh the server + server.stop(); + server.start(); +// // Validate what is actually served: +// String urlString = "http://127.0.0.1:55000/test.html"; +// Protocol protocol = new ProtocolFactory(conf).getProtocol(urlString); +// Content content = protocol.getProtocolOutput(new Text(urlString), +// new CrawlDatum()).getContent(); +// String contentStr = new String(content.getContent()); +// LOG.info("TEST CONTENT FROM SERVER: {}", contentStr); // makes the log file unreadable // force re-fetch and wait a bit urls.forEach(i -> @@ -195,6 +218,7 @@ public Map call() throws Exception { Long.MAX_VALUE, Long.MAX_VALUE, false, false, true, 1, null); Assertions.assertNotNull(generatedSegment2); + crawlDb.update(crawldbPath, generatedSegment2, true, true, true, true); Map args2 = Map.of("segment", generatedSegment2[0]); // next fetch should generate parse failure @@ -208,7 +232,6 @@ public Map call() throws Exception { try { return fetcher2.run(args2, "2"); } catch (Throwable t) { - System.out.println("ERROR! " + t.getMessage() + " StackTrace: " + t.getStackTrace()); LOG.error("ERROR!", t); } finally { fetchCount.incrementAndGet(); @@ -218,10 +241,6 @@ public Map call() throws Exception { Assertions.assertFalse(result2.isEmpty()); Assertions.assertEquals("0", result2.get(Nutch.VAL_RESULT)); Assertions.assertEquals(2, fetchCount.get()); - - /* Even if execution comes to this point, it does not print the next 2 System.out messages, - * but it continues, and fails the validation below. - * Log for the 2nd fetch ends with the QueueFeeder queuing status" */ // verify parsed data LOG.info("2ND VERIFY CONTENT"); diff --git a/src/testresources/fetch-parse-failure/robots.txt b/src/testresources/fetch-parse-failure/robots.txt index 796ccf7131..fc590f9733 100644 --- a/src/testresources/fetch-parse-failure/robots.txt +++ b/src/testresources/fetch-parse-failure/robots.txt @@ -11,6 +11,4 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. -#User-agent: * -#Allow: / \ No newline at end of file +# limitations under the License. \ No newline at end of file From 38f6152e7a5a2f999bd4916f5c20907b060e4d19 Mon Sep 17 00:00:00 2001 From: Isabelle Giguere Date: Mon, 2 Mar 2026 18:38:23 -0500 Subject: [PATCH 5/8] NUTCH-1732: add resource handler Replace the string content with random bytes injected from a resource handler. No change. Test still fails. --- .../fetcher/TestFetchWithParseFailures.java | 75 ++++++++++--------- 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java b/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java index 60f2286cb6..a2e7b84ae3 100644 --- a/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java +++ b/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java @@ -4,6 +4,7 @@ import java.io.BufferedReader; import java.io.BufferedWriter; +import java.io.IOException; import java.io.InputStream; import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; @@ -22,6 +23,10 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -35,7 +40,9 @@ import org.apache.nutch.protocol.Content; import org.apache.nutch.protocol.Protocol; import org.apache.nutch.protocol.ProtocolFactory; +import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.handler.ResourceHandler; import org.apache.nutch.crawl.CrawlDatum; import org.apache.nutch.crawl.CrawlDb; import org.junit.jupiter.api.AfterEach; @@ -56,8 +63,6 @@ public class TestFetchWithParseFailures { private static final String BASE_FOLDER = "build/test/data/fetch-parse-failure"; private static final String TEST_FILE = "test.html"; - private static final String BKP_FILE = "test.html.bkp"; - private static String badContent; private Configuration conf; private FileSystem fs; @@ -70,23 +75,19 @@ public class TestFetchWithParseFailures { private static java.nio.file.Path baseFolderPath; private static ExecutorService executor = Executors.newCachedThreadPool(); + private static final AtomicInteger FETCH_COUNT = new AtomicInteger(0); @BeforeEach public void setUp() throws Exception { baseFolderPath = java.nio.file.Paths.get(BASE_FOLDER); - java.nio.file.Files.copy(baseFolderPath.resolve(TEST_FILE), baseFolderPath.getParent().resolve(BKP_FILE)); files = java.nio.file.Files.list(baseFolderPath).map(p -> p.getFileName().toString()).filter(n -> !n.equals("robots.txt")).collect(Collectors.toList()); Collections.sort(files); LOG.info("TEST FILES : " + files); - byte[] randomBytes = new byte[256]; - new Random(123).nextBytes(randomBytes); - badContent = new String(randomBytes); - conf = CrawlDBTestUtil.createContext().getConfiguration(); - conf.set("plugin.includes", "protocol-http|urlfilter-regex|parse-(html|tika)|index-(basic|anchor)|indexer-csv|scoring-opic|urlnormalizer-(pass|regex|basic)"); + conf.set("plugin.includes", "protocol-http|urlfilter-regex|parse-html|index-(basic|anchor)|indexer-csv|scoring-opic|urlnormalizer-(pass|regex|basic)"); conf.setInt("fetcher.threads.fetch", 1); conf.setBoolean("fetcher.parse", true); conf.setBoolean("fetcher.store.content", true); @@ -99,7 +100,7 @@ public void setUp() throws Exception { urlPath = new Path(TEST_DIR, "urls"); server = CrawlDBTestUtil.getServer( conf.getInt("content.server.port", 1234), - BASE_FOLDER); + BASE_FOLDER, new ParseFailureResourceHandler()); server.start(); } @@ -113,8 +114,6 @@ public void tearDown() throws Exception { } } fs.delete(TEST_DIR, true); - java.nio.file.Files.copy(baseFolderPath.getParent().resolve(BKP_FILE), baseFolderPath.resolve(TEST_FILE), StandardCopyOption.REPLACE_EXISTING); - java.nio.file.Files.delete(baseFolderPath.getParent().resolve(BKP_FILE)); } @@ -122,7 +121,6 @@ public void tearDown() throws Exception { // Ref: full hadoop.log in logs/hadoop.log @Disabled("The default HTML Parser (Neko) is a lot more tolerant than expected. Even random bytes in an HTML file do not produce a parsing error.") public void testFetchWithParseFailure() throws Exception { - AtomicInteger fetchCount= new AtomicInteger(0); AbstractFetchSchedule schedule = new AbstractFetchSchedule(conf) {}; // generate seedlist @@ -153,13 +151,13 @@ public Map call() throws Exception { } catch (Throwable t) { LOG.error("ERROR!", t); } finally { - fetchCount.incrementAndGet(); + FETCH_COUNT.incrementAndGet(); } return null; }}).get(); Assertions.assertFalse(result1.isEmpty()); Assertions.assertEquals("0", result1.get(Nutch.VAL_RESULT)); - Assertions.assertEquals(1, fetchCount.get()); + Assertions.assertEquals(1, FETCH_COUNT.get()); CrawlDb crawlDb = new CrawlDb(conf); crawlDb.update(crawldbPath, generatedSegment1, true, true, true, true); @@ -182,26 +180,11 @@ public Map call() throws Exception { } assertEquals(urls.size(), fetchedUrls.size()); LOG.info("1ST FETCHED URLS: {}", fetchedUrls); - - LOG.info("CHANGE FILE CONTENT: {}", baseFolderPath.resolve(TEST_FILE)); - // change content to un-parseable - try(BufferedWriter writer = - java.nio.file.Files.newBufferedWriter(baseFolderPath.resolve(TEST_FILE), StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)){ - writer.write(badContent); - writer.flush(); - } -// try(InputStream input = java.nio.file.Files.newInputStream(baseFolderPath.resolve(TEST_FILE), StandardOpenOption.READ)){ -// String content = new String(input.readAllBytes(), StandardCharsets.UTF_8); -// LOG.info("TEST CONTENT IN FILE: {}", content); // makes the log file unreadable -// } - // refresh the server - server.stop(); - server.start(); -// // Validate what is actually served: + +// // Validate what is now served: // String urlString = "http://127.0.0.1:55000/test.html"; // Protocol protocol = new ProtocolFactory(conf).getProtocol(urlString); -// Content content = protocol.getProtocolOutput(new Text(urlString), -// new CrawlDatum()).getContent(); +// Content content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent(); // String contentStr = new String(content.getContent()); // LOG.info("TEST CONTENT FROM SERVER: {}", contentStr); // makes the log file unreadable @@ -234,13 +217,13 @@ public Map call() throws Exception { } catch (Throwable t) { LOG.error("ERROR!", t); } finally { - fetchCount.incrementAndGet(); + FETCH_COUNT.incrementAndGet(); } return null; }}).get(); Assertions.assertFalse(result2.isEmpty()); Assertions.assertEquals("0", result2.get(Nutch.VAL_RESULT)); - Assertions.assertEquals(2, fetchCount.get()); + Assertions.assertEquals(2, FETCH_COUNT.get()); // verify parsed data LOG.info("2ND VERIFY CONTENT"); @@ -268,4 +251,28 @@ public Map call() throws Exception { } } } + + public static class ParseFailureResourceHandler extends ResourceHandler { + + public ParseFailureResourceHandler() { + super(); + } + + @Override + public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) + throws IOException, ServletException { + if(FETCH_COUNT.get() == 1 && target.endsWith(TEST_FILE)) { + LOG.info("FEEDING RANDOM BYTES INTO THE RESPONSE"); + response.setContentType("text/html"); + response.setContentLength(100); // wrong length on purpose + + byte[] randomBytes = new byte[1024]; + new Random(123).nextBytes(randomBytes); + response.getOutputStream().write(randomBytes); + + } else { + super.handle(target, baseRequest, request, response); + } + } + } } From 0a26d3e42067c4dab0260b517f14dc8ed9310170 Mon Sep 17 00:00:00 2001 From: Isabelle Giguere Date: Wed, 4 Mar 2026 17:06:07 -0500 Subject: [PATCH 6/8] NUTCH-1732: unit test still disabled Tried "tagsoup", but parse result is success, just like "neko". --- .../fetcher/TestFetchWithParseFailures.java | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java b/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java index a2e7b84ae3..591fb782b0 100644 --- a/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java +++ b/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java @@ -2,14 +2,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -import java.io.BufferedReader; -import java.io.BufferedWriter; import java.io.IOException; -import java.io.InputStream; import java.lang.invoke.MethodHandles; -import java.nio.charset.StandardCharsets; -import java.nio.file.StandardCopyOption; -import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -19,7 +13,6 @@ import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -38,8 +31,6 @@ import org.apache.nutch.crawl.Injector; import org.apache.nutch.metadata.Nutch; import org.apache.nutch.protocol.Content; -import org.apache.nutch.protocol.Protocol; -import org.apache.nutch.protocol.ProtocolFactory; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.handler.ResourceHandler; @@ -89,6 +80,9 @@ public void setUp() throws Exception { conf = CrawlDBTestUtil.createContext().getConfiguration(); conf.set("plugin.includes", "protocol-http|urlfilter-regex|parse-html|index-(basic|anchor)|indexer-csv|scoring-opic|urlnormalizer-(pass|regex|basic)"); conf.setInt("fetcher.threads.fetch", 1); + conf.setInt("http.content.limit", 1024); + // Default parser.html.impl is 'neko', other option is 'tagsoup'. Both are too tolerant for this test. + conf.set("parser.html.impl", "tagsoup"); conf.setBoolean("fetcher.parse", true); conf.setBoolean("fetcher.store.content", true); conf.setBoolean(Nutch.DELETE_FAILED_PARSE, true); @@ -264,9 +258,9 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques if(FETCH_COUNT.get() == 1 && target.endsWith(TEST_FILE)) { LOG.info("FEEDING RANDOM BYTES INTO THE RESPONSE"); response.setContentType("text/html"); - response.setContentLength(100); // wrong length on purpose + response.setContentLength(2048); - byte[] randomBytes = new byte[1024]; + byte[] randomBytes = new byte[2048]; new Random(123).nextBytes(randomBytes); response.getOutputStream().write(randomBytes); From 81597d45325729f9874b8737765abdf778194b91 Mon Sep 17 00:00:00 2001 From: Isabelle Giguere Date: Fri, 6 Mar 2026 15:55:43 -0500 Subject: [PATCH 7/8] NUTCH-1732: allow deleting un-parseable docs Fix unit test: set "mime.type.magic"=false to force an exception from ParserFactory. Log a message in ParseSegment when skipping un-parseable content. --- .../apache/nutch/fetcher/FetcherThread.java | 14 ++++---- .../nutch/indexer/IndexerMapReduce.java | 3 +- src/java/org/apache/nutch/metadata/Nutch.java | 7 ---- .../org/apache/nutch/parse/ParseSegment.java | 12 +++++++ .../apache/nutch/crawl/CrawlDBTestUtil.java | 29 ++++++++++++++++ .../apache/nutch/crawl/TestCrawlDbStates.java | 4 +-- .../fetcher/TestFetchWithParseFailures.java | 34 +++++++------------ .../nutch/indexer/TestIndexerMapReduce.java | 3 +- 8 files changed, 66 insertions(+), 40 deletions(-) diff --git a/src/java/org/apache/nutch/fetcher/FetcherThread.java b/src/java/org/apache/nutch/fetcher/FetcherThread.java index 8595f7e767..9490d3c851 100644 --- a/src/java/org/apache/nutch/fetcher/FetcherThread.java +++ b/src/java/org/apache/nutch/fetcher/FetcherThread.java @@ -187,7 +187,7 @@ public FetcherThread(Configuration conf, AtomicInteger activeThreads, FetchItemQ this.scfilters = new ScoringFilters(conf); this.parseUtil = new ParseUtil(conf); this.skipTruncated = conf.getBoolean(ParseSegment.SKIP_TRUNCATED, true); - this.deleteFailedParse = conf.getBoolean(Nutch.DELETE_FAILED_PARSE, false); + this.deleteFailedParse = conf.getBoolean(ParseSegment.DELETE_FAILED_PARSE, false); this.signatureWithoutParsing = conf.getBoolean("fetcher.signature", false); this.protocolFactory = new ProtocolFactory(conf); this.normalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_FETCHER); @@ -478,11 +478,6 @@ public void run() { fit = queueRedirect(redirUrl, fit); } } - - // Got a ParseStatus, but isFailed, so call output() again to set set CrawlDatum.STATUS_PARSE_FAILED - if (pstatus != null && pstatus.isFailed() && deleteFailedParse) { - output(fit.url, fit.datum, null, status, CrawlDatum.STATUS_PARSE_FAILED); - } break; case ProtocolStatus.MOVED: // redirect @@ -771,8 +766,7 @@ private ParseStatus output(Text key, CrawlDatum datum, Content content, * Store status code in content So we can read this value during parsing * (as a separate job) and decide to parse or not. */ - content.getMetadata().add(Nutch.FETCH_STATUS_KEY, - Integer.toString(status)); + content.getMetadata().add(Nutch.FETCH_STATUS_KEY, Integer.toString(status)); } try { @@ -790,6 +784,10 @@ private ParseStatus output(Text key, CrawlDatum datum, Content content, LOG.warn("{} {} Error parsing: {}: {}", getName(), Thread.currentThread().getId(), key, parseStatus); parse = parseStatus.getEmptyParse(conf); + if (deleteFailedParse && content != null) { + // forward the failure status in the content + content.getMetadata().add(Nutch.FETCH_STATUS_KEY, Integer.toString(CrawlDatum.STATUS_PARSE_FAILED)); + } } // Calculate page signature. For non-parsing fetchers this will diff --git a/src/java/org/apache/nutch/indexer/IndexerMapReduce.java b/src/java/org/apache/nutch/indexer/IndexerMapReduce.java index 2d34d863dc..14ef3de678 100644 --- a/src/java/org/apache/nutch/indexer/IndexerMapReduce.java +++ b/src/java/org/apache/nutch/indexer/IndexerMapReduce.java @@ -51,6 +51,7 @@ import org.apache.nutch.parse.Parse; import org.apache.nutch.parse.ParseData; import org.apache.nutch.parse.ParseImpl; +import org.apache.nutch.parse.ParseSegment; import org.apache.nutch.parse.ParseText; import org.apache.nutch.protocol.Content; import org.apache.nutch.scoring.ScoringFilterException; @@ -246,7 +247,7 @@ public void setup(Reducer.Context c false); deleteSkippedByIndexingFilter = conf.getBoolean(INDEXER_DELETE_SKIPPED, false); - deleteFailedParse = conf.getBoolean(Nutch.DELETE_FAILED_PARSE, false); + deleteFailedParse = conf.getBoolean(ParseSegment.DELETE_FAILED_PARSE, false); skip = conf.getBoolean(INDEXER_SKIP_NOTMODIFIED, false); base64 = conf.getBoolean(INDEXER_BINARY_AS_BASE64, false); diff --git a/src/java/org/apache/nutch/metadata/Nutch.java b/src/java/org/apache/nutch/metadata/Nutch.java index 8724ccf7ac..fed67ae93a 100644 --- a/src/java/org/apache/nutch/metadata/Nutch.java +++ b/src/java/org/apache/nutch/metadata/Nutch.java @@ -114,11 +114,4 @@ public interface Nutch { public static final String FETCH_EVENT_FETCHTIME = "fetchTime"; /** Content-lanueage key in the Pub/Sub event metadata for the content-language of the parsed page*/ public static final String FETCH_EVENT_CONTENTLANG = "content-language"; - - - /** - * Configuration property to delete documents that failed to be parsed. Used - * in Fetcher and Indexer. - */ - public static final String DELETE_FAILED_PARSE = "parser.delete.failed.parse"; } diff --git a/src/java/org/apache/nutch/parse/ParseSegment.java b/src/java/org/apache/nutch/parse/ParseSegment.java index 0b2a6f2290..315cf37165 100644 --- a/src/java/org/apache/nutch/parse/ParseSegment.java +++ b/src/java/org/apache/nutch/parse/ParseSegment.java @@ -68,6 +68,12 @@ public class ParseSegment extends NutchTool implements Tool { public static final String SKIP_TRUNCATED = "parser.skip.truncated"; + /** + * Configuration property to delete documents that failed to be parsed. Also used + * in Fetcher and Indexer. + */ + public static final String DELETE_FAILED_PARSE = "parser.delete.failed.parse"; + public ParseSegment() { this(null); } @@ -83,6 +89,7 @@ public static class ParseSegmentMapper extends private Text newKey = new Text(); private ScoringFilters scfilters; private boolean skipTruncated; + private boolean deleteFailedParse; private LatencyTracker parseLatencyTracker; private ErrorTracker errorTracker; @@ -91,6 +98,7 @@ public void setup(Mapper, Content, Text, ParseImpl>.Contex Configuration conf = context.getConfiguration(); scfilters = new ScoringFilters(conf); skipTruncated = conf.getBoolean(SKIP_TRUNCATED, true); + deleteFailedParse = conf.getBoolean(DELETE_FAILED_PARSE, false); parseLatencyTracker = new LatencyTracker( NutchMetrics.GROUP_PARSER, NutchMetrics.PARSER_LATENCY); // Initialize error tracker with cached counters @@ -119,6 +127,9 @@ public void map(WritableComparable key, Content content, // no fetch status, skip document LOG.debug("Skipping {} as content has no fetch status", key); return; + } else if(deleteFailedParse && Integer.parseInt(fetchStatus) == CrawlDatum.STATUS_PARSE_FAILED) { + LOG.debug("Skipping {} as un-parseable content will be deleted", key); + return; } else if (Integer.parseInt(fetchStatus) != CrawlDatum.STATUS_FETCH_SUCCESS) { // content not fetched successfully, skip document LOG.debug("Skipping {} as content is not fetched successfully", key); @@ -126,6 +137,7 @@ public void map(WritableComparable key, Content content, } if (skipTruncated && isTruncated(content)) { + LOG.debug("Skipping {} as content is truncated", key); return; } diff --git a/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java b/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java index 581b528f31..4202b26180 100644 --- a/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java +++ b/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java @@ -477,4 +477,33 @@ public static Server getServer(int port, @NonNull String staticContent) webServer.insertHandler(staticContext); return webServer; } + + /** + * Creates a new JettyServer with one static root context and the provided resource handler. + * + * @param port + * port to listen to + * @param staticContent + * folder where static content lives + * @param resourceHandler + * resource handler to override the default behavior if needed. + * @return configured Jetty server instance + * @throws UnknownHostException + */ + @NonNull + public static Server getServer(int port, @NonNull String staticContent, ResourceHandler resourceHandler) + throws UnknownHostException { + Server webServer = new Server(); + + ServerConnector listener = new ServerConnector(webServer); + listener.setPort(port); + listener.setHost("127.0.0.1"); + webServer.addConnector(listener); + ContextHandler staticContext = new ContextHandler(); + staticContext.setContextPath("/"); + staticContext.setResourceBase(staticContent); + staticContext.insertHandler(resourceHandler); + webServer.insertHandler(staticContext); + return webServer; + } } diff --git a/src/test/org/apache/nutch/crawl/TestCrawlDbStates.java b/src/test/org/apache/nutch/crawl/TestCrawlDbStates.java index b69c6efedc..217039767c 100644 --- a/src/test/org/apache/nutch/crawl/TestCrawlDbStates.java +++ b/src/test/org/apache/nutch/crawl/TestCrawlDbStates.java @@ -21,7 +21,7 @@ import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.Reducer.Context; import org.apache.hadoop.util.StringUtils; -import org.apache.nutch.metadata.Nutch; +import org.apache.nutch.parse.ParseSegment; import org.apache.nutch.scoring.ScoringFilterException; import org.apache.nutch.scoring.ScoringFilters; import org.apache.nutch.util.ReducerContextWrapper; @@ -378,7 +378,7 @@ public void testCrawlDbReducerParseFailed() { LOG.info("NUTCH-1732: allow deleting un-parsable documents"); Context context = CrawlDBTestUtil.createContext(); Configuration conf = context.getConfiguration(); - conf.setBoolean(Nutch.DELETE_FAILED_PARSE, true); + conf.setBoolean(ParseSegment.DELETE_FAILED_PARSE, true); CrawlTestParserFailure crawlUtil = new CrawlTestParserFailure(context); try { if (!crawlUtil.run(20)) { diff --git a/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java b/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java index 591fb782b0..0ed666558d 100644 --- a/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java +++ b/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java @@ -30,6 +30,7 @@ import org.apache.nutch.crawl.Generator; import org.apache.nutch.crawl.Injector; import org.apache.nutch.metadata.Nutch; +import org.apache.nutch.parse.ParseSegment; import org.apache.nutch.protocol.Content; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Server; @@ -39,7 +40,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,14 +78,14 @@ public void setUp() throws Exception { LOG.info("TEST FILES : " + files); conf = CrawlDBTestUtil.createContext().getConfiguration(); + // Do not include 'parse-tika', because it would parse anything conf.set("plugin.includes", "protocol-http|urlfilter-regex|parse-html|index-(basic|anchor)|indexer-csv|scoring-opic|urlnormalizer-(pass|regex|basic)"); conf.setInt("fetcher.threads.fetch", 1); - conf.setInt("http.content.limit", 1024); - // Default parser.html.impl is 'neko', other option is 'tagsoup'. Both are too tolerant for this test. - conf.set("parser.html.impl", "tagsoup"); + // force an exception from ParserFactory.getParsers(...): ParserNotFound + conf.setBoolean("mime.type.magic", false); conf.setBoolean("fetcher.parse", true); conf.setBoolean("fetcher.store.content", true); - conf.setBoolean(Nutch.DELETE_FAILED_PARSE, true); + conf.setBoolean(ParseSegment.DELETE_FAILED_PARSE, true); fs = FileSystem.get(conf); fs.delete(TEST_DIR, true); @@ -112,8 +112,6 @@ public void tearDown() throws Exception { @Test - // Ref: full hadoop.log in logs/hadoop.log - @Disabled("The default HTML Parser (Neko) is a lot more tolerant than expected. Even random bytes in an HTML file do not produce a parsing error.") public void testFetchWithParseFailure() throws Exception { AbstractFetchSchedule schedule = new AbstractFetchSchedule(conf) {}; @@ -174,13 +172,7 @@ public Map call() throws Exception { } assertEquals(urls.size(), fetchedUrls.size()); LOG.info("1ST FETCHED URLS: {}", fetchedUrls); - -// // Validate what is now served: -// String urlString = "http://127.0.0.1:55000/test.html"; -// Protocol protocol = new ProtocolFactory(conf).getProtocol(urlString); -// Content content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent(); -// String contentStr = new String(content.getContent()); -// LOG.info("TEST CONTENT FROM SERVER: {}", contentStr); // makes the log file unreadable + fetchedUrls.entrySet().forEach(i -> Assertions.assertEquals(i.getValue(), "" + CrawlDatum.STATUS_FETCH_SUCCESS)); // force re-fetch and wait a bit urls.forEach(i -> @@ -239,9 +231,9 @@ public Map call() throws Exception { for (Map.Entry entry : fetchedUrls.entrySet()) { if(entry.getKey().endsWith(TEST_FILE)) { - Assertions.assertEquals(entry.getValue(), ""+CrawlDatum.STATUS_PARSE_FAILED); + Assertions.assertEquals(entry.getValue(), "" + CrawlDatum.STATUS_PARSE_FAILED); } else { - Assertions.assertEquals(entry.getValue(), ""+CrawlDatum.STATUS_FETCH_SUCCESS); + Assertions.assertEquals(entry.getValue(), "" + CrawlDatum.STATUS_FETCH_SUCCESS); } } } @@ -256,11 +248,11 @@ public ParseFailureResourceHandler() { public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { if(FETCH_COUNT.get() == 1 && target.endsWith(TEST_FILE)) { - LOG.info("FEEDING RANDOM BYTES INTO THE RESPONSE"); - response.setContentType("text/html"); - response.setContentLength(2048); - - byte[] randomBytes = new byte[2048]; + LOG.info("Set mime type 'application/unknown' and random bytes content"); + // force an exception from ParserFactory.getParsers(...): ParserNotFound + response.setContentType("application/unknown"); + response.setContentLength(1024); + byte[] randomBytes = new byte[1024]; new Random(123).nextBytes(randomBytes); response.getOutputStream().write(randomBytes); diff --git a/src/test/org/apache/nutch/indexer/TestIndexerMapReduce.java b/src/test/org/apache/nutch/indexer/TestIndexerMapReduce.java index a5001515d3..c49eef3578 100644 --- a/src/test/org/apache/nutch/indexer/TestIndexerMapReduce.java +++ b/src/test/org/apache/nutch/indexer/TestIndexerMapReduce.java @@ -26,6 +26,7 @@ import org.apache.nutch.metadata.Nutch; import org.apache.nutch.parse.Outlink; import org.apache.nutch.parse.ParseData; +import org.apache.nutch.parse.ParseSegment; import org.apache.nutch.parse.ParseStatus; import org.apache.nutch.parse.ParseText; import org.apache.nutch.protocol.Content; @@ -104,7 +105,7 @@ public class TestIndexerMapReduce { @Test public void testDeleteParseFailure() { configuration = NutchConfiguration.create(); - configuration.setBoolean(Nutch.DELETE_FAILED_PARSE, true); + configuration.setBoolean(ParseSegment.DELETE_FAILED_PARSE, true); // unrelated issue with "index.jexl.filter", don't use all plugins. Ref: src/test/nutch-site.xml configuration.set("plugin.includes", "protocol-http|urlfilter-regex|parse-(html|tika)|index-(basic|anchor)|indexer-csv|scoring-opic|urlnormalizer-(pass|regex|basic)"); From 5670941e9030aca688bdc7ff4289f191738b3b7b Mon Sep 17 00:00:00 2001 From: Isabelle Giguere Date: Fri, 12 Jun 2026 17:48:50 -0400 Subject: [PATCH 8/8] NUTCH-1732: allow deleting not parsable documents Remove spaces ant end of lines. Add ASF header. --- conf/nutch-default.xml | 12 ++-- .../apache/nutch/crawl/CrawlDbReducer.java | 10 +-- .../org/apache/nutch/parse/ParseSegment.java | 10 +-- .../org/apache/nutch/parse/ParseStatus.java | 2 +- .../apache/nutch/crawl/CrawlDBTestUtil.java | 24 ++++--- .../apache/nutch/crawl/TestCrawlDbStates.java | 12 ++-- .../fetcher/TestFetchWithParseFailures.java | 64 ++++++++++++------- .../nutch/indexer/TestIndexerMapReduce.java | 16 ++--- 8 files changed, 85 insertions(+), 65 deletions(-) diff --git a/conf/nutch-default.xml b/conf/nutch-default.xml index dd00848b7b..f5c5033943 100644 --- a/conf/nutch-default.xml +++ b/conf/nutch-default.xml @@ -1808,8 +1808,8 @@ CAUTION: Set the parser.timeout to -1 or a bigger value than 30, when using this parser.delete.failed.parse false - Boolean value for whether we should delete a page from the index when parsing the page fails. - By default this property is deactivated, because it will delete an existing page from the index, where a + Boolean value for whether we should delete a page from the index when parsing the page fails. + By default this property is deactivated, because it will delete an existing page from the index, where a previous fetch produced content that was successfully parsed. @@ -2264,14 +2264,14 @@ CAUTION: Set the parser.timeout to -1 or a bigger value than 30, when using this - @@ -2414,7 +2414,7 @@ CAUTION: Set the parser.timeout to -1 or a bigger value than 30, when using this The values (as strings) to pass into the POJO constructor. The POJO must accept a String representation of the NutchDocument's URL - as the first parameter in the constructor. The values you specify here + as the first parameter in the constructor. The values you specify here will populate the constructor arguments 1,..,n-1 where n=the count of arguments to the constructor. Argument #0 will be the NutchDocument's URL. diff --git a/src/java/org/apache/nutch/crawl/CrawlDbReducer.java b/src/java/org/apache/nutch/crawl/CrawlDbReducer.java index cd553af174..4a801f15bd 100644 --- a/src/java/org/apache/nutch/crawl/CrawlDbReducer.java +++ b/src/java/org/apache/nutch/crawl/CrawlDbReducer.java @@ -76,8 +76,8 @@ public void setup(Reducer.Context context) { * Get counter for status, caching for subsequent lookups. */ private Counter getStatusCounter(byte status, Context context) { - return statusCounters.computeIfAbsent(status, - s -> context.getCounter(NutchMetrics.GROUP_CRAWLDB, + return statusCounters.computeIfAbsent(status, + s -> context.getCounter(NutchMetrics.GROUP_CRAWLDB, CrawlDatum.getStatusName(s))); } @@ -130,7 +130,7 @@ public void reduce(Text key, Iterable values, } continue; } - + switch (datum.getStatus()) { // collect other info case CrawlDatum.STATUS_LINKED: CrawlDatum link; @@ -243,7 +243,7 @@ public void reduce(Text key, Iterable values, result.getMetaData().put(e.getKey(), e.getValue()); } } - + // determine the modification status int modified = FetchSchedule.STATUS_UNKNOWN; if (fetch.getStatus() == CrawlDatum.STATUS_FETCH_NOTMODIFIED) { @@ -327,7 +327,7 @@ public void reduce(Text key, Iterable values, result = schedule.setPageGoneSchedule(key, result, prevFetchTime, prevModifiedTime, fetch.getFetchTime()); break; - + case CrawlDatum.STATUS_FETCH_GONE: // permanent failure if (oldSet) result.setSignature(old.getSignature()); // use old signature diff --git a/src/java/org/apache/nutch/parse/ParseSegment.java b/src/java/org/apache/nutch/parse/ParseSegment.java index 7c6465baff..e31830dea6 100644 --- a/src/java/org/apache/nutch/parse/ParseSegment.java +++ b/src/java/org/apache/nutch/parse/ParseSegment.java @@ -129,7 +129,7 @@ public void map(WritableComparable key, Content content, return; } else if(deleteFailedParse && Integer.parseInt(fetchStatus) == CrawlDatum.STATUS_PARSE_FAILED) { LOG.debug("Skipping {} as un-parseable content will be deleted", key); - return; + return; } else if (Integer.parseInt(fetchStatus) != CrawlDatum.STATUS_FETCH_SUCCESS) { // content not fetched successfully, skip document LOG.debug("Skipping {} as content is not fetched successfully", key); @@ -199,7 +199,7 @@ public void map(WritableComparable key, Content content, /** * Checks if the page's content is truncated. - * + * * @param content the response {@link org.apache.nutch.protocol.Content} * @return If the page is truncated true. When it is not, or when * it couldn't be determined, false. @@ -259,7 +259,7 @@ public void reduce(Text key, Iterable values, } } - public void parse(Path segment) throws IOException, + public void parse(Path segment) throws IOException, InterruptedException, ClassNotFoundException { if (SegmentChecker.isParsed(segment, segment.getFileSystem(getConf()))) { LOG.warn("Segment: {} already parsed!! Skipped parsing this segment!!", segment); // NUTCH-1854 @@ -356,7 +356,7 @@ public Map run(Map args, String crawlId) throws else { String segment_dir = crawlId+"/segments"; File segmentsDir = new File(segment_dir); - File[] segmentsList = segmentsDir.listFiles(); + File[] segmentsList = segmentsDir.listFiles(); Arrays.sort(segmentsList, (f1, f2) -> { if(f1.lastModified()>f2.lastModified()) return -1; @@ -365,7 +365,7 @@ public Map run(Map args, String crawlId) throws }); segment = new Path(segmentsList[0].getPath()); } - + if (args.containsKey("nofilter")) { getConf().setBoolean("parse.filter.urls", false); } diff --git a/src/java/org/apache/nutch/parse/ParseStatus.java b/src/java/org/apache/nutch/parse/ParseStatus.java index 5d4e846800..b69bbe3f19 100644 --- a/src/java/org/apache/nutch/parse/ParseStatus.java +++ b/src/java/org/apache/nutch/parse/ParseStatus.java @@ -188,7 +188,7 @@ public void write(DataOutput out) throws IOException { public boolean isSuccess() { return majorCode == SUCCESS; } - + public boolean isFailed() { return majorCode == FAILED; } diff --git a/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java b/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java index 8a9b75ef5d..d84385fb2b 100644 --- a/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java +++ b/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java @@ -19,6 +19,7 @@ import java.lang.invoke.MethodHandles; import java.io.IOException; import java.net.URI; +import java.net.UnknownHostException; import java.nio.file.Files; import java.util.ArrayList; import java.util.Iterator; @@ -29,8 +30,11 @@ import static org.mockserver.model.HttpRequest.request; import static org.mockserver.model.HttpResponse.response; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.ContextHandler; +import org.eclipse.jetty.server.handler.ResourceHandler; import org.jspecify.annotations.NonNull; -import org.jspecify.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -70,7 +74,7 @@ public class CrawlDBTestUtil { private static CrawlDbReducer reducer = new CrawlDbReducer(); /** * Creates synthetic crawldb - * + * * @param conf * configuration to use * @param fs @@ -213,7 +217,7 @@ public URI[] getCacheFiles() throws IOException { public RawComparator getCombinerKeyGroupingComparator() { return null; } - + @Override public Configuration getConfiguration() { return conf; @@ -325,7 +329,7 @@ public Class getOutputValueClass() { public Class> getPartitionerClass() throws ClassNotFoundException { return null; } - + @Override public boolean getProfileEnabled() { return false; @@ -376,7 +380,7 @@ public Path getWorkingDirectory() throws IOException { * For now we need to manually construct our Configuration, because we need to * override the default one and it is currently not possible to use * dynamically set values. - * + * * @return a new Reducer Context with test configuration */ @NonNull @@ -405,7 +409,7 @@ public URLCrawlDatum(@NonNull Text url, @NonNull CrawlDatum datum) { /** * Generate seedlist - * + * * @param fs filesystem to use * @param urlPath path where seed file will be created * @param urls list of URLs to write @@ -418,7 +422,7 @@ public static void generateSeedList(@NonNull FileSystem fs, @NonNull Path urlPat /** * Generate seedlist with optional metadata - * + * * @param fs filesystem to use * @param urlPath path where seed file will be created * @param urls list of URLs to write @@ -500,15 +504,15 @@ private static String probeContentType(java.nio.file.Path file) String ct = Files.probeContentType(file); return ct != null ? ct : "application/octet-stream"; } - + /** * Creates a new JettyServer with one static root context and the provided resource handler. - * + * * @param port * port to listen to * @param staticContent * folder where static content lives - * @param resourceHandler + * @param resourceHandler * resource handler to override the default behavior if needed. * @return configured Jetty server instance * @throws UnknownHostException diff --git a/src/test/org/apache/nutch/crawl/TestCrawlDbStates.java b/src/test/org/apache/nutch/crawl/TestCrawlDbStates.java index 217039767c..6ed49793e3 100644 --- a/src/test/org/apache/nutch/crawl/TestCrawlDbStates.java +++ b/src/test/org/apache/nutch/crawl/TestCrawlDbStates.java @@ -239,9 +239,9 @@ public void testCrawlDbStatTransitionInject() { LOG.error(StringUtils.stringifyException(e)); } values.add(injected); - + List result = new ArrayList(); - Map res = new HashMap<>(); + Map res = new HashMap<>(); ReducerContextWrapper contextWrapper = new ReducerContextWrapper(injector, conf, res); try { injector.setup(contextWrapper.getContext()); @@ -371,7 +371,7 @@ public void testCrawlDbReducerPageGoneSchedule2() { e.printStackTrace(); } } - + @Test public void testCrawlDbReducerParseFailed() { @@ -785,7 +785,7 @@ protected boolean check(CrawlDatum result) { } } - + private class CrawlTestParserFailure extends ContinuousCrawlTestUtil { int counter = 0; @@ -794,7 +794,7 @@ private class CrawlTestParserFailure extends ContinuousCrawlTestUtil { public CrawlTestParserFailure(Context context) { super(context); } - + @Override protected CrawlDatum fetch(CrawlDatum datum, long currentTime) { counter++; @@ -807,7 +807,7 @@ protected CrawlDatum fetch(CrawlDatum datum, long currentTime) { } else { failing = false; datum.setStatus(STATUS_FETCH_SUCCESS); - LOG.info("expect fetch success"); + LOG.info("expect fetch success"); } datum.setFetchTime(currentTime); return datum; diff --git a/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java b/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java index 0ed666558d..f13ba2f4a0 100644 --- a/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java +++ b/src/test/org/apache/nutch/fetcher/TestFetchWithParseFailures.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.nutch.fetcher; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -49,34 +65,34 @@ public class TestFetchWithParseFailures { private static final Logger LOG = LoggerFactory .getLogger(MethodHandles.lookup().lookupClass()); - + private static final Path TEST_DIR = new Path("build/test/test-fail-parse"); - + private static final String BASE_FOLDER = "build/test/data/fetch-parse-failure"; private static final String TEST_FILE = "test.html"; - + private Configuration conf; private FileSystem fs; private Path crawldbPath; private Path segmentsPath; private Path urlPath; private Server server; - + private static List files; private static java.nio.file.Path baseFolderPath; - + private static ExecutorService executor = Executors.newCachedThreadPool(); private static final AtomicInteger FETCH_COUNT = new AtomicInteger(0); - - + + @BeforeEach public void setUp() throws Exception { baseFolderPath = java.nio.file.Paths.get(BASE_FOLDER); - + files = java.nio.file.Files.list(baseFolderPath).map(p -> p.getFileName().toString()).filter(n -> !n.equals("robots.txt")).collect(Collectors.toList()); Collections.sort(files); LOG.info("TEST FILES : " + files); - + conf = CrawlDBTestUtil.createContext().getConfiguration(); // Do not include 'parse-tika', because it would parse anything conf.set("plugin.includes", "protocol-http|urlfilter-regex|parse-html|index-(basic|anchor)|indexer-csv|scoring-opic|urlnormalizer-(pass|regex|basic)"); @@ -86,7 +102,7 @@ public void setUp() throws Exception { conf.setBoolean("fetcher.parse", true); conf.setBoolean("fetcher.store.content", true); conf.setBoolean(ParseSegment.DELETE_FAILED_PARSE, true); - + fs = FileSystem.get(conf); fs.delete(TEST_DIR, true); crawldbPath = new Path(TEST_DIR, "crawldb"); @@ -109,12 +125,12 @@ public void tearDown() throws Exception { } fs.delete(TEST_DIR, true); } - + @Test public void testFetchWithParseFailure() throws Exception { AbstractFetchSchedule schedule = new AbstractFetchSchedule(conf) {}; - + // generate seedlist ArrayList urls = new ArrayList(); files.forEach(f -> urls.add("http://127.0.0.1:" + server.getURI().getPort() + "/" + f)); @@ -129,7 +145,7 @@ public void testFetchWithParseFailure() throws Exception { Path[] generatedSegment1 = g1.generate(crawldbPath, segmentsPath, 1, Long.MAX_VALUE, Long.MAX_VALUE, false, false, true, 1, null); Assertions.assertNotNull(generatedSegment1); - + Map args1 = Map.of("segment", generatedSegment1[0]); // fetch once LOG.info("1ST FETCH"); @@ -150,10 +166,10 @@ public Map call() throws Exception { Assertions.assertFalse(result1.isEmpty()); Assertions.assertEquals("0", result1.get(Nutch.VAL_RESULT)); Assertions.assertEquals(1, FETCH_COUNT.get()); - + CrawlDb crawlDb = new CrawlDb(conf); crawlDb.update(crawldbPath, generatedSegment1, true, true, true, true); - + // verify content LOG.info("1ST VERIFY CONTENT"); Map fetchedUrls = new HashMap<>(); @@ -173,9 +189,9 @@ public Map call() throws Exception { assertEquals(urls.size(), fetchedUrls.size()); LOG.info("1ST FETCHED URLS: {}", fetchedUrls); fetchedUrls.entrySet().forEach(i -> Assertions.assertEquals(i.getValue(), "" + CrawlDatum.STATUS_FETCH_SUCCESS)); - + // force re-fetch and wait a bit - urls.forEach(i -> + urls.forEach(i -> schedule.forceRefetch(new Text(i), new CrawlDatum(CrawlDatum.STATUS_DB_UNFETCHED, 1), true)); long start = System.currentTimeMillis(); while(System.currentTimeMillis() < start + 1000L); @@ -193,7 +209,7 @@ public Map call() throws Exception { // next fetch should generate parse failure LOG.info("2ND FETCH"); Fetcher fetcher2 = new Fetcher(conf); - + Map result2 = executor.submit(new Callable>(){ @Override @@ -228,7 +244,7 @@ public Map call() throws Exception { } assertEquals(urls.size(), fetchedUrls.size()); LOG.info("2ND FETCHED STATUS: {}", fetchedUrls); - + for (Map.Entry entry : fetchedUrls.entrySet()) { if(entry.getKey().endsWith(TEST_FILE)) { Assertions.assertEquals(entry.getValue(), "" + CrawlDatum.STATUS_PARSE_FAILED); @@ -237,15 +253,15 @@ public Map call() throws Exception { } } } - + public static class ParseFailureResourceHandler extends ResourceHandler { - + public ParseFailureResourceHandler() { super(); } - + @Override - public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) + public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { if(FETCH_COUNT.get() == 1 && target.endsWith(TEST_FILE)) { LOG.info("Set mime type 'application/unknown' and random bytes content"); @@ -255,7 +271,7 @@ public void handle(String target, Request baseRequest, HttpServletRequest reques byte[] randomBytes = new byte[1024]; new Random(123).nextBytes(randomBytes); response.getOutputStream().write(randomBytes); - + } else { super.handle(target, baseRequest, request, response); } diff --git a/src/test/org/apache/nutch/indexer/TestIndexerMapReduce.java b/src/test/org/apache/nutch/indexer/TestIndexerMapReduce.java index c49eef3578..3c2b68362b 100644 --- a/src/test/org/apache/nutch/indexer/TestIndexerMapReduce.java +++ b/src/test/org/apache/nutch/indexer/TestIndexerMapReduce.java @@ -106,7 +106,7 @@ public class TestIndexerMapReduce { public void testDeleteParseFailure() { configuration = NutchConfiguration.create(); configuration.setBoolean(ParseSegment.DELETE_FAILED_PARSE, true); - + // unrelated issue with "index.jexl.filter", don't use all plugins. Ref: src/test/nutch-site.xml configuration.set("plugin.includes", "protocol-http|urlfilter-regex|parse-(html|tika)|index-(basic|anchor)|indexer-csv|scoring-opic|urlnormalizer-(pass|regex|basic)"); @@ -124,7 +124,7 @@ public void testDeleteParseFailure() { Content content = new Content(testUrl, testUrl, testHtmlDoc.getBytes(StandardCharsets.UTF_8), htmlContentType, htmlMeta, configuration); - + if(failParse) { data = failedParseData; text = failedParseText; @@ -159,7 +159,7 @@ public void testDeleteParseFailure() { public void testBinaryContentBase64() { configuration = NutchConfiguration.create(); configuration.setBoolean(IndexerMapReduce.INDEXER_BINARY_AS_BASE64, true); - + // unrelated issue with "index.jexl.filter", don't use all plugins. Ref: src/test/nutch-site.xml configuration.set("plugin.includes", "protocol-http|urlfilter-regex|parse-(html|tika)|index-(basic|anchor)|indexer-csv|scoring-opic|urlnormalizer-(pass|regex|basic)"); @@ -216,8 +216,8 @@ public void testBinaryContentBase64() { * @param content * (optional, if index binary content) protocol content * @return "indexed" document - * @throws InterruptedException - * @throws IOException + * @throws InterruptedException + * @throws IOException */ @SuppressWarnings({ "unchecked", "rawtypes" }) public NutchDocument runIndexer(CrawlDatum dbDatum, CrawlDatum fetchDatum, @@ -228,14 +228,14 @@ public NutchDocument runIndexer(CrawlDatum dbDatum, CrawlDatum fetchDatum, values.add(new NutchWritable(parseText)); values.add(new NutchWritable(parseData)); values.add(new NutchWritable(content)); - Map reduceResult = new HashMap<>(); + Map reduceResult = new HashMap<>(); ReducerContextWrapper contextWrapper = new ReducerContextWrapper(reducer, configuration, reduceResult); NutchDocument doc = null; - try { + try { reducer.setup(contextWrapper.getContext()); // test reducer.reduce(testUrlText, values, contextWrapper.getContext()); - + for (Map.Entry e : reduceResult.entrySet()) { if (e.getValue().action != NutchIndexAction.DELETE) { doc = e.getValue().doc;