update pipeline api

Author: code4craft

webmagic-core/src/main/java/us/codecraft/webmagic/Page.java

@@ -6,8 +6,6 @@
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
 
 /**
  * <pre>
@@ -27,52 +25,24 @@ public class Page {
 
     private Request request;
 
-    private Map<String, Selectable> fields = new ConcurrentHashMap<String, Selectable>();
+    private ResultItems resultItems = new ResultItems();
 
     private Selectable html;
 
     private Selectable url;
 
     private List<Request> targetRequests = new ArrayList<Request>();
 
-    private boolean skip;
-
-    private Object extra;
-
-    /**
-     * 是否忽略这个页面，用于pipeline来判断是否对这个页面进行处理
-     * @return 是否忽略 true 忽略
-     */
-    public boolean isSkip() {
-        return skip;
-    }
-
-    /**
-     * 设置是否忽略这个页面，用于pipeline来判断是否对这个页面进行处理
-     * @param skip 是否忽略 true 忽略
-     */
-    public void setSkip(boolean skip) {
-        this.skip = skip;
-    }
-
     public Page() {
     }
 
-    /**
-     * 获取抽取的结果，在{@link us.codecraft.webmagic.pipeline.Pipeline} 中调用
-     * @return fields 抽取的结果
-     */
-    public Map<String, Selectable> getFields() {
-        return fields;
-    }
-
     /**
      * 保存抽取的结果
      * @param key 结果的key
      * @param field 结果的value
      */
-    public void putField(String key, Selectable field) {
-        fields.put(key, field);
+    public void putField(String key, Object field) {
+        resultItems.put(key, field);
     }
 
     /**
@@ -157,23 +127,10 @@ public Request getRequest() {
 
     public void setRequest(Request request) {
         this.request = request;
+        this.resultItems.setRequest(request);
     }
 
-    /**
-     * 获取附加对象
-     * @param <T> 对象类型
-     * @return 对象内容
-     */
-    public <T> T getExtra() {
-        return (T)extra;
-    }
-
-    /**
-     * 设置附加对象
-     * @param extra 对象内容
-     * @param <T> 对象类型
-     */
-    public <T> void setExtra(T extra) {
-        this.extra = extra;
+    public ResultItems getResultItems() {
+        return resultItems;
     }
 }

webmagic-core/src/main/java/us/codecraft/webmagic/ResultItems.java

@@ -0,0 +1,64 @@
+package us.codecraft.webmagic;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * 保存抽取结果的类，由PageProcessor处理得到，传递给{@link us.codecraft.webmagic.pipeline.Pipeline}进行持久化。<br>
+ * @author [email protected] <br>
+ * @date: 13-7-25 <br>
+ * Time: 下午12:20 <br>
+ */
+public class ResultItems {
+
+    private Map<String, Object> fields = new HashMap<String, Object>();
+
+    private Request request;
+
+    private boolean skip;
+
+    public <T> T get(String key) {
+        Object o = fields.get(key);
+        if (o == null) {
+            return null;
+        }
+        return (T) fields.get(key);
+    }
+
+    public  Map<String, Object> getAll() {
+        return fields;
+    }
+
+    public <T> ResultItems put(String key, T value) {
+        fields.put(key, value);
+        return this;
+    }
+
+    public Request getRequest() {
+        return request;
+    }
+
+    public ResultItems setRequest(Request request) {
+        this.request = request;
+        return this;
+    }
+
+    /**
+     * 是否忽略这个页面，用于pipeline来判断是否对这个页面进行处理
+     * @return 是否忽略 true 忽略
+     */
+    public boolean isSkip() {
+        return skip;
+    }
+
+
+    /**
+     * 设置是否忽略这个页面，用于pipeline来判断是否对这个页面进行处理
+     * @param skip
+     * @return this
+     */
+    public ResultItems setSkip(boolean skip) {
+        this.skip = skip;
+        return this;
+    }
+}

webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java

@@ -196,7 +196,7 @@ private void processRequest(Request request) {
         pageProcessor.process(page);
         addRequest(page);
         for (Pipeline pipeline : pipelines) {
-            pipeline.process(page, this);
+            pipeline.process(page.getResultItems(), this);
         }
         sleep(site.getSleepTime());
     }

webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/ConsolePipeline.java

@@ -1,8 +1,7 @@
 package us.codecraft.webmagic.pipeline;
 
-import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.ResultItems;
 import us.codecraft.webmagic.Task;
-import us.codecraft.webmagic.selector.Selectable;
 
 import java.util.Map;
 
@@ -15,13 +14,10 @@
 public class ConsolePipeline implements  Pipeline{
 
     @Override
-    public void process(Page page,Task task) {
-        System.out.println("get page: "+page.getUrl());
-        for (Map.Entry<String, Selectable> entry : page.getFields().entrySet()) {
-            System.out.println(entry.getKey()+":\t"+entry.getValue().toStrings());
-        }
-        if (page.getExtra()!=null){
-            System.out.println(page.getExtra());
+    public void process(ResultItems resultItems,Task task) {
+        System.out.println("get page: "+resultItems.getRequest().getUrl());
+        for (Map.Entry<String, Object> entry : resultItems.getAll().entrySet()) {
+            System.out.println(entry.getKey()+":\t"+entry.getValue());
         }
     }
 }

webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/FilePipeline.java

@@ -2,13 +2,14 @@
 
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.log4j.Logger;
-import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.ResultItems;
 import us.codecraft.webmagic.Task;
 
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.util.Map;
 
 /**
  * 持久化到文件的接口。
@@ -38,16 +39,18 @@ public FilePipeline(String path) {
     }
 
     @Override
-    public void process(Page page, Task task) {
+    public void process(ResultItems resultItems, Task task) {
         String path = this.path + "/" + task.getUUID() + "/";
         File file = new File(path);
         if (!file.exists()) {
             file.mkdirs();
         }
         try {
-            PrintWriter printWriter = new PrintWriter(new FileWriter(path + DigestUtils.md5Hex(page.getUrl().toString())));
-            printWriter.println("url:\t" + page.getUrl());
-            printWriter.println("html:\t" + page.getHtml());
+            PrintWriter printWriter = new PrintWriter(new FileWriter(path + DigestUtils.md5Hex(resultItems.getRequest().getUrl())));
+            printWriter.println("url:\t" + resultItems.getRequest().getUrl());
+            for (Map.Entry<String, Object> entry : resultItems.getAll().entrySet()) {
+                printWriter.println(entry.getKey()+":\t"+entry.getValue());
+            }
             printWriter.close();
         } catch (IOException e) {
             logger.warn("write file error",e);

webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/Pipeline.java

@@ -1,6 +1,6 @@
 package us.codecraft.webmagic.pipeline;
 
-import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.ResultItems;
 import us.codecraft.webmagic.Task;
 
 /**
@@ -11,5 +11,5 @@
  */
 public interface Pipeline {
 
-    public void process(Page page,Task task);
+    public void process(ResultItems resultItems,Task task);
 }

webmagic-core/src/main/java/us/codecraft/webmagic/processor/SimplePageProcessor.java

@@ -30,12 +30,13 @@ public SimplePageProcessor(String startUrl, String urlPattern) {
 
     @Override
     public void process(Page page) {
-        List<String> requests = page.getHtml().links().regex(urlPattern).toStrings();
+        List<String> requests = page.getHtml().links().regex(urlPattern).all();
         //调用page.addTargetRequests()方法添加待抓取链接
         page.addTargetRequests(requests);
         //xpath方式抽取
         page.putField("title", page.getHtml().xpath("//title"));
         //sc表示使用Readability技术抽取正文
+        page.putField("html", page.getHtml().toString());
         page.putField("content", page.getHtml().smartContent());
     }
 

webmagic-core/src/main/java/us/codecraft/webmagic/selector/PlainText.java

@@ -82,14 +82,14 @@ public Selectable replace(String regex, String replacement) {
     }
 
     @Override
-    public List<String> toStrings() {
+    public List<String> all() {
         return strings;
     }
 
     @Override
     public String toString() {
-        if (CollectionUtils.isNotEmpty(toStrings())) {
-            return toStrings().get(0);
+        if (CollectionUtils.isNotEmpty(all())) {
+            return all().get(0);
         } else {
             return null;
         }

webmagic-core/src/main/java/us/codecraft/webmagic/selector/Selectable.java

@@ -69,5 +69,5 @@ public interface Selectable {
      *
      * @return multi string result
      */
-    public List<String> toStrings();
+    public List<String> all();
 }

webmagic-core/src/test/java/us/codecraft/webmagic/selector/XpathSelectorTest.java

@@ -1351,7 +1351,7 @@ public void test() {
     public void testOschina() {
         Html html1 = new Html(html);
         Assert.assertEquals("再次吐槽easyui", html1.xpath(".//*[@class='QTitle']/h1/a").toString());
-        Assert.assertNotNull(html1.$("a[href]").xpath("//@href").toStrings());
+        Assert.assertNotNull(html1.$("a[href]").xpath("//@href").all());
     }
 
 }