|  | 
| 12 | 12 | import org.apache.http.client.methods.CloseableHttpResponse; | 
| 13 | 13 | import org.apache.http.impl.client.CloseableHttpClient; | 
| 14 | 14 | import org.apache.http.util.EntityUtils; | 
| 15 |  | -import org.slf4j.Logger; | 
| 16 |  | -import org.slf4j.LoggerFactory; | 
| 17 | 15 | 
 | 
| 18 | 16 | import us.codecraft.webmagic.Page; | 
| 19 | 17 | import us.codecraft.webmagic.Request; | 
|  | 
| 33 | 31 |  */ | 
| 34 | 32 | public class HttpClientDownloader extends AbstractDownloader { | 
| 35 | 33 | 
 | 
| 36 |  | -    private Logger logger = LoggerFactory.getLogger(getClass()); | 
| 37 |  | - | 
| 38 | 34 |     private final Map<String, CloseableHttpClient> httpClients = new HashMap<String, CloseableHttpClient>(); | 
| 39 | 35 | 
 | 
| 40 | 36 |     private HttpClientGenerator httpClientGenerator = new HttpClientGenerator(); | 
| @@ -84,16 +80,10 @@ public Page download(Request request, Task task) { | 
| 84 | 80 |         try { | 
| 85 | 81 |             httpResponse = httpClient.execute(requestContext.getHttpUriRequest(), requestContext.getHttpClientContext()); | 
| 86 | 82 |             page = handleResponse(request, request.getCharset() != null ? request.getCharset() : task.getSite().getCharset(), httpResponse, task); | 
| 87 |  | - | 
| 88 | 83 |             onSuccess(page, task); | 
| 89 |  | -            logger.info("Download page success: {}", request.getUrl()); | 
| 90 |  | - | 
| 91 | 84 |             return page; | 
| 92 | 85 |         } catch (IOException e) { | 
| 93 |  | - | 
| 94 | 86 |             onError(page, task, e); | 
| 95 |  | -            logger.info("Download page error: {}", request.getUrl(), e); | 
| 96 |  | - | 
| 97 | 87 |             return page; | 
| 98 | 88 |         } finally { | 
| 99 | 89 |             if (httpResponse != null) { | 
| @@ -138,7 +128,6 @@ private String getHtmlCharset(String contentType, byte[] contentBytes, Task task | 
| 138 | 128 |         String charset = CharsetUtils.detectCharset(contentType, contentBytes); | 
| 139 | 129 |         if (charset == null) { | 
| 140 | 130 |             charset = Optional.ofNullable(task.getSite().getDefaultCharset()).orElseGet(Charset.defaultCharset()::name); | 
| 141 |  | -            logger.info("Charset autodetect failed, use {} as charset.", task.getSite().getDefaultCharset()); | 
| 142 | 131 |         } | 
| 143 | 132 |         return charset; | 
| 144 | 133 |     } | 
|  | 
0 commit comments