From 7343ed846c102dd6870cb2fabeadc5c95a17c8f6 Mon Sep 17 00:00:00 2001 From: "dave.lowless" Date: Wed, 21 Nov 2018 14:44:00 +0000 Subject: [PATCH] Initial attempt at adding captureJavascriptContent option --- README.md | 2 +- .../lightbody/bmp/BrowserMobProxyServer.java | 15 +++++++++++- .../bmp/filters/HarCaptureFilter.java | 23 +++++++++++++++---- .../BrowserMobProxyServerLegacyAdapter.java | 11 +++++++++ .../bmp/proxy/LegacyProxyServer.java | 2 ++ .../net/lightbody/bmp/proxy/ProxyServer.java | 5 ++++ .../bmp/proxy/http/BrowserMobHttpClient.java | 17 +++++++++++++- .../bmp/proxy/bricks/ProxyResource.java | 15 ++++++++---- 8 files changed, 78 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index e932b282c..d41a58530 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ Description | HTTP method | Request path | Request parameters --- | :---: | :---: | --- Get a list of ports attached to `ProxyServer` instances managed by `ProxyManager` | GET | */proxy* || Creates a new proxy to run requests off of | POST | */proxy* |

*port* - Integer, The specific port to start the proxy service on. Optional, default is generated and returned in response.

*proxyUsername* - String, The username to use to authenticate with the chained proxy. Optional, default to null.

*proxyPassword* - String, The password to use to authenticate with the chained proxy. Optional, default to null.

*bindAddress* - String, If running BrowserMob Proxy in a multi-homed environment, specify a desired bind address. Optional, default to "0.0.0.0".

*serverBindAddress* - String, If running BrowserMob Proxy in a multi-homed environment, specify a desired server bind address. Optional, default to "0.0.0.0".

*useEcc* - Boolean. True, Uses Elliptic Curve Cryptography for certificate impersonation. Optional, default to "false".

*trustAllServers* - Boolean. True, Disables verification of all upstream servers' SSL certificates. All upstream servers will be trusted, even if they do not present valid certificates signed by certification authorities in the JDK's trust store. Optional, default to "false".

| -Creates a new HAR attached to the proxy and returns the HAR content if there was a previous HAR. *[port]* in request path it is port where your proxy was started | PUT |*/proxy/[port]/har* |

*captureHeaders* - Boolean, capture headers or not. Optional, default to "false".

*captureCookies* - Boolean, capture cookies or not. Optional, default to "false".

*captureContent* - Boolean, capture content bodies or not. Optional, default to "false".

*captureBinaryContent* - Boolean, capture binary content or not. Optional, default to "false".

*initialPageRef* - The string name of The first page ref that should be used in the HAR. Optional, default to "Page 1".

*initialPageTitle* - The title of first HAR page. Optional, default to *initialPageRef*.

+Creates a new HAR attached to the proxy and returns the HAR content if there was a previous HAR. *[port]* in request path it is port where your proxy was started | PUT |*/proxy/[port]/har* |

*captureHeaders* - Boolean, capture headers or not. Optional, default to "false".

*captureCookies* - Boolean, capture cookies or not. Optional, default to "false".

*captureContent* - Boolean, capture content bodies or not. Optional, default to "false".

*captureBinaryContent* - Boolean, capture binary content or not. Optional, default to "false".

*captureJavascriptContent* - Boolean, capture Javascript content or not. Optional, default to "true".

*initialPageRef* - The string name of The first page ref that should be used in the HAR. Optional, default to "Page 1".

*initialPageTitle* - The title of first HAR page. Optional, default to *initialPageRef*.

Starts a new page on the existing HAR. *[port]* in request path it is port where your proxy was started | PUT | */proxy/[port]/har/pageRef* |

*pageRef* - The string name of the first page ref that should be used in the HAR. Optional, default to "Page N" where N is the next page number.

*pageTitle* - The title of new HAR page. Optional, default to `pageRef`.

Shuts down the proxy and closes the port. *[port]* in request path it is port where your proxy was started | DELETE | */proxy/[port]* || Returns the JSON/HAR content representing all the HTTP traffic passed through the proxy (provided you have already created the HAR with [this method](#harcreate)) | GET | */proxy/[port]/har* || diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/BrowserMobProxyServer.java b/browsermob-core/src/main/java/net/lightbody/bmp/BrowserMobProxyServer.java index d4ddc5543..cd8e841a6 100644 --- a/browsermob-core/src/main/java/net/lightbody/bmp/BrowserMobProxyServer.java +++ b/browsermob-core/src/main/java/net/lightbody/bmp/BrowserMobProxyServer.java @@ -150,6 +150,11 @@ public class BrowserMobProxyServer implements BrowserMobProxy { */ private volatile EnumSet harCaptureTypes = EnumSet.noneOf(CaptureType.class); + /** + * Capture Javascript by default (providing captureContent is set) + */ + public volatile Boolean captureJavascriptContent = true; + /** * The current HAR being captured. */ @@ -483,6 +488,14 @@ public void setHarCaptureTypes(Set harCaptureSettings) { } } + public void setCaptureJavascriptContent(Boolean captureJavascriptContent) { + this.captureJavascriptContent = captureJavascriptContent; + } + + public Boolean getCaptureJavascriptContent() { + return this.captureJavascriptContent; + } + @Override public void setHarCaptureTypes(CaptureType... captureTypes) { if (captureTypes == null) { @@ -1151,7 +1164,7 @@ protected void addHarCaptureFilter() { public HttpFilters filterRequest(HttpRequest originalRequest, ChannelHandlerContext ctx) { Har har = getHar(); if (har != null && !ProxyUtils.isCONNECT(originalRequest)) { - return new HarCaptureFilter(originalRequest, ctx, har, getCurrentHarPage() == null ? null : getCurrentHarPage().getId(), getHarCaptureTypes()); + return new HarCaptureFilter(originalRequest, ctx, har, getCurrentHarPage() == null ? null : getCurrentHarPage().getId(), getHarCaptureTypes(), getCaptureJavascriptContent()); } else { return null; } diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/filters/HarCaptureFilter.java b/browsermob-core/src/main/java/net/lightbody/bmp/filters/HarCaptureFilter.java index ca8c044c2..cd307d21d 100644 --- a/browsermob-core/src/main/java/net/lightbody/bmp/filters/HarCaptureFilter.java +++ b/browsermob-core/src/main/java/net/lightbody/bmp/filters/HarCaptureFilter.java @@ -90,6 +90,8 @@ public class HarCaptureFilter extends HttpsAwareFiltersAdapter { private volatile long responseReceiveStartedNanos; + private volatile Boolean captureJavascriptContent; + /** * The address of the client making the request. Captured in the constructor and used when calculating and capturing ssl handshake and connect * timing information for SSL connections. @@ -133,8 +135,10 @@ public class HarCaptureFilter extends HttpsAwareFiltersAdapter { * @param currentPageRef the ProxyServer's currentPageRef at the time this request is received from the client * @param dataToCapture the data types to capture for this request. null or empty set indicates only basic information will be * captured (see {@link net.lightbody.bmp.proxy.CaptureType} for information on data collected for each CaptureType) + * @param captureJavascriptContent whether to capture Javascript or not */ - public HarCaptureFilter(HttpRequest originalRequest, ChannelHandlerContext ctx, Har har, String currentPageRef, Set dataToCapture) { + + public HarCaptureFilter(HttpRequest originalRequest, ChannelHandlerContext ctx, Har har, String currentPageRef, Set dataToCapture, Boolean captureJavascriptContent) { super(originalRequest, ctx); if (har == null) { @@ -153,6 +157,8 @@ public HarCaptureFilter(HttpRequest originalRequest, ChannelHandlerContext ctx, this.dataToCapture = EnumSet.noneOf(CaptureType.class); } + this.captureJavascriptContent = captureJavascriptContent; + // we may need to capture both the request and the response, so set up the request/response filters and delegate to them when // the corresponding filter methods are invoked. to save time and memory, only set up the capturing filters when // we actually need to capture the data. @@ -442,7 +448,7 @@ protected void captureRequestContent(HttpRequest httpRequest, byte[] fullMessage harEntry.getRequest().getPostData().setText(postBody); } } - + protected void captureResponseContent(HttpResponse httpResponse, byte[] fullMessage) { // force binary if the content encoding is not supported boolean forceBinary = false; @@ -472,10 +478,17 @@ protected void captureResponseContent(HttpResponse httpResponse, byte[] fullMess charset = BrowserMobHttpUtil.DEFAULT_HTTP_CHARSET; log.debug("No charset specified; using charset {} to decode contents from {}", charset, originalRequest.getUri()); } - + if (!forceBinary && BrowserMobHttpUtil.hasTextualContent(contentType)) { - String text = BrowserMobHttpUtil.getContentAsString(fullMessage, charset); - harEntry.getResponse().getContent().setText(text); + if (this.captureJavascriptContent && (contentType.startsWith("application/x-javascript") || contentType.startsWith("application/javascript"))) { + + String text = BrowserMobHttpUtil.getContentAsString(fullMessage, charset); + harEntry.getResponse().getContent().setText(text); + } + else if (!(contentType.startsWith("application/x-javascript") || contentType.startsWith("application/javascript"))) { + String text = BrowserMobHttpUtil.getContentAsString(fullMessage, charset); + harEntry.getResponse().getContent().setText(text); + } } else if (dataToCapture.contains(CaptureType.RESPONSE_BINARY_CONTENT)) { harEntry.getResponse().getContent().setText(BaseEncoding.base64().encode(fullMessage)); harEntry.getResponse().getContent().setEncoding("base64"); diff --git a/browsermob-legacy/src/main/java/net/lightbody/bmp/BrowserMobProxyServerLegacyAdapter.java b/browsermob-legacy/src/main/java/net/lightbody/bmp/BrowserMobProxyServerLegacyAdapter.java index b2c0ad061..ed7964a51 100644 --- a/browsermob-legacy/src/main/java/net/lightbody/bmp/BrowserMobProxyServerLegacyAdapter.java +++ b/browsermob-legacy/src/main/java/net/lightbody/bmp/BrowserMobProxyServerLegacyAdapter.java @@ -42,6 +42,12 @@ public class BrowserMobProxyServerLegacyAdapter extends BrowserMobProxyServer im */ private volatile boolean errorOnUnsupportedOperation = false; + /** + * When true will capture Javascript content + */ + + private volatile Boolean captureJavascriptContent = true; + /** * The port to start the proxy on, if set using {@link #setPort(int)}. */ @@ -393,6 +399,11 @@ public void setCaptureContent(boolean captureContent) { } } + @Deprecated + public void setCaptureJavascriptContent(Boolean captureJavascriptContent) { + this.captureJavascriptContent = captureJavascriptContent; + } + /** * @deprecated use {@link #getHarCaptureTypes()} to check for relevant {@link net.lightbody.bmp.proxy.CaptureType} */ diff --git a/browsermob-legacy/src/main/java/net/lightbody/bmp/proxy/LegacyProxyServer.java b/browsermob-legacy/src/main/java/net/lightbody/bmp/proxy/LegacyProxyServer.java index 103f2f25a..0ee9bb3d6 100644 --- a/browsermob-legacy/src/main/java/net/lightbody/bmp/proxy/LegacyProxyServer.java +++ b/browsermob-legacy/src/main/java/net/lightbody/bmp/proxy/LegacyProxyServer.java @@ -125,6 +125,8 @@ public interface LegacyProxyServer { void setCaptureHeaders(boolean captureHeaders); void setCaptureContent(boolean captureContent); + + void setCaptureJavascriptContent(Boolean captureJavascriptContent); void setCaptureBinaryContent(boolean captureBinaryContent); diff --git a/browsermob-legacy/src/main/java/net/lightbody/bmp/proxy/ProxyServer.java b/browsermob-legacy/src/main/java/net/lightbody/bmp/proxy/ProxyServer.java index 29c5f897f..a0a153630 100644 --- a/browsermob-legacy/src/main/java/net/lightbody/bmp/proxy/ProxyServer.java +++ b/browsermob-legacy/src/main/java/net/lightbody/bmp/proxy/ProxyServer.java @@ -848,6 +848,11 @@ public void setCaptureContent(boolean captureContent) { client.setCaptureContent(captureContent); } + @Override + public void setCaptureJavascriptContent(Boolean captureJavascriptContent) { + client.setCaptureJavascriptContent(captureJavascriptContent); + } + @Override public void setCaptureBinaryContent(boolean captureBinaryContent) { client.setCaptureBinaryContent(captureBinaryContent); diff --git a/browsermob-legacy/src/main/java/net/lightbody/bmp/proxy/http/BrowserMobHttpClient.java b/browsermob-legacy/src/main/java/net/lightbody/bmp/proxy/http/BrowserMobHttpClient.java index 00cdd8179..82d48e1cb 100644 --- a/browsermob-legacy/src/main/java/net/lightbody/bmp/proxy/http/BrowserMobHttpClient.java +++ b/browsermob-legacy/src/main/java/net/lightbody/bmp/proxy/http/BrowserMobHttpClient.java @@ -143,6 +143,11 @@ public class BrowserMobHttpClient { */ private volatile boolean captureContent; + /** + * keep Javascript contents + */ + private volatile Boolean captureJavascriptContent = true; + /** * keep binary contents (if captureContent is set to true, default policy is to capture binary contents too) */ @@ -1004,7 +1009,12 @@ public HeaderElement[] getElements() throws ParseException { } if (hasTextualContent(contentType)) { - setTextOfEntry(entry, copy, contentType); + if (captureJavascriptContent && ( contentType.startsWith("application/javascript") || contentType.startsWith("application/x-javascript"))) { + setTextOfEntry(entry, copy, contentType); + } + else if (!contentType.startsWith("application/javascript") && !contentType.startsWith("application/x-javascript")) { + setTextOfEntry(entry, copy, contentType); + } } else if(captureBinaryContent){ setBinaryContentOfEntry(entry, copy); } @@ -1381,6 +1391,11 @@ public void setCaptureHeaders(boolean captureHeaders) { public void setCaptureContent(boolean captureContent) { this.captureContent = captureContent; } + + public void setCaptureJavascriptContent(Boolean captureJavascriptContent) { + this.captureJavascriptContent = captureJavascriptContent; + } + public void setCaptureBinaryContent(boolean captureBinaryContent) { this.captureBinaryContent = captureBinaryContent; diff --git a/browsermob-rest/src/main/java/net/lightbody/bmp/proxy/bricks/ProxyResource.java b/browsermob-rest/src/main/java/net/lightbody/bmp/proxy/bricks/ProxyResource.java index 07654db32..45a5338c7 100644 --- a/browsermob-rest/src/main/java/net/lightbody/bmp/proxy/bricks/ProxyResource.java +++ b/browsermob-rest/src/main/java/net/lightbody/bmp/proxy/bricks/ProxyResource.java @@ -144,19 +144,26 @@ public Reply newHar(@Named("port") int port, Request request) { String initialPageRef = request.param("initialPageRef"); String initialPageTitle = request.param("initialPageTitle"); Har oldHar = proxy.newHar(initialPageRef, initialPageTitle); - String captureHeaders = request.param("captureHeaders"); String captureContent = request.param("captureContent"); String captureBinaryContent = request.param("captureBinaryContent"); + proxy.setCaptureHeaders(Boolean.parseBoolean(captureHeaders)); proxy.setCaptureContent(Boolean.parseBoolean(captureContent)); - proxy.setCaptureBinaryContent(Boolean.parseBoolean(captureBinaryContent)); String captureCookies = request.param("captureCookies"); - if (proxy instanceof BrowserMobProxyServer && Boolean.parseBoolean(captureCookies)) { + String captureJavascriptContent = request.param("captureJavascriptContent"); + + proxy.setCaptureJavascriptContent(Boolean.parseBoolean(captureJavascriptContent)); + + if (proxy instanceof BrowserMobProxyServer) { BrowserMobProxyServer browserMobProxyServer = (BrowserMobProxyServer) proxy; - browserMobProxyServer.enableHarCaptureTypes(CaptureType.getCookieCaptureTypes()); + browserMobProxyServer.captureJavascriptContent = Boolean.parseBoolean(captureJavascriptContent); + if (Boolean.parseBoolean(captureCookies)) { + browserMobProxyServer.enableHarCaptureTypes(CaptureType.getCookieCaptureTypes()); + } } + proxy.setCaptureBinaryContent(Boolean.parseBoolean(captureBinaryContent)); if (oldHar != null) { return Reply.with(oldHar).as(Json.class);