Merge pull request #53 from NREL/pp/ignore_https_errors

Add option to ignore https errors

Author: ppinchuk

elm/web/html_pw.py

@@ -67,7 +67,9 @@ async def _load_html(  # pragma: no cover
         logger.trace("launching chromium; browser_semaphore=%r",
                      browser_semaphore)
         browser = await p.chromium.launch(**pw_launch_kwargs)
-        async with pw_page(browser, intercept_routes=True) as page:
+        page_kwargs = {"browser": browser, "intercept_routes": True,
+                       "ignore_https_errors": True}  # no sensitive inputs
+        async with pw_page(**page_kwargs) as page:
             logger.trace("Navigating to: %r", url)
             await page.goto(url)
             logger.trace("Waiting for load with timeout: %r", timeout)

elm/web/search/base.py

@@ -122,7 +122,8 @@ async def _search(self, query, num_results=10):
         logger.debug("Searching %s: %r", self._SE_NAME, query)
         num_results = min(num_results, self.MAX_RESULTS_CONSIDERED_PER_PAGE)
 
-        page_kwargs = {"browser": self._browser, "stealth_config": self._SC}
+        page_kwargs = {"browser": self._browser, "stealth_config": self._SC,
+                       "ignore_https_errors": True}  # no sensitive inputs
         async with pw_page(**page_kwargs) as page:
             await _navigate_to_search_engine(page, se_url=self._SE_URL,
                                              timeout=self.PAGE_LOAD_TIMEOUT)

elm/web/utilities.py

@@ -170,7 +170,8 @@ def write_url_doc_to_file(doc, file_content, out_dir, make_name_unique=False):
 
 
 @asynccontextmanager
-async def pw_page(browser, intercept_routes=False, stealth_config=None):
+async def pw_page(browser, intercept_routes=False, stealth_config=None,
+                  ignore_https_errors=False):
     """Create new page from playwright browser context
 
     Parameters
@@ -184,6 +185,13 @@ async def pw_page(browser, intercept_routes=False, stealth_config=None):
         Optional playwright stealth configuration object.
         By default, ``None``, which uses all the default stealth
         options.
+    ignore_https_errors : bool, default=False
+        Option to ignore https errors (i.e. SSL cert errors). This is
+        not generally safe to do - you are susceptible to MITM attacks.
+        However, if you are doing a simple scrape without providing
+        any sensitive information (which you probably shouldn't be doing
+        programmatically anyways), then it's probably ok to ignore these
+        errors. By default, ``False``.
 
     Yields
     ------
@@ -202,6 +210,7 @@ async def pw_page(browser, intercept_routes=False, stealth_config=None):
         extra_http_headers=DEFAULT_HEADERS,
         user_agent=ua,
         viewport={"width": randint(800, 1400), "height": randint(800, 1400)},
+        ignore_https_errors=ignore_https_errors,
     )
 
     try: