Fix code format

archive_query_log/cli/corpus.py

@@ -275,7 +275,7 @@ def _build_search_result(
         .get(archived_search_result_snippet.id)
     # archived_parsed_search_result_loc = archived_parsed_search_result_index \
     #     .get(archived_search_result_snippet.id)
-    archived_parsed_search_result_loc = None
+    # archived_parsed_search_result_loc = None
     return CorpusSearchResult(
         id=archived_search_result_snippet.id,
         url=archived_search_result_snippet.url,

archive_query_log/cli/external.py

@@ -3,7 +3,6 @@
 from pathlib import Path
 from re import compile, escape
 from urllib.parse import quote
-from numpy import nan
 
 from click import argument
 from pandas import DataFrame, read_csv, Series, concat
@@ -20,6 +19,7 @@
 sheet_query_parsers = "Query Parsers"
 sheet_page_parsers = "Page Parsers"
 
+
 @main.group("external")
 def external():
     pass
@@ -90,6 +90,7 @@ def load_query_parsers() -> DataFrame:
     df["query_parser"] = df["value"]
     return df[["name", "query_parser"]]
 
+
 def load_page_offset_parsers() -> DataFrame:
     df = from_sheets(sheet_page_parsers, transpose=True)
     df["value"].replace("NULL", "{}", inplace=True)
@@ -128,9 +129,12 @@ def query_parser(row: Series) -> dict:
     else:
         raise NotImplementedError()
 
+
 page_offset_parser_map = {"parameter": "query_parameter",
                           "suffix": "path_suffix",
                           "fragment": "fragment_parameter"}
+
+
 def page_offset_parser(row: Series, count="results") -> dict:
     row = row.to_dict()
     row.update(loads(row["page_offset_parser"]))
@@ -140,9 +144,10 @@ def page_offset_parser(row: Series, count="results") -> dict:
             "url_pattern": url_pattern,
             "type": page_offset_parser_map[row["type"]],
             "parameter": row["key"]
-            }
+        }
     else:
-        return NotImplementedError()
+        raise NotImplementedError()
+
 
 def page_offset_parser_series(page_offset_parsers, services, count):
     return [
@@ -151,12 +156,15 @@ def page_offset_parser_series(page_offset_parsers, services, count):
             for _, row in
             page_offset_parsers[
                 (page_offset_parsers["name"].str.fullmatch(service["name"])) &
-                (page_offset_parsers["page_offset_parser"].str.contains(f'"count": "{count}"'))
-            ].iterrows()
+                (page_offset_parsers["page_offset_parser"].str.contains(
+                    f'"count": "{count}"'
+                ))
+                ].iterrows()
         ), key=lambda pp: str(pp["url_pattern"]))
         for _, service in services.iterrows()
     ]
 
+
 @external.command("import-services")
 @argument(
     "services-file",
@@ -200,8 +208,12 @@ def import_services(services_file: Path):
             load_page_offset_parsers()[["page_offset_parser"]]
         ],
         axis="columns")
-    services["page_parsers"] = page_offset_parser_series(page_offset_parsers, services, count="pages")
-    services["offset_parsers"] = page_offset_parser_series(page_offset_parsers, services, count="results")
+    services["page_parsers"] = page_offset_parser_series(
+        page_offset_parsers, services, count="pages"
+    )
+    services["offset_parsers"] = page_offset_parser_series(
+        page_offset_parsers, services, count="results"
+    )
     services["interpreted_query_parsers"] = [
         []
         for _, service in services.iterrows()

archive_query_log/cli/stats.py

@@ -204,11 +204,13 @@ def stats_command(
                     if document["archived_snippet_location"] is not None:
                         results[service_name]["archived-snippets"] += 1
                     if document[
-                        "archived_raw_search_result_location"] is not None:
+                        "archived_raw_search_result_location"
+                    ] is not None:
                         results[service_name][
                             "archived-raw-search-results"] += 1
                     if document[
-                        "archived_parsed_search_result_location"] is not None:
+                        "archived_parsed_search_result_location"
+                    ] is not None:
                         results[service_name][
                             "archived-parsed-search-results"] += 1
 

archive_query_log/download/warc.py

@@ -365,7 +365,7 @@ async def download_service(
         if focused:
             pages = tqdm(
                 pages,
-                desc=f"Deduplicate query URLs",
+                desc="Deduplicate query URLs",
                 unit="page",
             )
 

archive_query_log/results/test/generate_tests.py

@@ -149,6 +149,7 @@ def main():
         if not test_path.exists():
             with test_path.open("wt") as o:
                 o.write(dedent("""
+                # flake8: noqa
                 # This file is auto-generated by generate_tests.py.
                 """).lstrip())
                 o.write(dedent("""

archive_query_log/results/test/test_360_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_aliexpress_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_amazon_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_baidu_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_bing_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_bongacams_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_canva_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_cnn_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_csdn_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_ebay_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_espn_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_etsy_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_facebook_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_github_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_google_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_imdb_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_imgur_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_indeed_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_jd_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_linkedin_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_manual_facebook_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 
 

archive_query_log/results/test/test_manual_google_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 
 

archive_query_log/results/test/test_manual_youtube_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 
 

archive_query_log/results/test/test_naver_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_pornhub_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_qq_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_reddit_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_roblox_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_sogou_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_stackoverflow_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_tribunnews_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_twitch_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_twitter_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_vk_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_weibo_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_wikimedia_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_xvideos_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_yahoo_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_yandex_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/results/test/test_youtube_serp_parsing.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 # This file is auto-generated by generate_tests.py.
 from archive_query_log.results.test.test_utils import verify_serp_parsing
 

archive_query_log/service_stats.py

@@ -26,7 +26,8 @@
         for service in SERVICES.values()
     )
     print(
-        f"Number of interpreted query parsers: {num_interpreted_query_parsers}")
+        f"Number of interpreted query parsers: {num_interpreted_query_parsers}"
+    )
     num_results_parsers = sum(
         len(service.results_parsers)
         for service in SERVICES.values()

archive_query_log/services/__init__.py

@@ -6,7 +6,9 @@
 from archive_query_log.model import Service
 
 
-def read_services(path: Path, ignore_parsing_errors=True) -> Mapping[str, Service]:
+def read_services(
+        path: Path, ignore_parsing_errors=True
+) -> Mapping[str, Service]:
     with path.open("r") as file:
         services_dict = safe_load(file)
         services = []

archive_query_log/services/search_forms.py

@@ -98,10 +98,12 @@ def check_url(self, url: str):
         # Look for elements with the pattern in them and save the snippets
         soup = BeautifulSoup(html, 'html.parser')
         found_input, input_snippets = find_input_tag(soup=soup)
-        found_search_form, form_snippets = find_search_tag(soup=soup, tag='form')
+        found_search_form, form_snippets = find_search_tag(
+            soup=soup, tag='form')
         found_search_div, div_snippets = find_search_tag(soup=soup, tag='div')
 
-        return found_input, found_search_form, found_search_div, input_snippets, form_snippets, div_snippets
+        return found_input, found_search_form, found_search_div, \
+            input_snippets, form_snippets, div_snippets
 
     def get_internet_archive_html(self, url: str, year=2022, byte_digits=4):
         """
@@ -170,24 +172,28 @@ def find_search_tag(soup: BeautifulSoup, tag='form'):
     return found, snippet_list
 
 
-
 if __name__ == "__main__":
     # Parse input
-    parser = argparse.ArgumentParser(prog='Search form identification',
-                                     description='Takes in a CSV-File of services and looks for search forms '
-                                                 'in their HTML')
+    parser = argparse.ArgumentParser(
+        prog='Search form identification',
+        description='Takes in a CSV-File of services '
+                    'and looks for search forms in their HTML')
     parser.add_argument('-f', '--csv_file', type=str)
     parser.add_argument('-o', '--outfile_num', type=str)
     parser.add_argument('-s', '--start_row', type=int)
     parser.add_argument('-e', '--end_row', type=int)
     args = parser.parse_args()
 
     # Set/Update default values
-    csv_file = './alexa-top-1m-fused-domains-rrf-top-10000.csv' if args.csv_file is None else args.csv_file
+    csv_file = './alexa-top-1m-fused-domains-rrf-top-10000.csv' \
+        if args.csv_file is None else args.csv_file
     outfile_num = "0" if args.outfile_num is None else args.outfile_num
     start_row = 0 if args.start_row is None else args.start_row
     end_row = None if args.end_row is None else args.end_row
 
     # Run the search for specified services
-    identifier = SearchFormIdentifier(csv_file=csv_file, outfile_num=outfile_num, start_row=start_row, end_row=end_row)
+    identifier = SearchFormIdentifier(
+        csv_file=csv_file, outfile_num=outfile_num,
+        start_row=start_row, end_row=end_row
+    )
     identifier.process_services()