Merge pull request #207 from censys/adh/deprecate-v1

Remove Deprecated v1 API Endpoints

censys/asm/__init__.py

@@ -1,4 +1,4 @@
-"""An easy-to-use and lightweight API wrapper for Censys ASM (censys.io)."""
+"""An easy-to-use and lightweight API wrapper for Censys ASM (app.censys.io)."""
 from .assets import Assets, CertificatesAssets, DomainsAssets, HostsAssets
 from .client import AsmClient
 from .clouds import Clouds

censys/cli/commands/search.py

@@ -5,31 +5,14 @@
 from typing import List
 from urllib.parse import urlencode
 
-from censys.cli.utils import INDEXES, V1_INDEXES, V2_INDEXES, console, write_file
+from censys.cli.utils import INDEXES, V1_INDEXES, V2_INDEXES, err_console, write_file
 from censys.common.exceptions import CensysCLIException
 from censys.search import SearchClient
 
 Fields = List[str]
 Results = List[dict]
 
 DEFAULT_FIELDS = {
-    "ipv4": [
-        "updated_at",
-        "protocols",
-        "metadata.description",
-        "autonomous_system.name",
-        "23.telnet.banner.banner",
-        "80.http.get.title",
-        "80.http.get.metadata.description",
-        "8080.http.get.metadata.description",
-        "8888.http.get.metadata.description",
-        "443.https.get.metadata.description",
-        "443.https.get.title",
-        "443.https.tls.certificate.parsed.subject_dn",
-        "443.https.tls.certificate.parsed.names",
-        "443.https.tls.certificate.parsed.subject.common_name",
-        "443.https.tls.certificate.parsed.extensions.subject_alt_name.dns_names",
-    ],
     "certs": [
         "metadata.updated_at",
         "parsed.issuer.common_name",
@@ -44,15 +27,6 @@
         "metadata.seen_in_scan",
         "tags",
     ],
-    "websites": [
-        "443.https.tls.version",
-        "alexa_rank",
-        "domain",
-        "ports",
-        "protocols",
-        "tags",
-        "updated_at",
-    ],
 }
 
 
@@ -69,17 +43,12 @@ def cli_search(args: argparse.Namespace):
 
     if args.open:
         url_query = {"q": args.query}
-        if index_type in V1_INDEXES:
-            if index_type == "certs":
-                index_type = "certificates"
-                # TODO: Remove when v1 is fully deprecated
-                webbrowser.open(
-                    f"https://search.censys.io/{index_type}?{urlencode(url_query)}"
-                )
-                sys.exit(0)
-            webbrowser.open(f"https://censys.io/{index_type}?{urlencode(url_query)}")
+        if index_type in {"certs", "certificates"}:
+            webbrowser.open(
+                f"https://search.censys.io/certificates?{urlencode(url_query)}"
+            )
             sys.exit(0)
-        elif index_type in V2_INDEXES:
+        if index_type in V2_INDEXES:
             url_query.update({"resource": index_type})
             webbrowser.open(f"https://search.censys.io/search?{urlencode(url_query)}")
             sys.exit(0)
@@ -120,7 +89,7 @@ def cli_search(args: argparse.Namespace):
 
         search_args["fields"] = fields
 
-        with console.status("Searching"):
+        with err_console.status("Searching"):
             results = list(index.search(args.query, **search_args))
     elif index_type in V2_INDEXES:
         if args.format == "csv":
@@ -132,7 +101,7 @@ def cli_search(args: argparse.Namespace):
         if args.pages:
             search_args["pages"] = args.pages
 
-        with console.status("Searching"):
+        with err_console.status("Searching"):
             query = index.search(args.query, **search_args)
 
             results = []
@@ -142,7 +111,7 @@ def cli_search(args: argparse.Namespace):
     try:
         write_file(results, **write_args)
     except ValueError as error:  # pragma: no cover
-        console.print(f"Error writing log file. Error: {error}")
+        err_console.print(f"Error writing log file. Error: {error}")
 
 
 def include(parent_parser: argparse._SubParsersAction, parents: dict):

censys/cli/utils.py

@@ -4,6 +4,7 @@
 import datetime
 import json
 import os.path
+import sys
 from typing import Any, Dict, List, Optional, Union
 
 from rich.console import Console
@@ -14,13 +15,15 @@
 Fields = List[str]
 Results = Union[List[dict], Dict[str, Any]]
 
-V1_INDEXES = ["ipv4", "certs", "websites"]
+V1_INDEXES = ["certs"]
 V2_INDEXES = ["hosts"]
 INDEXES = V1_INDEXES + V2_INDEXES
 
 config = get_config()
 color = config.get(DEFAULT, "color")
-console = Console(color_system=("auto" if color else None))
+color_system = "auto" if color else None
+console = Console(color_system=color_system)  # type: ignore
+err_console = Console(color_system=color_system, file=sys.stderr)  # type: ignore
 
 
 def print_wrote_file(file_path: str):

censys/search/__init__.py

@@ -1,15 +1,13 @@
-"""An easy-to-use and lightweight API wrapper for Censys Search API (censys.io)."""
+"""An easy-to-use and lightweight API wrapper for Censys Search API (search.censys.io)."""
 from .client import SearchClient
-from .v1 import CensysCertificates, CensysData, CensysIPv4, CensysWebsites
+from .v1 import CensysCertificates, CensysData
 from .v2 import CensysCerts, CensysHosts
 
 __copyright__ = "Copyright 2021 Censys, Inc."
 __all__ = [
     "SearchClient",
     "CensysCertificates",
     "CensysData",
-    "CensysIPv4",
-    "CensysWebsites",
     "CensysCerts",
     "CensysHosts",
 ]

censys/search/client.py

@@ -1,6 +1,6 @@
 """Interact with all Search APIs."""
 from ..common.deprecation import DeprecationDecorator
-from .v1 import CensysCertificates, CensysData, CensysIPv4, CensysWebsites
+from .v1 import CensysCertificates, CensysData
 from .v2 import CensysCerts, CensysHosts
 
 
@@ -19,8 +19,6 @@ class SearchClient:
 
         >>> certs = c.v1.certificates # CensysCertificates()
         >>> data = c.v1.data # CensysData()
-        >>> ipv4 = c.v1.ipv4 # CensysIPv4()
-        >>> websites = c.v1.websites # CensysWebsites()
         >>> hosts = c.v2.hosts # CensysHosts()
         >>> certs = c.v2.certs # CensysCerts()
     """
@@ -42,8 +40,6 @@ def __init__(self, *args, **kwargs):
             # Alias certs to certificates
             self.certs = self.certificates
             self.data = CensysData(*args, **kwargs)
-            self.ipv4 = CensysIPv4(*args, **kwargs)
-            self.websites = CensysWebsites(*args, **kwargs)
 
     class _V2:
         """Class for v2 Search APIs."""

censys/search/v1/__init__.py

@@ -1,7 +1,5 @@
 """Interact with the Censys Search v1 APIs."""
 from .certificates import CensysCertificates
 from .data import CensysData
-from .ipv4 import CensysIPv4
-from .websites import CensysWebsites
 
-__all__ = ["CensysCertificates", "CensysData", "CensysIPv4", "CensysWebsites"]
+__all__ = ["CensysCertificates", "CensysData"]

censys/search/v1/api.py

@@ -18,7 +18,7 @@
 class CensysSearchAPIv1(CensysAPIBase):
     """This class is the base class for all v1 API indexes."""
 
-    DEFAULT_URL: str = "https://censys.io/api/v1"
+    DEFAULT_URL: str = "https://search.censys.io/api/v1"
     """Default Search API base URL."""
     INDEX_NAME: Optional[str] = None
     """Name of Censys Index."""
@@ -112,7 +112,7 @@ def search(
     ) -> Iterator[dict]:
         """Searches the given index for all records that match the given query.
 
-        For more details, see our documentation: https://censys.io/api/v1/docs/search
+        For more details, see our documentation: https://search.censys.io/api
 
         Args:
             query (str): The query to be executed.
@@ -152,7 +152,7 @@ def search(
     def view(self, document_id: str) -> dict:
         """View the current structured data we have on a specific document.
 
-        For more details, see our documentation: https://censys.io/api/v1/docs/view
+        For more details, see our documentation: https://search.censys.io/api
 
         Args:
             document_id (str): The ID of the document you are requesting.
@@ -165,7 +165,7 @@ def view(self, document_id: str) -> dict:
     def report(self, query: str, field: str, buckets: int = 50) -> dict:
         """Creates a report on the breakdown of the values of a field in a result set.
 
-        For more details, see our documentation: https://censys.io/api/v1/docs/report
+        For more details, see our documentation: https://search.censys.io/api
 
         Args:
             query (str): The query to be executed.
@@ -177,6 +177,3 @@ def report(self, query: str, field: str, buckets: int = 50) -> dict:
         """
         data = {"query": query, "field": field, "buckets": int(buckets)}
         return self._post(self.report_path, data=data)
-
-
-CensysSearchAPI = CensysSearchAPIv1

censys/search/v1/data.py

@@ -5,7 +5,7 @@
 class CensysData(CensysSearchAPIv1):
     """Interacts with the Data index.
 
-    For more details, see our documentation: https://censys.io/api/v1/docs/data
+    For more details, see our documentation: https://search.censys.io/api
     """
 
     _PREFIX = "/data"

censys/search/v2/api.py

@@ -14,7 +14,6 @@
 )
 from censys.common.types import Datetime
 from censys.common.utils import format_rfc3339
-from censys.search.v1.api import CensysSearchAPIv1
 
 Fields = Optional[List[str]]
 
@@ -28,12 +27,10 @@ class CensysSearchAPIv2(CensysAPIBase):
         >>> c = CensysSearchAPIv2()
     """
 
-    DEFAULT_URL: str = "https://search.censys.io/api/v2"
+    DEFAULT_URL: str = "https://search.censys.io/api"
     """Default Search API base URL."""
     INDEX_NAME: str = ""
     """Name of Censys Index."""
-    v1: CensysSearchAPIv1
-    """Search V1 Endpoints on V2"""
 
     def __init__(
         self, api_id: Optional[str] = None, api_secret: Optional[str] = None, **kwargs
@@ -70,22 +67,12 @@ def __init__(
         self._session.auth = (self._api_id, self._api_secret)
 
         # Generate concrete paths to be called
-        self.view_path = f"/{self.INDEX_NAME}/"
-        self.search_path = f"/{self.INDEX_NAME}/search"
-        self.aggregate_path = f"/{self.INDEX_NAME}/aggregate"
-        self.metadata_path = f"/metadata/{self.INDEX_NAME}"
-        self.tags_path = "/tags"
-
-        # Set up the v1 API
-        v1_kwargs = kwargs.copy()
-        v1_kwargs.update(
-            {
-                "url": "https://search.censys.io/api/v1",
-                "api_id": self._api_id,
-                "api_secret": self._api_secret,
-            }
-        )
-        self.v1 = CensysSearchAPIv1(**v1_kwargs)
+        self.view_path = f"/v2/{self.INDEX_NAME}/"
+        self.search_path = f"/v2/{self.INDEX_NAME}/search"
+        self.aggregate_path = f"/v2/{self.INDEX_NAME}/aggregate"
+        self.metadata_path = f"/v2/metadata/{self.INDEX_NAME}"
+        self.tags_path = "/v2/tags"
+        self.account_path = "/v1/account"
 
     def _get_exception_class(  # type: ignore
         self, res: Response
@@ -100,8 +87,7 @@ def account(self) -> dict:
         Returns:
             dict: Quota response.
         """
-        # Make account call to v1 endpoint
-        return self.v1.account()
+        return self._get(self.account_path)
 
     def quota(self) -> dict:
         """Returns metadata of a given search query.
@@ -190,11 +176,14 @@ def __iter__(self) -> Iterator[List[dict]]:
             """
             return self
 
-        def view_all(self) -> Dict[str, dict]:
+        def view_all(self, max_workers: int = 20) -> Dict[str, dict]:
             """View each document returned from query.
 
             Please note that each result returned by the query will be looked up using the view method.
 
+            Args:
+                max_workers (int): The number of workers to use. Defaults to 20.
+
             Returns:
                 Dict[str, dict]: Dictionary mapping documents to that document's result set.
             """
@@ -203,7 +192,7 @@ def view_all(self) -> Dict[str, dict]:
 
             document_key = INDEX_TO_KEY.get(self.api.INDEX_NAME, "ip")
 
-            with ThreadPoolExecutor(max_workers=20) as executor:
+            with ThreadPoolExecutor(max_workers) as executor:
                 for hit in self.__call__():
                     document_id = hit[document_key]
                     threads.append(executor.submit(self.api.view, document_id))
@@ -261,6 +250,42 @@ def view(
 
         return self._get(self.view_path + document_id, args)["result"]
 
+    def bulk_view(
+        self,
+        document_ids: List[str],
+        at_time: Optional[Datetime] = None,
+        max_workers: int = 20,
+    ) -> Dict[str, dict]:
+        """Bulk view documents from current index.
+
+        View the current structured data we have on a list of documents.
+        For more details, see our documentation: https://search.censys.io/api
+
+        Args:
+            document_ids (List[str]): The IDs of the documents you are requesting.
+            at_time ([str, datetime.date, datetime.datetime]):
+                Optional; Fetches a document at a given point in time.
+            max_workers (int): The number of workers to use. Defaults to 20.
+
+        Returns:
+            Dict[str, dict]: Dictionary mapping document IDs to that document's result set.
+        """
+        args = {}
+        if at_time:
+            args["at_time"] = format_rfc3339(at_time)
+
+        threads = []
+        documents = {}
+        with ThreadPoolExecutor(max_workers) as executor:
+            for document_id in document_ids:
+                threads.append(executor.submit(self.view, document_id, at_time))
+
+            for task in as_completed(threads):
+                result = task.result()
+                documents[result["ip"]] = result
+
+        return documents
+
     def aggregate(
         self, query: str, field: str, num_buckets: Optional[int] = None
     ) -> dict:

censys/search/v2/hosts.py

@@ -115,9 +115,9 @@ def view_host_events(
         if end_time:
             args["end_time"] = format_rfc3339(end_time)
 
-        return self._get(f"/experimental/{self.INDEX_NAME}/{ip_address}/events", args)[
-            "result"
-        ]["events"]
+        return self._get(
+            f"/v2/experimental/{self.INDEX_NAME}/{ip_address}/events", args
+        )["result"]["events"]
 
     def list_hosts_with_tag(self, tag_id: str) -> List[str]:
         """Returns a list of hosts which are tagged with the specified tag.