export-data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Please install and enable the following extensions to correctly display this notebook:\n",
    "  - [ToC2 jupyter extension](https://jupyter-contrib-nbextensions.readthedocs.io/en/latest/nbextensions/toc2/README.html)\n",
    "  - [Collapsible Headings](https://jupyter-contrib-nbextensions.readthedocs.io/en/latest/nbextensions/collapsible_headings/readme.html)\n",
    "2. Python 3.8.2+ is required."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width: 80% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width: 80% !important; }</style>\"))\n",
    "\n",
    "from jupyterthemes import jtplot\n",
    "jtplot.style(theme='chesterish', context='notebook', ticks=True, grid=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocess"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import multiprocessing as mp\n",
    "import json, datetime, dateutil, sqlite3, pickle, hashlib, re, datetime\n",
    "from pathlib import Path\n",
    "from collections import defaultdict, OrderedDict\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib.parse import urlparse as _urlparse\n",
    "# https://stackoverflow.com/a/43609542/11712282\n",
    "def urlparse(url):\n",
    "    parsed = _urlparse(url)\n",
    "    if parsed.netloc.endswith(':80'):\n",
    "        parsed = parsed._replace(netloc=parsed.netloc[:-len(':80')])\n",
    "    elif parsed.netloc.endswith(':443'):\n",
    "        parsed = parsed._replace(netloc=parsed.netloc[:-len(':443')])\n",
    "    return parsed\n",
    "\n",
    "def parallel(func, data, process_num=None, chunksize=None, total=None, desc=None):\n",
    "    process_num = mp.cpu_count() if process_num is None else process_num\n",
    "    chunksize = (total // (process_num * 32) + 1) if chunksize is None else chunksize\n",
    "    print(f'Parallel {process_num=} {chunksize=}')\n",
    "    with mp.Pool(process_num) as p:\n",
    "        for res in tqdm(p.imap_unordered(func, data, chunksize=chunksize), total=total, desc=desc):\n",
    "            yield res\n",
    "\n",
    "from publicsuffixlist import PublicSuffixList\n",
    "psl = PublicSuffixList()\n",
    "\n",
    "def get_ps_domain(dm):\n",
    "    return psl.privatesuffix(dm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "class TrieSet:\n",
    "    def __init__(self, _dms, include_subdomain=False):\n",
    "        self._set = set(_dms)\n",
    "        self.trie = {}\n",
    "        if include_subdomain:\n",
    "            dms = ['*.' + i for i in self._set] + list(self._set)\n",
    "        else:\n",
    "            dms = self._set\n",
    "        for dm in dms:\n",
    "            cur_dic = self.trie\n",
    "            nodes = dm.split('.')\n",
    "            if len(nodes) <= 1 or ('*' in dm and nodes[0] != '*'):\n",
    "                #print('ignore', repr(dm))\n",
    "                continue\n",
    "            for node in reversed(nodes):\n",
    "                if '*' in cur_dic:\n",
    "                    cur_dic = cur_dic['*']\n",
    "                    break\n",
    "                if node not in cur_dic:\n",
    "                    cur_dic[node] = {}\n",
    "                cur_dic = cur_dic[node]\n",
    "            if '.value' in cur_dic:\n",
    "                pass\n",
    "                #print('dup?', dm, json.dumps(cur_dic, indent=2))\n",
    "            cur_dic['.value'] = True\n",
    "\n",
    "    def __contains__(self, dm):\n",
    "        cur_dic = self.trie\n",
    "        nodes = dm.split('.')\n",
    "        if len(nodes) <= 1 or '*' in dm:\n",
    "            return False\n",
    "        for node in reversed(nodes):\n",
    "            if '*' in cur_dic:\n",
    "                return True\n",
    "            if node not in cur_dic:\n",
    "                return False\n",
    "            cur_dic = cur_dic[node]\n",
    "        if '.value' not in cur_dic:\n",
    "            return False\n",
    "        assert cur_dic['.value'] == True\n",
    "        return True\n",
    "    \n",
    "    def __iter__(self):\n",
    "        return iter(self._set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sqlalchemy import create_engine\n",
    "from sqlalchemy.orm import sessionmaker, scoped_session\n",
    "from sqlalchemy.ext.declarative import declarative_base\n",
    "from sqlalchemy import Column, BigInteger, String, Text, DateTime, Boolean\n",
    "\n",
    "'''\n",
    "+----------------+--------------+------+-----+-------------------+-------------------+\n",
    "| Field          | Type         | Null | Key | Default           | Extra             |\n",
    "+----------------+--------------+------+-----+-------------------+-------------------+\n",
    "| id             | bigint       | NO   | PRI | NULL              | auto_increment    |\n",
    "| domain         | varchar(256) | NO   | MUL | NULL              |                   |\n",
    "| datetime       | datetime     | NO   |     | NULL              |                   |\n",
    "| request        | longtext     | NO   |     | NULL              |                   |\n",
    "| request_raw    | longtext     | NO   |     | NULL              |                   |\n",
    "| initiator      | longtext     | NO   |     | NULL              |                   |\n",
    "| initiator_raw  | longtext     | NO   |     | NULL              |                   |\n",
    "| crawl_datetime | datetime     | NO   |     | CURRENT_TIMESTAMP | DEFAULT_GENERATED |\n",
    "+----------------+--------------+------+-----+-------------------+-------------------+\n",
    "\n",
    "2016-05-17 00:00:12 - 2020-03-28 23:57:55\n",
    "'''\n",
    "\n",
    "Base = declarative_base()\n",
    "class Request(Base):\n",
    "    __tablename__ = 'requests'\n",
    "    id = Column(BigInteger, primary_key=True)\n",
    "    domain = Column(String)\n",
    "    request = Column(Text)\n",
    "    datetime = Column(DateTime)\n",
    "    initiator = Column(Text)\n",
    "    blocked = Column(Boolean)\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return f'{self.domain}: {self.datetime} {self.request}'\n",
    "\n",
    "# Thread-local Sessions object\n",
    "class Session():\n",
    "    def __init__(self):\n",
    "        self.engine = create_engine('mysql+pymysql://nslab:8WurxctvQjEExG5K@docker_db_1/wayback_machine_crawl')\n",
    "        #self.engine = create_engine('mysql://root:PLN7KU3LX3TCLYj2@127.0.0.1/wayback_machine_crawl')\n",
    "        self._Session = scoped_session(sessionmaker(bind=self.engine))\n",
    "        self.session = self._Session()\n",
    "    def __enter__(self):\n",
    "        return self.session\n",
    "    def __exit__(self, type, value, traceback):\n",
    "        self.session.close()\n",
    "        self._Session.remove()\n",
    "        self._Session.close()\n",
    "        self.engine.dispose()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "with Session() as session:\n",
    "    count = session.query(Request.id).count()\n",
    "    print(count)\n",
    "\"\"\";"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data from cache......\n",
      "Done\n"
     ]
    }
   ],
   "source": [
    "n = 100_000\n",
    "\n",
    "def func(i):\n",
    "    rows = []\n",
    "    for row in session.query(Request).slice(i, i+n).all():\n",
    "        site, time, url = row.domain, int(row.datetime.timestamp()), row.request\n",
    "        dm = urlparse(url).netloc\n",
    "        if dm == 'web.archive.org':\n",
    "            continue\n",
    "        rows.append((site,time,url))\n",
    "    return rows\n",
    "\n",
    "cache = Path('data/site2time_parsed.pickle')\n",
    "if cache.is_file():\n",
    "    print('Loading data from cache......')\n",
    "    site2time_parsed = pickle.loads(cache.read_bytes())\n",
    "    print('Done')\n",
    "else:\n",
    "    with Session() as session:\n",
    "        site2timeurls = {}\n",
    "        for rows in parallel(func, range(0, count, n), chunksize=1, total=count//n+1):\n",
    "            for site, time, url in rows:\n",
    "                if site not in site2timeurls:\n",
    "                    site2timeurls[site] = {}\n",
    "                time2urls = site2timeurls[site]\n",
    "                if time not in time2urls:\n",
    "                    time2urls[time] = set()\n",
    "                time2urls[time].add(url)\n",
    "        with open(cache, 'wb') as f:\n",
    "            pickle.dump(site2timeurls, f)\n",
    "        \n",
    "        site2timeurls = pickle.loads(cache.read_bytes())\n",
    "        def func(site):\n",
    "            return site, {\n",
    "                time: {urlparse(url) for url in urls}\n",
    "                for time, urls in site2timeurls[site].items()\n",
    "            }\n",
    "\n",
    "        site2time_parsed = {}\n",
    "        for site, time2urls in parallel(func, site2timeurls, total=len(site2timeurls)):\n",
    "            site2time_parsed[site] = time2urls\n",
    "\n",
    "        with open('data/site2time_parsed.pickle', 'wb') as f:\n",
    "            pickle.dump(site2time_parsed, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#site2blocked2t = pickle.loads(Path('data/site2blocked2t.pickle').read_bytes())\n",
    "site2_ad_parsed2t = pickle.loads(Path('data/site2_ad_parsed2t.pickle').read_bytes())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a5c76fe7a41d4fa18471907bc2abb21e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/46922 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "all_urls = set()\n",
    "\n",
    "class URL:\n",
    "    def __init__(self, parsed, request_from, time):\n",
    "        self.parsed = parsed\n",
    "        self.request_from = request_from\n",
    "        self.is_ad = parsed in site2_ad_parsed2t[site]\n",
    "        self.time = time\n",
    "    \n",
    "    def __eq__(self, other):\n",
    "        return self.parsed == other.parsed and self.request_from == other.request_from and self.time == other.time\n",
    "        \n",
    "    def __hash__(self):\n",
    "        return hash((self.parsed, self.request_from, self.time))\n",
    "        \n",
    "for site, time_parsed in tqdm(site2time_parsed.items()):\n",
    "    for time, parsed_urls in time_parsed.items():\n",
    "        all_urls.update({URL(u, request_from=site, time=time) for u in parsed_urls})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "61824\n"
     ]
    }
   ],
   "source": [
    "with open('data/merged_history.json') as f:\n",
    "    ad_dm_history = json.load(f)\n",
    "with open('data/domain_list.json') as f:\n",
    "    all_ad_dms = set(json.load(f))\n",
    "\n",
    "all_ad_dm_history = {\n",
    "    **{ad_dm: dateutil.parser.isoparse(dt) for ad_dm, dt in ad_dm_history.items()},\n",
    "    **{ad_dm: datetime.datetime.fromtimestamp(0) for ad_dm in all_ad_dms if ad_dm not in ad_dm_history}\n",
    "}\n",
    "\n",
    "assert all('*' not in dm and not dm.startswith('.') for dm in all_ad_dm_history)\n",
    "\n",
    "ad_dms = TrieSet([\n",
    "    ad_dm for ad_dm, dt in all_ad_dm_history.items()\n",
    "], include_subdomain=True)\n",
    "\n",
    "print(len(ad_dms._set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "+------+------------+----+---+-------+--------------+\n",
    "|Field |Type        |Null|Key|Default|Extra         |\n",
    "+------+------------+----+---+-------+--------------+\n",
    "|id    |bigint      |NO  |PRI|NULL   |auto_increment|\n",
    "|domain|varchar(256)|YES |   |NULL   |              |\n",
    "|result|text        |YES |   |NULL   |              |\n",
    "+------+------------+----+---+-------+--------------+\n",
    "\n",
    "'''\n",
    "\n",
    "Base = declarative_base()\n",
    "class Crtsh(Base):\n",
    "    __tablename__ = 'crtsh'\n",
    "    id = Column(BigInteger, primary_key=True)\n",
    "    domain = Column(String)\n",
    "    result = Column(Text)\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return f'{self.domain}: {self.result}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "217979"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cache = Path('data/dm2cert_names.pickle')\n",
    "if cache.is_file():\n",
    "    dm2cert_names = pickle.loads(cache.read_bytes())\n",
    "else:\n",
    "    with Session() as session:\n",
    "        rows = session.query(Crtsh).all()\n",
    "    to_ts = lambda iso: int(datetime.datetime.fromisoformat(iso).timestamp())\n",
    "    dm2cert_names = {}\n",
    "    \n",
    "    for d in tqdm(rows):\n",
    "        names = set()\n",
    "        res = d.result\n",
    "        dm = d.domain\n",
    "        \n",
    "        for cert in json.loads(res):\n",
    "            start_t, end_t = to_ts(cert['not_before']), to_ts(cert['not_after'])\n",
    "            #if start_t > T.timestamp():\n",
    "            #    continue\n",
    "            for key in ('subject_alt_name','name_values','common_name'):\n",
    "                if cert.get(key) is not None:\n",
    "                    names.update({name.strip() for name in cert[key].split(',') if name != ''})\n",
    "        \n",
    "        if names:\n",
    "            dm2cert_names[dm] = names\n",
    "\n",
    "    cache.write_bytes(pickle.dumps(dm2cert_names))\n",
    "len(dm2cert_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\"\"\"cache = Path('data/url2response.pickle')\n",
    "if cache.is_file():\n",
    "    url2response = pickle.loads(cache.read_bytes())\n",
    "else:\n",
    "    url2response = dict()\n",
    "    files = [\"./data-collection/fp/responses-before_august.txt\", \"./data-collection/fp/responses-august.txt\"]\n",
    "    for file in files:\n",
    "        print(file)\n",
    "        with open(Path(file), 'r') as f:\n",
    "            for line in tqdm(f.readlines()):\n",
    "                if line.strip() != \"\":\n",
    "                    url2response[line.split(\" {\")[0].strip()] = json.loads(\" {\" + \" {\".join(line.split(\" {\")[1:]))\n",
    "\n",
    "    for response in tqdm(url2response.values()):\n",
    "        if response['ok']:\n",
    "            response['content'] = hashlib.md5(response['content'].encode()).hexdigest()\n",
    "    cache.write_bytes(pickle.dumps(url2response))\"\"\";"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data from cache......\n",
      "Done\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "{'', 'CAA', 'DS', 'CNAME', 'MX', 'NSEC3PARAM', 'AAAA', 'TXT',\n",
    "'PTR', 'AFSDB', 'MB', 'LOC', 'WKS', 'SOA', 'HINFO', 'SPF',\n",
    "'DNAME', 'RP', 'RRSIG', 'SSHFP', 'DNSKEY', 'NS', 'CDNSKEY',\n",
    "'A', 'CDS', 'SRV', 'NAPTR', 'NSEC'}\n",
    "'''\n",
    "cache = Path('data/dm2dns.pickle')\n",
    "if cache.is_file():\n",
    "    print('Loading data from cache......')\n",
    "    dm2dns = pickle.loads(cache.read_bytes())\n",
    "    print('Done')\n",
    "else:\n",
    "    dm2dns = {}\n",
    "    with open('data/dm2dns.json') as f:\n",
    "        for raw_json in tqdm(re.sub(r'\\n}{\\n', '}!FOO@BAR#BAZZ${', f.read()).split('!FOO@BAR#BAZZ$')):\n",
    "            jsn = json.loads(raw_json)\n",
    "            if jsn['dns_server'] != '8.8.8.8':\n",
    "                continue\n",
    "            dm, typ = jsn['domain'], jsn['rdtype']\n",
    "            if dm not in dm2dns:\n",
    "                dm2dns[dm] = {}\n",
    "            assert typ not in dm2dns[dm]\n",
    "            dm2dns[dm][typ] = jsn['answers']\n",
    "    \n",
    "    #interested_types = {'NSEC3PARAM', 'DNSKEY', 'A', 'AAAA', 'CNAME', 'PTR', 'SPF'}\n",
    "    interested_types = {'A', 'AAAA', 'CNAME'}\n",
    "    def parse_all_answers(dns):\n",
    "        # Too many false positives:\n",
    "        #   NS, MX, TXT\n",
    "        # identical DNSKEY, NSEC3PARAM, PTR, CNAME, A, AAAA, SPF\n",
    "        answers = {}\n",
    "        for _, raw_answers in dns.items():\n",
    "            for (answer_type, answer_text) in raw_answers:\n",
    "                if answer_type not in interested_types:\n",
    "                    continue\n",
    "                #if answer_type == 'NSEC3PARAM' or answer_type == 'DNSKEY':\n",
    "                #    answer_text = answer_text.split(' ', 3)[-1]\n",
    "                #elif answer_type == 'MX':\n",
    "                #    answer_text = answer_text.split(' ', 1)[-1]\n",
    "                if answer_type == 'CNAME':\n",
    "                    answer_text = answer_text.rstrip('.')\n",
    "                if answer_type not in answers:\n",
    "                    answers[answer_type] = set()\n",
    "                answers[answer_type].add(answer_text.lower())\n",
    "        return answers\n",
    "    dm2dns = {dm: parse_all_answers(dns) for dm, dns in dm2dns.items()}\n",
    "    with open(cache, 'wb') as f:\n",
    "        pickle.dump(dm2dns, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "dm2whois = pickle.loads(Path('data/dm2whois.pickle').read_bytes())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "585fe44904514628b1157f58e2aaf271",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/46922 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "70877 12932 239669 252601\n"
     ]
    }
   ],
   "source": [
    "all_dms = set()\n",
    "for site, time2parsed_urls in tqdm(site2time_parsed.items()):\n",
    "    _dms = set()\n",
    "    for time, urls in time2parsed_urls.items():\n",
    "        #if time >= T.timestamp():\n",
    "        #    continue\n",
    "        _dms.update([url.netloc for url in urls])\n",
    "    all_dms.update(_dms)\n",
    "\n",
    "crawled_ad_dms = set()\n",
    "crawled_benign_dms = set()\n",
    "for dm in all_dms:\n",
    "    if dm in ad_dms:\n",
    "        crawled_ad_dms.add(dm)\n",
    "    else:\n",
    "        crawled_benign_dms.add(dm)\n",
    "\n",
    "all_ad_dms = crawled_ad_dms | {ad_dm for ad_dm in ad_dms}\n",
    "all_benign_dms = crawled_benign_dms\n",
    "print(len(all_ad_dms), len(crawled_ad_dms), len(all_benign_dms), len(all_dms))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ad-related Domains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "Path('cache').mkdir(exist_ok=True)\n",
    "approach2dm2linked_ad_dms = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "import ipaddress\n",
    "\n",
    "cdn_ipv4 = pickle.loads(Path('./data-collection/cdnip/cdn_ipv4.pickle').read_bytes())\n",
    "cdn_ipv6 = pickle.loads(Path('./data-collection/cdnip/cdn_ipv6.pickle').read_bytes())\n",
    "def get_cdn(dm):\n",
    "    if dm not in dm2dns:\n",
    "        return dm, None\n",
    "    ipv4 = dm2dns[dm].get('A', {})\n",
    "    ipv6 = dm2dns[dm].get('AAAA', {})\n",
    "    for dm_ip in ipv4:\n",
    "        dm_ip_parsed = ipaddress.ip_address(dm_ip)\n",
    "        for cdn, ipns in cdn_ipv4.items():\n",
    "            for ipn in ipns:\n",
    "                if dm_ip_parsed in ipn:\n",
    "                    return dm, cdn\n",
    "    for dm_ip in ipv6:\n",
    "        dm_ip_parsed = ipaddress.ip_address(dm_ip)\n",
    "        for cdn, ipns in cdn_ipv6.items():\n",
    "            for ipn in ipns:\n",
    "                if dm_ip_parsed in ipn:\n",
    "                    return dm, cdn\n",
    "    return dm, None\n",
    "\n",
    "dm2cdn = {}\n",
    "n = len(all_dms)\n",
    "\n",
    "for this_dm, cdn in parallel(get_cdn, list(all_dms)[:n], total=n):\n",
    "    if not cdn:\n",
    "        continue\n",
    "    dm2cdn[this_dm] = cdn\"\"\";"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parallel process_num=36 chunksize=220\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3e83c861c72475f8a26fac89db3c84d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/252601 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# DNS\n",
    "def func(this_dm):\n",
    "    linked_ad_dms = set()\n",
    "    this_dns = dm2dns.get(this_dm)\n",
    "    if this_dns is None:\n",
    "        return this_dm, linked_ad_dms\n",
    "    #this_cdn = dm2cdn.get(this_dm, None)\n",
    "    for ad_dm in all_ad_dms:\n",
    "        ad_dns = dm2dns.get(ad_dm)\n",
    "        if ad_dm == this_dm or ad_dns is None:\n",
    "            continue\n",
    "        #if this_cdn is not None and this_cdn == dm2cdn.get(ad_dm, None):\n",
    "        #    linked_ad_dms.add(ad_dm)\n",
    "        #    continue\n",
    "        for typ in {'A', 'AAAA', 'CNAME'}:\n",
    "            if typ == 'CNAME' and (\n",
    "                ({ad_dm} | ad_dns.get(typ, set()))\n",
    "                & ({this_dm} | this_dns.get(typ, set()))\n",
    "            ):\n",
    "                linked_ad_dms.add(ad_dm)\n",
    "            if this_dns.get(typ, set()) & ad_dns.get(typ, set()):\n",
    "                linked_ad_dms.add(ad_dm)\n",
    "    return this_dm, linked_ad_dms\n",
    "\n",
    "dm2linked_ad_dms = {}\n",
    "for this_dm, linked_ad_dms in parallel(func, all_dms, total=len(all_dms)):\n",
    "    if not linked_ad_dms:\n",
    "        continue\n",
    "    dm2linked_ad_dms[this_dm] = linked_ad_dms\n",
    "approach2dm2linked_ad_dms['dns'] = dm2linked_ad_dms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Server fingerprinting\n",
    "\"\"\"\n",
    "def get_header_jc(h0, h1):\n",
    "    IGNORE_HEADERS = {\n",
    "        'connection',\n",
    "        'age',\n",
    "        'cache-control',\n",
    "        'date',\n",
    "        'expires',\n",
    "        'content-length'\n",
    "    }\n",
    "    h0 = set((k, str(v)) for k, v in h0.items() if k.lower() not in IGNORE_HEADERS)\n",
    "    h1 = set((k, str(v)) for k, v in h1.items() if k.lower() not in IGNORE_HEADERS)\n",
    "    return len(h0 & h1) / len(h0 | h1) if len(h0 | h1) > 0 else 0\n",
    "\n",
    "def is_response_similar(r0, r1, thres=0.9):\n",
    "    status_eq = (r0['status'] == r1['status'])\n",
    "    content_eq = (r0['content'] == r1['content'])\n",
    "    header_sim = get_header_jc(r0['headers'], r1['headers'])\n",
    "    return status_eq and content_eq and header_sim > thres\n",
    "\n",
    "def is_domain_similar(d0, d1):\n",
    "    paths = ('', 'robots.txt')\n",
    "    for path in paths:\n",
    "        r0 = url2response.get(f'https://{d0}/{path}')\n",
    "        if r0 is None or not r0.get('ok', False):\n",
    "            r0 = url2response.get(f'http://{d0}/{path}')\n",
    "\n",
    "        r1 = url2response.get(f'https://{d1}/{path}')\n",
    "        if r1 is None or not r1.get('ok', False):\n",
    "            r1 = url2response.get(f'http://{d1}/{path}')\n",
    "\n",
    "        if r0 is None or r1 is None or not r0.get('ok', False) or not r1.get('ok', False):\n",
    "            return False\n",
    "        if not is_response_similar(r0, r1):\n",
    "            return False\n",
    "    return True\n",
    "\n",
    "def func(this_dm):\n",
    "    linked_ad_dms = set()\n",
    "    d = this_dm\n",
    "    paths = ('', 'robots.txt')\n",
    "    for path in paths:\n",
    "        r = url2response.get(f'https://{d}/{path}') or url2response.get(f'http://{d}/{path}') or {'ok': False}\n",
    "        if not r.get('ok', False):\n",
    "            return this_dm, linked_ad_dms\n",
    "    for ad_dm in all_ad_dms:\n",
    "        if ad_dm == this_dm:\n",
    "            continue\n",
    "        if is_domain_similar(ad_dm, this_dm):\n",
    "            linked_ad_dms.add(ad_dm)\n",
    "    return this_dm, linked_ad_dms\n",
    "\n",
    "dm2linked_ad_dms = {}\n",
    "n = len(all_dms)\n",
    "for this_dm, linked_ad_dms in parallel(func, list(all_dms)[:n], total=n):\n",
    "    if not linked_ad_dms:\n",
    "        continue\n",
    "    dm2linked_ad_dms[this_dm] = linked_ad_dms\n",
    "approach2dm2linked_ad_dms['fp'] = dm2linked_ad_dms\"\"\";"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "939a7dd1aa4d4dd59edcca32a4d1e6a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/217979 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# SSL certificate preprocess\n",
    "cert_tries_linked_ad_dms = []\n",
    "for dm, names in tqdm(dm2cert_names.items()):\n",
    "    group_dms = names | {dm}\n",
    "    if all(dm.replace('*', 'zjt8axqolduuyye') not in ad_dms for dm in group_dms):\n",
    "        continue\n",
    "    cert_tries_linked_ad_dms.append((\n",
    "        TrieSet(group_dms, include_subdomain=False),\n",
    "        {group_dm for group_dm in group_dms if group_dm in ad_dms}\n",
    "    ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parallel process_num=36 chunksize=220\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2597487cc0b640ce95bd6d56ae8152c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/252601 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# SSL Certificate\n",
    "def func(this_dm):\n",
    "    linked_ad_dms = set()\n",
    "    for cert_tries, _linked_ad_dms in cert_tries_linked_ad_dms:\n",
    "        if this_dm in cert_tries:\n",
    "            linked_ad_dms.update(_linked_ad_dms)\n",
    "    return this_dm, linked_ad_dms\n",
    "\n",
    "dm2linked_ad_dms = {}\n",
    "n = len(all_dms)\n",
    "for this_dm, linked_ad_dms in parallel(func, list(all_dms)[:n], total=n):\n",
    "    if not linked_ad_dms:\n",
    "        continue\n",
    "    dm2linked_ad_dms[this_dm] = linked_ad_dms\n",
    "approach2dm2linked_ad_dms['cert'] = dm2linked_ad_dms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf231fb4563c45f4af7578b5b6cb1a5c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "dns:   0%|          | 0/63827 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3dcecefb1e664efab2534b1a7ebddd2a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "cert:   0%|          | 0/42985 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dm2related_ads = {}\n",
    "for approach, dm2linked_ad_dms in approach2dm2linked_ad_dms.items():\n",
    "    for dm, raw_linked_ad_dms in tqdm(dm2linked_ad_dms.items(), desc=approach):\n",
    "        linked_ad_dms = raw_linked_ad_dms - {dm}\n",
    "        if linked_ad_dms:\n",
    "            if dm not in dm2related_ads:\n",
    "                dm2related_ads[dm] = {}\n",
    "            for ad_dm in linked_ad_dms:\n",
    "                if ad_dm not in dm2related_ads[dm]:\n",
    "                    dm2related_ads[dm][ad_dm] = set()\n",
    "                dm2related_ads[dm][ad_dm].add(approach)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "save cache to cache/dm2related_ads.pickle\n"
     ]
    }
   ],
   "source": [
    "cache = Path('cache/dm2related_ads.pickle')\n",
    "if not cache.is_file():\n",
    "    cache.write_bytes(pickle.dumps(dm2related_ads))\n",
    "    print('save cache to', str(cache))\n",
    "else:\n",
    "    dm2related_ads = pickle.loads(Path('cache/dm2related_ads.pickle').read_bytes())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARwAAADqCAYAAABwSPpQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAjMElEQVR4nO3dd3xcZ53v8c/znDOjGY2KVd3t2HGKHVIICZAKhF1qCtnsTS4sEFiWXCAk8KIs7XVh2Xvv3qXsJcnCLvCiBdiEspBAlhJKKEsgiZPFpCfYcY0ky5Y00vQ55bl/jCTsuEn2zGn6vV8vvzQeSTPnHJ3zned5zlPUmee+zCCEEAHQYW+AEGLhkMARQgRGAkcIERgJHCFEYCRwhBCBkcARQgRGAkcIERgJHCFEYCRwhBCBkcARQgRGAkcIERgJHCFEYCRwhBCBkcARQgRGAkcIERgJHCFEYCRwhBCBkcARQgRGAkcIERgJHCFEYCRwhBCBkcARQgRGAkcIERg77A0Qc2MDGQUppbAASzH9tfF/DcwsMGYAf/qxbwwe4BhwgboxOAacoHdACCRwIsUCclrRoSCjFG37fD36oqg66LMGqBqoGENl5qvfeOwe9XsJcXgSOCHRQNd0uOS0IqcUmYNnQ0soIKsgq9Q+zzRUDRSMoeA3/pVkbVbRJBI4AepQsEgrurWiU6vINqBlpktWA7oRQh5Q9A153zDhG8oSQOIoSeC0kAZ6tKLPUizSKrYH2wK6p4NyNY0S0MR0+Ez6BskfMVdxvQYiS9EoxfRbil7daNBNmoyCpZZiqaVwgXHfMOoZpnyJHnF4EjhN0q5giaXpt+JbkjkaNjCoFYNaUTWwx/MZ9Q01yR5xEAvp2mg6BfRpxRJb0xVgg29UZRSstDUrgUnfMOwZxqXUI/YhgXMUUsASW7PYUqTD3piImmnzqRh42vPZ40lbj5DAmZcUsGI6aKJ6hylqsgrW2ZpVNoy4PiOekX4+C5gEzhxI0By7NLDK1iyz4WnXZ9gzs72hxcIhgXMYmkbQLJOgaRobWG1rltqww/UZ9aSitZBI4BxCv1Ycl9LSRtMiaRpVrWUWbHd9JqRxeUGQwHmGdgVrU5bcdQpIu4L1Kc2Eb9ji+NTD3iDRUhI40zSNov4SSx1iuKNopR6teHabxfbphmWRTBI4QJeCdSkr0MGT4kAWsNbWDFiw2fGoSO4kzoIOHMV0A6aUaiKlU8EZaYudrs+uOZd2DJYNlt34qqYnCDJmep6gmcem8dj3Fa4DGPnLB2nBBk67ghNTFu1yvkWSonEbfZEFT9Y96oCdNqQzjX922mDboKcDxrKOrjjkuQqn3ggft65w6wqn3njsuXJyNNuCDJylluI4W0upJsKMBWQ13RnFc9ts9thVaqnmz1PYKBEdPKxcR1ErK6rlxtd6VXGoCc3E3CyowFHA8SnNoJaTJor8NoXOadI5TVt2/55POXKMOjUmrEpg22OnDHa3Idc9vX2eolJSVEuKSkHjOnIezdeCCZw0cHLaokPOkcgwgGlX2B0WmQ6NZR/+jzOYaiPrWgyrMkYF309ZW4ZclyHXBSz1qFUUxbymNKnxPTmx5mJBBE6XgpPSFqmwN0QA4NsK3aVp77awU/O7UDttmza/g11eCcfyWrSFc9OWNbRlPXqXeJQLmlJeUy5ItetwEh84g5bieGmvCZ0BTIemrdsikzu2gSJprVmlOtjpFanrcEMHQCnIdfnkunw8T1GaVBTGLZyanHXPlOjAWTbdOCzCYxSoLov2Pgv7CFWm+bCVYrXVwQ63TM2KzqI3lmXo6jV09fqUpjT5UQmefSU2cI6bHnQpwmEU0G3R3tvcoNmXVopVdju73DKVCIXOjJlSjwTPnyQycNbZmkEJm1AYBarbItdrHbERuBm0UqxM5Xi6XqZkR3MklgTPnyQqcBRwYkrTJ7e9Q2HaFdnFKVLzbAg+VgpYkW5nqK4o2LVA33s+9g2e8WFrQXYsTFTgrJOwCYVvK9KDNtmOcNvLlqWzjDiKSasa6nYcSa7LJ9thGB/RFCeSuK7HoSUmcI639ezCbSIYBlC9Fp29NjoibfNLUhl0XTFhB9dB8Ghobehf5tHRbdg7ZOHWF8a5G5HT5NgcNz39pwiOSSsyq1N09kcnbGYMptvodNvC3ow5yeR8lq9z6er3YAFMMx+xU2X+VlpK7kYFzHRbdKxKk26L7umzNJ0l48Wjq6dSht7FHsuOd0m1JTt0onvGzMGgpVgp/WwCYzRYS1N0LY5eqeaZFLDcbsf249NGks4Ylh3v0LEoudPLR/y0ObRO1Wi3EcHws5rs6jTtnfE55rZSrFA5lInPNisF/ctd+pa5JLGKFZ+/xD5mBmJKRSoYplPTtSL4293N0GZplpn22F27nT0+S9e6h5w6I65iFzgaWC8DMYPTa9G1NIWKX9bM6rBtBvxc2Jsxb21Zw9K1yWrXiV3gnJDS5GJ88seFAfRim87+ZPSc6E2l6HYzYW/GvNmpRuhkO5LRrhOrwFlqKenYFwBfQ3p5ilx3fBpc52JJOkPWjd9KY1obFq92yXXHP3RiEzjtqjHhuWgtX0NmeeqYp5CIqmWpLNqP574NrHBp74p36MTiyDfGSFnx2NgYMxoyy1IHTO+ZJLZSDJps2Jtx1AZWurR3xjd0YnFmHWdrWV2hxYyC9LIUbe2xOCWOSXcqRXsMq1bQ+PAdWOWSjWnoRP7s6tGKpdKTuKVmwiazAMJmxlI7G6v+OftSwOCqeDYkR/qIWzRWWRCtY4DU0uS22RyKrRWDfnyrVjOhk4lZ6ET6LFtta+JZ8I0PPRD+tBJhWZRKxfKu1QylYHClRyodn346kT3TOhQyArzFTLdFR0+ybn3PV5yrVtC4ZT642kXpeIROZI/02pQMXWglP6voHExGp75jkdKKgRhXrQBSaUP/8vBXr5iLSAbOEkvJgnUtZGxFR8yHKzRTTyoVm6ksDiXX5U/PqRNtkQsci8Yi9qI1DNC21A5kgvM4GdTxLuUA9C72yOSi3YgcuSt7ua2TM+9pBKleK9Ed+45W1tLkvPg2IM8YWOlhpaLbnhOpMy8F0uemhfw2RUefxPmhDOj4De58JssyDK6M7lw6kQqcFbZmYd8zaR2jILvElnabw2izdGzmQj6ctqyhszeaVavIBE4auQ3eSrrPivQcxFExYGeiWjiYl57FfiQn74rMGbjS1tHZmITxs5pcr1Sl5iKlFZ1e/Es5Wht6l0TvrlUkrvEUyNK8LWKAzID0aZqPfjv+gQOQ6/YjN/QhEoGzxNZyQbRKp6YtE4k/c2yktSYX4yEP++pb6oGKTtUq9DNR0ejoJ5rPKGgfkKrU0ei34n/HChq9kBcNRKeUE3rgDFhKJkRvEdVjYUsHv6OSsXTsex/P6B7wsCMywDP0wFlmhb4JiWRsJQ3Fx6grIXMVKBqhEwWhXu1dWslMfi1i9VqRXx0z6rptu1EvTYCORX4kSjmhnpKDsgJDS/g2tCdsxYUwaKXIJaRaFZVSTmiBo4BeaSxuCavbkh7FTdKtk1GtgkYpJ+zOgKEFTq9WMkizBYyG9kVyZJulw7bRMZ6ga18K6OoP945VaEdyQEo3LaE6LbTUpppGAZ1+cko5nT1+qLMDhhI4No3VGERzGSDTK2nTbEmqVmlt6OwJr5QTSuD0Wkp6FreAyWlSKTmyzZa1NCk/OUHeseACR0o3LZHqTEZbQxQlqVqVbjOk2sKpVgV+hiqgWwKn6YyGTEdyPoWjptNKxu3xGR2LwinlBB44nVrJJFut0KGlo18LZSydmLtVAO1dCyRwFknppiXSnRLjrZbxk9PdIJU2pLPBh07ggSPtN83n29C2wJbqDUPGJCvUO7qDb8cJ9Cy1QcZOtYBql/mEgpCzklPCgcYEXUHPpxpo4HRK6aYl7HYp3QQha1mohAzmBLBsQ1t7ggOnQwKnJdokcAKhgLYEteMAZHJJDhzJm6bz25SsohmgbMJGAGakhCPmQ8sqmoFqV8lqOE5slSqjkKlEW8CWu1OBareSFThaB9vrOLCzNScTtLSErMgQLK1UosZVAbS1B9cfJ7CzNSt503QmpWQqihDYCepxDMFWqwKsUkniNJ3UUUNhJ6wDYJANx8EFjjQYN52StcJDkUpYN8tU2gQ2KVegjcaiuay0HNQwpFTygj4V0IoOgRw5DQlZ4SdarDYJnDAkMXDsgKrngRw5uS5aIy1VqlCkEzgPiJVKUAknJQ3GTedrZCmYkKQS2B6ZqBJOsjqDR4OSVS9ClbRb44kq4chQn+YzclBDZSXs1riUcMRhJWxIT+zYCZqmAsBOVgknWX+cSJAqVaiSdvSDqlIFUvgIuoRjpWz+8gMfwLJttGWxeeP93HP77fSvWslFV1+NnUrhex6/+OrX2L11K539fbz+H/6BiZERAEa2bOGum78KwBXvfx+57m5cxwHgtk98kkqhwLNf+hJOufBCjO9TKRT46Re/RGFsLLB9jEIJJ9fZw4WXvoFsrgtjDE9s+g2PbryLZ19wMSeecT7VcgGAB375PXZteZhlx63nrBe9Cm3Z+J7Lxru+y/D2J7DTbbzyde/Z73W3PHwv9/7s26w79RzOfvFfUC7kAXjs/l/y5B/uDmN39zPfibj+8uL/wfoTzqRYmuJTn38vAK+5/B0M9C0FIJPJUa2WuPEL7+eMZ53HC55/yezvLlm8ipu+8AGGd2/npS+8ijNPu5BsJseHP/6G2Z+xLJurLr2W5UvXUK4UueW7NzIxuWfu+9PYKWhxyS2QLAi6Ud9zXL77sY/j1Gpoy+K/ffADbHvoQZ5/+eXce/v32P7QQxx32mmcf9WVfOcfPwZAfnSUWz78kYO+3o8/93lGt23b77k923fwjY/+PW69zqkvehHnX3klP/rXf231rs2KQqHR9z3u+9m/M7Z7J3a6jcve+EGGtj4GwCP3/ZyH7/3pfj9frRT56bf/hUpxkkUDy3jpf7+eb/7z+3HrNb73xf8z+3OXvvEDbHvi97P/3/roA9zzk28Es1NzNN+lHB948Ff89v47uerSa2efu+W2G2cfv/LPXku1VgZg08N3s+nhRqguGVjJ6698D8O7twPw2B8f4Lf338l733bDfq9/9hkvolIt8ol/eSenbziHl1/0mv1ef077pMC0uKATSJUqjGvDqdUA0JaFtuzGgTSQzmaBxtfSRP6oX3/X44/j1utAo0TU0dtzrJs8PxEInEppirHdOwFw6zXyYyO0dyw65M+P795JpTgJQH7PEJZlo58xT3BXzyCZXCe7d25u2XY3w3wDf+uOx6lUSof8/mkbzmHTw7894PnTn3Uef3jkT8/veHozhWL+gJ875cSzeODBXwPw0GP3sm7NKfPbQIL5EEtse65Sild/9O/oHhzkwZ/fxe6nnuJXt9zC5e95NxdcdRVKK771v//0qdo9MMCrP/p31CsVfvfd7zL05B9nv/fnb3oTxvhsvv9+7vv+HQe81ykXXsi2Bx8KZL+iqqO7j77FK9kztJXFK49n/XNeyLpTn8fe4e3c9/PvUK+W9/v5404+k/HdO/E9d7/n155yFlsffeAZP/tslqxax+T4KPf99NuUChMt358grVl1MsVinrGJkQO+d/qGc7j5W5844mt0dfYyOdWo0vvGp1qr0J7tpFwpNH17j0ViA8cYwy0f/gjp9iwXX3cdfcuX86wXvoBf33orm+9/gBPOPps/++s3ctsnPkk5P8mX3vVuqqUSg6tXc/H11/P1D32IerXKjz/7OUr5PKlMhle+/VpOPvdcHv/tnz5xTjrnHAbXHMd3/u8/hri34bJTbVz0F9dw78++hVOv8th//YpNv/kBxsBzXnApz33xFfzmB1+b/flF/Us560WXc+etBxb512w4m19//8uz/9+5+UGeenQjvudy0rMv4IJLrubHt9wQxG4dXhOrHqefch6bHjmwdLNy2TrqTo3de3Yd8TUOXjoJZznfw0lW76WDqJcrPP34E6w+9VTWn3cem+9vfHr+ceNGFq9dC4DnulRLjeLu6PbtTO4ZZdGSJQCU8nkAnGqVJ+65hyXTvwOwcsMGnnvJxdxxw4147v6f1AuF0pqLrriGLY/cx/YnNgFQLRUw03XYJzb9hoFlx83+fHvnIl58xVv49R1foZDfu99r9Q4uRyvN2MiO2edqldJsKejJTb+hf8nqVu/SnDTrUtZK86yTzubBR393wPdOP+Xc/apThzM5NU53V9/sa2baspQrxSZtZfMEEjhB52y2s5N0e6OtxkqlWLlhAxPDw5TyeZaffBIAK9evJ7979+zPq+mPiK6BARYtXszknj0orcl0dACNtqA1p5/O2NONT5uBVau46A1Xc8eNN1EphFBsjciH1wWvfD2Te0d45L6fzz6XzXXNPl594hlM7BkCIN2W5SVXvp37f3k7o7u2HPBaazeczVOPbtzvuX1fa9UJp5MfG272LoRq3ZpT2TM2xGRhfL/nFYrT1j9vzoHz6JMP8JzTLgTg1PXPY8u2R+a9La1uMIaAqlR+wBdHrrubP3/z36B1Y8DRH+/byNY//IFaucyFf/UatNZ4jsNdX/4KAMtPOpHnX345vudhfMNdN99MrVTCTqd51XvejWVZKK3Z8cijPPzLXwFw/lVXkm5r4xXXvg2AwtgYd9x4U2D7GMTJcSSLVxzPulOfz/joLi5704eAxi3wtRvOonfxSsBQzI9x94/+DYD1Z72Qzp4Bzjj/FZxx/isAuPPWm2Zvn69Z/xx+8q1P7/ceG86+iFUnnIbxfWrVEv/5HzcHt4OHYeaZ+K++/DrWrtpArr2TD17/GX76639n46ZfcPop5x60OrVm9Xomp8YZz4/u9/zLL3oNz37WeaRSaT54/We4b9Mv+Nn0a1112bW89203UKkUueW2+Z+LQZxT6sxzX9byt1lla1ZIR7WmMt0WXYsT2wQXeU87ZYpWPezNaBpjYPujrZ9EJpAqlRuFj+Ok8eSYhilpR991gikQBBM4QbzJAmO8sLdgYXMTFjmem6TASdbfJhrkoIbKVclKfNcJ5n2kShVXUqUKjQE8HdxaTkHwklSlCig8FxTtg5+scz423KBvuwYgUSWcavL+PpHg1CVxwlBPYNInqtHYAMm5gRgdbk2SPAwOyQscL0klHJBSTiuYuhzUMDgmeYHj1BNUwgGoJLDeGzavlrwTPw6SVsKp1xTGT1jgVOVOVdMpqaeGwk1Y4NTKwY0CCK6EI3nTdMo1+MnqDhILyQuc4CaNCOydSlLCaYlaJVknf9T5xuBYyUr5RJZwakb647SCU5bACVLJS1bY+J4KrMEYAp6AqygNx80ngROocsLqsNVKsLM4SODEnKobPBlXFZiqStZQ5CCrUxBw4BTkumiJWklKOUEwQFUnK3CqxQQHjpRwWsOVhuNAVDyvsVhcQriOolYJdlrzQN/NBYrJ+XtFR8mPxJSjSVfyk1W6KU0Fv4ZC4O+Yl1JO0ykPqtJ43HKVhM16VsoHP+1v4IEz4cmF0QrOVLIuhiiqWckp4Th1Rb26AEo4RSNTjrZE0Zf5cVqo6vn4KjkHuDQZzpJ0gb+rASalWtV0ykC1IKWcVikENX9DQEr5BRI4ABMSOC3hSuC0TEEnZ6RsvRZs7+J9hRI4Y55J2PC3iCgbHJkjp+kqno+jkxPmhfHwVvgO5Z095G5VKyigOpGcCyMq8n5ySje+pyhOLLDAARiVVQdawkx5MtShiQxQTNDEQ4UJjTHhrYIbWuBM+EbuVrWAMlDOSymnWQqui5+QJWEMMDUWXukGQgwcQ6MtRzSfP+lJz+MmmTLJKd0UJ3RgK2weSqhxNyqdAFtCe1CelFLOsfKMoWwl43a4AfJ7rLA3I9zAKRgZW9Uq7rgrHQGP0aTrYhKyhnhxQge2uubhhFuhA4ZduSpaQbtQHpdWsmMxlZDV1IyJRukGIhA4e32TkD9r9PgTHo6TjE/ooFU8n1pCqlP5PVYkSjcQgcAxwIiUclpCGajslVLO0djrV8PehKZw6orJPaFf5rMisSUj0vO4ZVTBl5Ud5qnm+5StZJS7x4YsGl1CoyESgeMiHQFbqTrqJqTpMxh73WSUbop5TbUUiUt8VmS2ZqfrSymnRXTNUBqTqtVc1H2foh3/0o3vKcZHotFQvK/IBI5Do2olWsMf86hVJdKPZI+XjNLN+G6N70WnKjUjMoEDsMv1ZbhDiyigNuJKD+TDqHo+xQS03VTLmuJE9Eo3ELHAcYEhuWPVMqpuKMpdq0NKQunGcxV7dkYzbCBigQMw5BlZEriFzIQnE64fRNnzKMe87cYAozut0MdLHU7kAscHtkspp2UaVSsHV6aw2M9oAvrdjA9b1MqRu6T3E8mtG/WMrNLZQtqF8pC058wYd5zY9youTmoK49GtSs2IZOAAPOV40nekhVTVpzAS74usGeq+z15dDnszjkm9phh7OvphAxEOnJKBYblN3lKq4FNc4AM8h70KJsbL9/q+YnSHHeosfvMR2cAB2OH61MLeiITz93qUF+gieuOOQzXGVSljFLu3W7ghrcBwNCIdOD6w2ZEG5FZSgDviUikurOPs+IaxGFelZsIm6o3EzxT5rZ30DUNStWopBTjDDtXSwgmdYa+MH9OqlDGwe4cVuXFScxGLLd7u+jIzYIspA/UhZ0H00ck7DpWYVqWMgdEdNtViLC7dA8Riqw3wR8eTwZ0tpgzUn0526Di+YVRXwt6Mo2JohE0lpmEDMQkcgIqBrdIhsOVmQieJbToGGPLKGBW/fTPAnpiHDcQocAB2e4Y9smJnyykDzpBDKWHrW+12qrG8K+X7it3bbMqFWF2uBxW7PdjsSHtOEBTgj7oURpPRT2fMqTNpxW/4glNXDG+xY9lAfDCx2wsDPFb3ZOL1oOQ9JoecWA+DKLgue6343QKvljXDT9k4MepncySxCxxoTNb1eF0akYOiiz5TO+o49filTtXzGVGlsDdj3qbGLEa2WpGcROtYxDJwoLGAnnQKDI6uGSo76rFa0dM1hl2mGKv+NsYo9uyyp6cHTVbYQIwDBxprWm2TO1eBUT54u12mhp3Ir+ppjGGXW8bTEd/QfVTLmqHNNqXJWF+Wh2WHvQHHasgz2Mqwwkrep0FUqYJPoVons8SmLRvNi2PIqVCz43FHyvcVE7vjMb3EsYp94EBjkKeFZqmETmC0Y6jtdKh3W+T6bXSErpURpxqblRfKRc3YUHRWxmy1RAQONDoFWmgGJXQCowAmPQolj1SfTXt3+KkzXK8yZUf/9rfnKcaHrURXnw4mMYEDsNn1UUozoCV0gqTdRtvO5JRH++IUqXTwx98AQ/VyLEo2xbxmfCR5d6DmIlGBA/BHx8e1pXoVBl0xVLbVqXRbZHstUqlg/gaNBuJK5CdBL05q8qPxmr+m2RIXONCoXrlGsdJeWMXVKJipZlUmPSpdFtm+1gaPbww73XKkhywU85r8noUdNDMSGTgAOz2Dg88aWyewN0P0KYApj8qUR6VLk+2zmx48rjHsdEvUrWgOv5CgOVBiAwcaSwe7xueElIROWBrB41OZqlNpV6S6bTIdGnWMfxDXN+zwizhWtDoiuq6ilNcUJrQEzUEkOnCg0TmwWvc4OW2RDntjFjAFUDa4ZYcpG3SnRabbOqoG5prns8uUcHU0wsYYRWlSUcxrqiVFEnsIN0viAwcawyD+UPM4KW3RJedC6LQLTHhUJzwqWY3OKdo65hY+k67LblXC6PCHK1RKmuKEpjylYrNqQtgWROBAY8DnI3WPNbZmidzBigxV8TEVqO71qKQUKqdJdWjasvtXuwyN+WzCnGLCdRXVoqJa0lSKKtJL6kbVggkcaJy0T7k+RaNYa+t4DyRLIOUYyHs4eY+aBtWmUFmNaoNRu4bTFuydKKeuqJYVtbKiVtKJmiYiLAsqcGaMeoaC73FCyqJDzqFI0j5QMUyUXDY7Pg4Ky06RzhjSGYOdNlg2WPafvs63IdoAbl1N/wPH2edxXWF8OTmabUEGDjTmSH6o7rHCUqyQW+eR49NYrWPf1Vc9V1EpKirFg/+O0gbbBm0btGZ20jBjph+bRgMvBnwfPBekgTdYCzZwoPEJt9MzTEyXdrJy7kVC3jc85fpU59kubHyFUwek6hNZCzpwZhQN/KHuscLWLLeUfOaFxAW2uT6jsvBhYkngTPNpTHMx6sEaW9MjA0ADtdc3bHV8ojtAQTSDBM4zVA085vj0aMUaW5OR3GmpommUaqZk+Z8FQQLnECZ8Q77usdxSLLO1HKgmq5pGo/CYBM2CItfRYRhgl2cY8TyW2ZpllpK+O8fIAXa5PiOeQaJm4ZHAmQOXRvvOkAvLpnsqy4Gbn6qBYc9nt2dkeZ8FTK6beZgJnqddWGwplljSxnMkRQNDrs9eqToJJHCOikdjtYghz6NHK5ZaikVyV2uWodEGNuwZJiVoxD6kSeIYTfiGRx2f/6p7DHlmQd/WLU/fcbq/5vG44ycybL5561dZvHgw7M2ILSnhNEl1+mLb5kK3VvRrRd8CaOtxgDHPMOr5FJOXL7O01vhRX/0vBtSZ574swadJuBTT4TNd5UrKBGBlA+O+YcLzKcTk7Onr7eUNV7+Wk08+EaU0d//2d3z5K1/jhS+8kEsvfgXd3d1s2fIUn//Cl9i7dwxolGa++KWbecXLX4plWYyNjbF+/clUqzXA8NnPfZHf3XNvuDsWM0n/AA6VoTEuKD9dtWhXjQBapBVdWhH+Kk5z4wAFv9EeM+4bajEJmRlKKd73t+/i4Uce5dPXfxbfN6xdu4azzjqTyy+7hI9/4lMMj4zwqssu5vrr3saHP/K/Zn/37LOfw4f+50ep1+s4jsM3b/0qf/v+D7F792iIexRfEjgBKhsoe43GVAV0KOjQig6tyClFVoU/dtnQGElf8A1TvqFgzLwHUUbNunXH09OziK//2zdmq0VPPPEk73/fu7n9e3fw9NAQALfdfgevuuwS+vv7Zks5t3/vDkqlUmjbnjQSOCExQMFAwTMwPVhR0ygF5bQiqxRpBRmlaFOQavL7uzTanSrGUPENlenHVUPi+sn09fWyZ+/YAW0wA/39vOHq1/K617569jmlFL09PbOBMzY2Hui2Jp0EToT4NPqtFL3pyVv2oWE2eCzVqI7ZCqzp/+vp35j9Z8zsYxdwTGNZFQeoJzBUDmdsbJz+vr4DGn7Hxsa57fbv85u7f3fI3zUm5sW7iJHb4jExPQEeU6ZxK36vbxjxDE97hh2uzzbXZ7vrs8P12en67Jr+3pBnGPUME76hYEhkCeZINm/eQj6f5zWvvpK2tjSpVIqTTjyBn/7sLl512SWsWLEcgGw2y/Ofd/ZhXyufz7N4UG6LHy0p4YjEM8bwsU98ijde/Vo+8883YDDcfffv+MrNXyeTaeMd172N/v5+ypUyDz30CPfcu/GQr/Xt79zG2956Del0is9/4cvcc899Ae5J/MltcSFEYKRKJYQIjASOECIwEjhCiMBI4AghAiOBI4QIjASOECIwEjgR89a3vJmrrrwi7M0QoiUkcIQQgZHAEUIERoY2hOy441bzlmvexJIli/n9pgdherDghvUn8/Zr38IPf/RjLr3kYnzf5xvf/Da//NV/AnDGGafxur96NX19vVQqFX7wwzv5jx/8KMxdEeKIJHBCZFkW73nXO/jhj+7kzp/8jLOecybXX/dWvn/HDwBYtKibbLadt177Dk499RTe9c7r2Hj/A5RKZd5yzd9ww42f5vEnniSXa2dwYCDkvRHiyKRKFaITT1iHZVv88Ed34nke9963kS1PbZ39vud5fOe7t+N5Hps2PUi1WmPZ0qWz31uxYjnZbIZSqczWbdvD2g0h5kwCJ0Q9PYuYGJ/Y77m9e/fOPi4UivvN31Kr18hkMgD8v0/dxBlnnM6nb/oUH/nwBznhhHXBbLQQx0ACJ0QT+Tw9vT37Pdff1zen393y1FY++U83cM1b3s7GjQ/wzndc24pNFKKpJHBC9OSTm/E9n5e/7CVorXnu2Wdx/PFrj/h7lmVx/nnnkM1m8TyPSqUiS5iIWJBG4xB5nsc/feomrnnzX3PVlVfw+00Pct/GB+b0uxdccB5vfMPr0VozPDzMpz/zuRZvrRDHTibgEkIERqpUQojASOAIIQIjgSOECIwEjhAiMBI4QojASOAIIQIjgSOECIwEjhAiMP8fmbt6eYBq+i4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib_venn import venn2\n",
    "labels = ['dns', 'cert']\n",
    "venn2(\n",
    "    [{k for k, v in approach2dm2linked_ad_dms[name].items() if v} for name in labels], \n",
    "    labels\n",
    ");"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RAD Domains"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TrieDict:\n",
    "    def __init__(self, dm2time, include_subdomain=False):\n",
    "        self._dict = dm2time\n",
    "        self.trie = {}\n",
    "        dm_time = list(dm2time.items())\n",
    "        if include_subdomain:\n",
    "            dm_time += [('*.' + dm, time) for dm, time in dm2time.items()]\n",
    "        for dm, time in dm_time:\n",
    "            cur_dic = self.trie\n",
    "            nodes = dm.split('.')\n",
    "            if len(nodes) <= 1 or ('*' in dm and nodes[0] != '*'):\n",
    "                print('ignore', repr(dm))\n",
    "                continue\n",
    "            for node in reversed(nodes):\n",
    "                if '*' in cur_dic:\n",
    "                    cur_dic = cur_dic['*']\n",
    "                    break\n",
    "                if node not in cur_dic:\n",
    "                    cur_dic[node] = {}\n",
    "                cur_dic = cur_dic[node]\n",
    "            #if '.value' in cur_dic:\n",
    "            #    print('dup?', dm, cur_dic)\n",
    "            cur_dic['.value'] = time\n",
    "\n",
    "    def __contains__(self, dm):\n",
    "        return self._getValue(dm, return_bool=True)\n",
    "    \n",
    "    def __getitem__(self, dm):\n",
    "        v = self._getValue(dm, return_bool=False)\n",
    "        if v is None:\n",
    "            raise KeyError(dm)\n",
    "        return v\n",
    "    \n",
    "    def get(self, dm, default_value=None):\n",
    "        v = self._getValue(dm, return_bool=False)\n",
    "        return default_value if v is None else v\n",
    "    \n",
    "    def _getValue(self, dm, return_bool):\n",
    "        cur_dic = self.trie\n",
    "        nodes = dm.split('.')\n",
    "        #assert len(nodes) > 1 and '*' not in dm, 'Invalid dm ' + repr(dm)\n",
    "        if len(nodes) <= 1 or '*' in dm:\n",
    "            return False if return_bool else None\n",
    "        for node in reversed(nodes):\n",
    "            if '*' in cur_dic:\n",
    "                return True if return_bool else cur_dic['*']['.value']\n",
    "            if node not in cur_dic:\n",
    "                return False if return_bool else None\n",
    "            cur_dic = cur_dic[node]\n",
    "        if '.value' not in cur_dic:\n",
    "            return False if return_bool else None\n",
    "        return True if return_bool else cur_dic['.value']\n",
    "    \n",
    "    def __iter__(self):\n",
    "        return iter(self._dict)\n",
    "    \n",
    "    def items(self):\n",
    "        return self._dict.items()\n",
    "    \n",
    "    def keys(self):\n",
    "        return self._dict.keys()\n",
    "    \n",
    "    def values(self):\n",
    "        return self._dict.values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PathTrie:\n",
    "    V = '!@__VALUE__@!'\n",
    "    def __init__(self, urls, progress=False):\n",
    "        prog = tqdm if progress else lambda x: x\n",
    "        self.trie = {}\n",
    "        dicts = []\n",
    "        for url in prog(urls):\n",
    "            path = url.path\n",
    "            cur_dic = self.trie\n",
    "            nodes = self.getNormalizedNodes(path)\n",
    "            for node in nodes:\n",
    "                if node not in cur_dic:\n",
    "                    cur_dic[node] = {}\n",
    "                cur_dic = cur_dic[node]\n",
    "                if __class__.V not in cur_dic:\n",
    "                    cur_dic[__class__.V] = set()\n",
    "                cur_dic[__class__.V].add(url.netloc)\n",
    "                dicts.append(cur_dic)\n",
    "        self.max_n = 0\n",
    "        for d in dicts:\n",
    "            if isinstance(d[__class__.V], int):\n",
    "                continue\n",
    "            l = len(d[__class__.V])\n",
    "            d[__class__.V] = l\n",
    "            self.max_n = max(l, self.max_n)\n",
    "    \n",
    "    def getNormalizedNodes(self, path):\n",
    "        normalized_path = path.lstrip('/')\n",
    "        if normalized_path == '':\n",
    "            return ['']\n",
    "        return [''] + normalized_path.split('/')\n",
    "    \n",
    "    def getMaxSubpath(self, path):\n",
    "        cur_dic = self.trie\n",
    "        normalized_path = path.lstrip('/')\n",
    "        nodes = self.getNormalizedNodes(path)\n",
    "        n = 0\n",
    "        for node in nodes:\n",
    "            if node not in cur_dic:\n",
    "                break\n",
    "            n += 1\n",
    "            cur_dic = cur_dic[node]\n",
    "        return nodes[:n], cur_dic.get(__class__.V, 0)\n",
    "\"\"\"class PathTrie:\n",
    "    V = '!@__VALUE__@!'\n",
    "    def __init__(self, urls, progress=False):\n",
    "        if progress:\n",
    "            prog = tqdm\n",
    "        else:\n",
    "            prog = lambda x: x\n",
    "        self.trie = {}\n",
    "        dicts = []\n",
    "        for url in prog(urls):\n",
    "            path = url.path\n",
    "            cur_dic = self.trie\n",
    "            nodes = path.lstrip('/').split('/')\n",
    "            for node in nodes:\n",
    "                if node not in cur_dic:\n",
    "                    cur_dic[node] = {}\n",
    "                cur_dic = cur_dic[node]\n",
    "                if __class__.V not in cur_dic:\n",
    "                    cur_dic[__class__.V] = set()\n",
    "                cur_dic[__class__.V].add(url.netloc)\n",
    "                dicts.append(cur_dic)\n",
    "        self.max_n = 0\n",
    "        for d in dicts:\n",
    "            if isinstance(d[__class__.V], set):\n",
    "                l = len(d[__class__.V])\n",
    "                self.max_n = l if l > self.max_n else self.max_n\n",
    "                d[__class__.V] = l\n",
    "    \n",
    "    def getMaxSubpath(self, path):\n",
    "        cur_dic = self.trie\n",
    "        nodes = path.lstrip('/').split('/')\n",
    "        n = 0\n",
    "        for node in nodes:\n",
    "            if node not in cur_dic:\n",
    "                break\n",
    "            n += 1\n",
    "            cur_dic = cur_dic[node]\n",
    "        return nodes[:n], cur_dic.get(__class__.V, 0)\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.0, 0.0, 0.0, 1501977600.0, 1447372800.0, 1447372800.0, None]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# todo: ad_dm2time is not accurate\n",
    "\n",
    "ad_dm2time = TrieDict({\n",
    "    dm: t.timestamp() for dm, t in all_ad_dm_history.items()\n",
    "}, include_subdomain=True)\n",
    "\n",
    "[\n",
    "    ad_dm2time['doubleclick.com'],\n",
    "    ad_dm2time['foo.doubleclick.com'],\n",
    "    ad_dm2time['bazz.optmstr.com'],\n",
    "    ad_dm2time['jscdn.appier.net'],\n",
    "    ad_dm2time['boom.1ccbt.com'],\n",
    "    ad_dm2time['123.boom.1ccbt.com'],\n",
    "    ad_dm2time.get('benign.example.com', None)\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2ddff21db314bb9951610cd4604b034",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/46922 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4d8785329821488daa7f93c7abaab25f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/115753322 [00:03<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(['', 'pixel'], 133)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "sys.setrecursionlimit(0x100000)\n",
    "cache = Path('cache/all_path_trie.pickle')\n",
    "if cache.is_file():\n",
    "    all_path_trie = pickle.loads(cache.read_bytes())\n",
    "else:\n",
    "    urls = []\n",
    "    for site, time2parsed_urls in tqdm(site2time_parsed.items()):\n",
    "        for time, parsed_urls in time2parsed_urls.items():\n",
    "            urls += list(parsed_urls)\n",
    "\n",
    "    all_path_trie = PathTrie(urls, progress=True)\n",
    "    cache.write_bytes(pickle.dumps(all_path_trie))\n",
    "all_path_trie.getMaxSubpath('/pixel')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34c6f7ba4ccc463fb250d30e7469d2bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/46922 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d58efb9ad0054605ab1d642aaa4cf5d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/46922 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "252601\n"
     ]
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "cache = Path('cache/dm2info.pickle')\n",
    "if cache.is_file():\n",
    "    dm2info = pickle.loads(cache.read_bytes())\n",
    "else:\n",
    "    dm2info = {}\n",
    "    for site, time2parsed_urls in tqdm(site2time_parsed.items()):\n",
    "        for time, parsed_urls in time2parsed_urls.items():\n",
    "            for url in parsed_urls:\n",
    "                if url.netloc not in dm2info:\n",
    "                    dm2info[url.netloc] = {\n",
    "                        'first_appear': datetime(2100, 1, 1).timestamp(),\n",
    "                        'last_appear': datetime(1900, 1, 1).timestamp(),\n",
    "                        'paths': set()\n",
    "                    }\n",
    "                info = dm2info[url.netloc]\n",
    "                if time < info['first_appear']:\n",
    "                    info['first_appear'] = time\n",
    "                if time > info['last_appear']:\n",
    "                    info['last_appear'] = time\n",
    "                info['paths'].add(url.path)\n",
    "    # extended\n",
    "    for dm in dm2info:\n",
    "        dm2info[dm]['requests'] = []\n",
    "\n",
    "    for site, time2parsed_urls in tqdm(site2time_parsed.items()):\n",
    "        for t, parsed_urls in time2parsed_urls.items():\n",
    "            for url in parsed_urls:\n",
    "                req = {\n",
    "                    'url': url,\n",
    "                    'first_party': site,\n",
    "                    't': t,\n",
    "                    'is_ad': url in site2_ad_parsed2t[site]\n",
    "                }\n",
    "                if req['is_ad']:\n",
    "                    req['blocked_t'] = site2_ad_parsed2t[site][url]\n",
    "                    assert req['blocked_t'] is not None and int(req['blocked_t']) > 0\n",
    "                dm2info[url.netloc]['requests'].append(req)\n",
    "    cache.write_bytes(pickle.dumps(dm2info))\n",
    "print(len(dm2info))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "575.5px",
    "left": "22px",
    "top": "156.5px",
    "width": "264.25px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}