ecfr_parser_pyspark_.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ecfr_parser(pyspark).ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "qMvcXZNVpZp2",
        "outputId": "0115b559-42c9-4d48-a41c-c3ea10ddd412",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 258
        }
      },
      "source": [
        "!pip install pyspark"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pyspark\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/87/21/f05c186f4ddb01d15d0ddc36ef4b7e3cedbeb6412274a41f26b55a650ee5/pyspark-2.4.4.tar.gz (215.7MB)\n",
            "\u001b[K     |████████████████████████████████| 215.7MB 25.8MB/s \n",
            "\u001b[?25hCollecting py4j==0.10.7 (from pyspark)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e3/53/c737818eb9a7dc32a7cd4f1396e787bd94200c3997c72c1dbe028587bd76/py4j-0.10.7-py2.py3-none-any.whl (197kB)\n",
            "\u001b[K     |████████████████████████████████| 204kB 45.0MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-2.4.4-py2.py3-none-any.whl size=216130387 sha256=753732181d8553ea3ff9ee0033f3714e3960fbc6d325c2e54eb7d09bff7c613b\n",
            "  Stored in directory: /root/.cache/pip/wheels/ab/09/4d/0d184230058e654eb1b04467dbc1292f00eaa186544604b471\n",
            "Successfully built pyspark\n",
            "Installing collected packages: py4j, pyspark\n",
            "Successfully installed py4j-0.10.7 pyspark-2.4.4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Himuefmdow0f",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import requests\n",
        "import re\n",
        "from bs4 import BeautifulSoup as bs \n",
        "import pyspark\n",
        "\n",
        "from pyspark import SparkConf,  SparkContext\n",
        "from pyspark.sql import SparkSession, functions as F\n",
        "from pyspark.sql.types import *\n",
        "import sys\n",
        "sys.setrecursionlimit(100000)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7sAi_SY09w-L",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sc = SparkContext(conf=SparkConf())\n",
        "spark = SparkSession(sc)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_hCLFV9DpI2W",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def ecfr_parser(title_num):\n",
        "    \"\"\"\n",
        "    Simply enter the title number of the eCFR you'd like to parse (e.g. 16).\n",
        "    This parser does not work on eCFR Titles 2, 3, 6, 13, 34, 35, and 38.\n",
        "    \"\"\"\n",
        "\n",
        "    SCHEMA = StructType([StructField('chapter', StringType(), True),\n",
        "                         StructField('chapter_title', StringType(), True),\n",
        "                         StructField('subchapter', StringType(), True),\n",
        "                         StructField('subchapter_title', StringType(), True),\n",
        "                         StructField('part', StringType(), True),\n",
        "                         StructField('part_title', StringType(), True),\n",
        "                         StructField('section', StringType(), True),\n",
        "                         StructField('section_title', StringType(), True),\n",
        "                         StructField('section_text', StringType(), True)])\n",
        "\n",
        "    xml = requests.get(\"https://www.govinfo.gov/bulkdata/ECFR/title-\"+str(title_num)+\"/ECFR-title\"+str(title_num)+\".xml\")\n",
        "    soup = bs(xml.content, 'xml') \n",
        "    \n",
        "    list_of_dicts = []               \n",
        " \n",
        "    chapters = soup.find_all('DIV3')\n",
        "    for chapter in chapters:\n",
        "      chapter_num = chapter.attrs['N']\n",
        "      chapter_title = chapter.find('HEAD').text\n",
        "\n",
        "      subchapters = chapter.find_all('DIV4')\n",
        "      for subchapter in subchapters:\n",
        "          subchapter_num = subchapter.attrs['N']\n",
        "          subchapter_title = subchapter.find('HEAD').text\n",
        "\n",
        "          parts = subchapter.find_all('DIV5')\n",
        "          for part in parts:\n",
        "              part_num = part.attrs['N']\n",
        "              part_title = part.find('HEAD').text\n",
        "\n",
        "              sections = part.find_all('DIV8') \n",
        "              for section in sections:\n",
        "                  section_num = section.attrs['N'][2:]\n",
        "                  section_title = section.find('HEAD').text\n",
        "                  section_text = section.find_all('P')\n",
        "\n",
        "                  list_of_dicts.append({'chapter': chapter_num, 'chapter_title': chapter_title,\n",
        "                                        'subchapter': subchapter_num, 'subchapter_title': subchapter_title, \n",
        "                                        'part': part_num, 'part_title': part_title, 'section': section_num, \n",
        "                                        'section_title': section_title, 'section_text': str(section_text)})\n",
        "\n",
        "    df = spark.createDataFrame(list_of_dicts, SCHEMA)\n",
        "\n",
        "    regex = \"\\[+|\\]+|<[A-Z]+>+|<\\/[A-Z]+>+|\\\\n+\"\n",
        "    df = df.withColumn('section_text', F.regexp_replace(df.section_text, regex,''))\n",
        "\n",
        "    df = df.withColumn('id', F.md5(F.concat('chapter_title', 'subchapter_title', 'part_title', 'section_title')))\n",
        "                \n",
        "    return df.show() #using .show() only for the purposes of displaying my results"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_Schy_ZIpfGT",
        "colab_type": "code",
        "outputId": "bbf13c51-5c51-4359-e89f-176951a60820",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 459
        }
      },
      "source": [
        "ecfr_parser(16)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+-------+--------------------+----------+--------------------+----+--------------------+-------+--------------------+--------------------+--------------------+\n",
            "|chapter|       chapter_title|subchapter|    subchapter_title|part|          part_title|section|       section_title|        section_text|                  id|\n",
            "+-------+--------------------+----------+--------------------+----+--------------------+-------+--------------------+--------------------+--------------------+\n",
            "|      I| CHAPTER I - FEDE...|         A|SUBCHAPTER A - OR...|   0|PART 0 - ORGANIZA...|    0.1|§ 0.1   The Commi...|The Federal Trade...|2fe0b83177937e17d...|\n",
            "|      I| CHAPTER I - FEDE...|         A|SUBCHAPTER A - OR...|   0|PART 0 - ORGANIZA...|    0.2|§ 0.2   Official ...|The principal off...|aff4c7f07a8f5c9c5...|\n",
            "|      I| CHAPTER I - FEDE...|         A|SUBCHAPTER A - OR...|   0|PART 0 - ORGANIZA...|    0.3|      § 0.3   Hours.|Principal and fie...|b20c61abfc2b34ccc...|\n",
            "|      I| CHAPTER I - FEDE...|         A|SUBCHAPTER A - OR...|   0|PART 0 - ORGANIZA...|    0.4|§ 0.4   Laws admi...|The Commission ex...|110f4e48b541d8b61...|\n",
            "|      I| CHAPTER I - FEDE...|         A|SUBCHAPTER A - OR...|   0|PART 0 - ORGANIZA...|    0.5|§ 0.5   Laws auth...|The Commission is...|e7c16db052e75bc49...|\n",
            "|      I| CHAPTER I - FEDE...|         A|SUBCHAPTER A - OR...|   0|PART 0 - ORGANIZA...|    0.6|  § 0.6   [Reserved]|                    |53947b7c753cbc546...|\n",
            "|      I| CHAPTER I - FEDE...|         A|SUBCHAPTER A - OR...|   0|PART 0 - ORGANIZA...|    0.7|§ 0.7   Delegatio...|(a) The Commissio...|f38f1c9f421a4967a...|\n",
            "|      I| CHAPTER I - FEDE...|         A|SUBCHAPTER A - OR...|   0|PART 0 - ORGANIZA...|    0.8|§ 0.8   The Chair...|The Chairman of t...|a4301b0d805bc2a77...|\n",
            "|      I| CHAPTER I - FEDE...|         A|SUBCHAPTER A - OR...|   0|PART 0 - ORGANIZA...|    0.9|§ 0.9   Organizat...|The Federal Trade...|54757ec2405d9af7e...|\n",
            "|      I| CHAPTER I - FEDE...|         A|SUBCHAPTER A - OR...|   0|PART 0 - ORGANIZA...|   0.10|§ 0.10   Office o...|The Executive Dir...|ed3312bc28fb5dac9...|\n",
            "|      I| CHAPTER I - FEDE...|         A|SUBCHAPTER A - OR...|   0|PART 0 - ORGANIZA...|   0.11|§ 0.11   Office o...|The General Couns...|eaad7c4d1ebb95660...|\n",
            "|      I| CHAPTER I - FEDE...|         A|SUBCHAPTER A - OR...|   0|PART 0 - ORGANIZA...|   0.12|§ 0.12   Office o...|The Secretary is ...|a01db2e34066d85fc...|\n",
            "|      I| CHAPTER I - FEDE...|         A|SUBCHAPTER A - OR...|   0|PART 0 - ORGANIZA...|   0.13|§ 0.13   Office o...|The Office of Ins...|de0d2c729ef825a46...|\n",
            "|      I| CHAPTER I - FEDE...|         A|SUBCHAPTER A - OR...|   0|PART 0 - ORGANIZA...|   0.14|§ 0.14   Office o...|Administrative la...|6ba9f88ce4caf0ebe...|\n",
            "|      I| CHAPTER I - FEDE...|         A|SUBCHAPTER A - OR...|   0|PART 0 - ORGANIZA...|   0.15| § 0.15   [Reserved]|                    |0607dd8042f952f64...|\n",
            "|      I| CHAPTER I - FEDE...|         A|SUBCHAPTER A - OR...|   0|PART 0 - ORGANIZA...|   0.16|§ 0.16   Bureau o...|The Bureau is res...|9cad1a7940fa450ef...|\n",
            "|      I| CHAPTER I - FEDE...|         A|SUBCHAPTER A - OR...|   0|PART 0 - ORGANIZA...|   0.17|§ 0.17   Bureau o...|The Bureau invest...|0ff44e78354aaad86...|\n",
            "|      I| CHAPTER I - FEDE...|         A|SUBCHAPTER A - OR...|   0|PART 0 - ORGANIZA...|   0.18|§ 0.18   Bureau o...|The bureau aids a...|aab87b0ed8a8987af...|\n",
            "|      I| CHAPTER I - FEDE...|         A|SUBCHAPTER A - OR...|   0|PART 0 - ORGANIZA...|   0.19|§ 0.19   The Regi...|(a) These offices...|c42b9f84d2e76f6b4...|\n",
            "|      I| CHAPTER I - FEDE...|         A|SUBCHAPTER A - OR...|   0|PART 0 - ORGANIZA...|   0.20|§ 0.20   Office o...|The Office of Int...|9c72c1aea530e6efa...|\n",
            "+-------+--------------------+----------+--------------------+----+--------------------+-------+--------------------+--------------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7XS9iSTwkIPD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}