USDA_Apr_26.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/cpatiffanynguyen/usda/blob/main/USDA_Apr_26.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "YvVZRc5YuyGq"
      },
      "outputs": [],
      "source": [
        "import requests\n",
        "from requests.exceptions import ConnectionError\n",
        "from bs4 import BeautifulSoup\n",
        "import pandas as pd\n",
        "from PyPDF2 import PdfReader\n",
        "import io"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#List of PDF URLs\n"
      ],
      "metadata": {
        "id": "fVcDehYfBxeq"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Colorado"
      ],
      "metadata": {
        "id": "m6-WQl7rJlJS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pdf_urls_string = \"\"\"https://extension.colostate.edu/docs/pubs/insect/ihm-bee.pdf\n",
        "https://sam.extension.colostate.edu/wp-content/uploads/sites/2/2017/04/sam-irr-guide.pdf\n",
        "https://bookstore.ksre.ksu.edu/pubs/MF3523.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/02/Avian-Health-Deliverable-Draft98.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/02/Dead-Bird-Now-What-Infographic-w-Check-List1.pdf\n",
        "https://sam.extension.colostate.edu/wp-content/uploads/sites/2/2016/07/forage-guide.pdf\n",
        "https://sam.extension.colostate.edu/wp-content/uploads/sites/2/2018/07/GrazingGuide.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/11/FactSheet_E-Colorado-Grassland-Birds-Grazing_12-2022.pdf\n",
        "https://sam.extension.colostate.edu/wp-content/uploads/sites/2/2017/08/LivestockManagementintheMountains2017.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/03/Composting-Manual-2023_Web-FINAL.pdf\n",
        "https://sam.extension.colostate.edu/wp-content/uploads/sites/2/2018/07/manure-mgt.pdf\n",
        "https://extension.colostate.edu/docs/pubs/crops/xcm178.pdf\n",
        "https://extension.colostate.edu/docs/pubs/crops/xcm178.pdf\n",
        "https://extension.colostate.edu/docs/pubs/crops/xcm178.pdf\n",
        "https://extension.colostate.edu/docs/pubs/garden/xcm221.pdf\n",
        "https://extension.colostate.edu/docs/pubs/garden/xcm221.pdf\n",
        "https://extension.colostate.edu/docs/pubs/garden/xcm221.pdf\n",
        "https://webdoc.agsci.colostate.edu/soiltestinglab/soilsample_horticulture.pdf\n",
        "https://webdoc.agsci.colostate.edu/soiltestinglab/Forage_online_submittal_form.pdf\n",
        "https://agsci.colostate.edu/soiltestinglab/wp-content/uploads/sites/98/2022/01/Soil-Submittal-Form-for-Horticulture-Fillable.pdf\n",
        "https://agsci.colostate.edu/soiltestinglab/wp-content/uploads/sites/98/2022/01/Submittal-Form-for-Water-Fillable.pdf\n",
        "https://extension.colostate.edu/docs/pubs/insect/ihm-bee.pdf\n",
        "https://nativebeewatch.files.wordpress.com/2018/03/fieldguide_12march2018_lowrez.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/03/BeginnerBeeFieldGuide_LowRezWeb_28Feb2022.pdf\n",
        "https://extension.colostate.edu/docs/pubs/insect/ihm-bee.pdf\n",
        "https://nativebeewatch.files.wordpress.com/2018/03/fieldguide_12march2018_lowrez.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/03/BeginnerBeeFieldGuide_LowRezWeb_28Feb2022.pdf\n",
        "https://extension.colostate.edu/docs/pubs/insect/ihm-bee.pdf\n",
        "https://nativebeewatch.files.wordpress.com/2018/03/fieldguide_12march2018_lowrez.pdf\n",
        "https://extension.colostate.edu/docs/pubs/insect/ihm-bee.pdf\n",
        "https://extension.colostate.edu/docs/pubs/insect/ihm-bee.pdf\n",
        "https://sam.extension.colostate.edu/wp-content/uploads/sites/2/2016/07/forage-guide.pdf\n",
        "https://sam.extension.colostate.edu/wp-content/uploads/sites/2/2018/07/GrazingGuide.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/FrontRangeSm.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/FrontRange.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/MountainsSm.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/Mountains.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/PrairieSm.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/Prairie.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/SESm.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/SE.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/WestSlopeSm.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/WestSlope.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/sources-native-plants-2022.pdf\n",
        "https://extension.colostate.edu/docs/pubs/crops/xcm179.pdf\n",
        "https://extension.colostate.edu/docs/pubs/garden/xcm221.pdf\n",
        "https://extension.colostate.edu/docs/pubs/garden/xcm221.pdf\n",
        "https://extension.colostate.edu/docs/pubs/garden/xcm221.pdf\n",
        "https://extension.colostate.edu/docs/pubs/spanish/09329.pdf\n",
        "https://extension.colostate.edu/docs/pubs/spanish/09341.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/07/Roasting-Chile-Peppers.pdf\n",
        "https://extension.colostate.edu/docs/pubs/spanish/09380.pdf\n",
        "https://extension.colostate.edu/docs/pubs/spanish/09377.pdf\n",
        "https://extension.colostate.edu/docs/pubs/spanish/09367.pdf\n",
        "https://extension.colostate.edu/docs/pubs/spanish/09386.pdf\n",
        "https://extension.colostate.edu/docs/pubs/spanish/09334.pdf\n",
        "https://extension.colostate.edu/docs/pubs/spanish/09318.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/Defenders-of-Wildlife-Conflict-Reduction.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/livestock_and_wolves.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/PC-Tool-Resource-Guide.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/ReducingConflict_WLA-Guide_low-res-1.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/Tool-Kit-Grizzly-Bear-Conflict-Reduction-2017.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/Wilson-et-al.-2017-Learning-to-live-with-wolves-Blackfoot-Valley.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/WolfResourcesGuide.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/bro-livestock-protection-dog-public.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/A-Wyoming-Landowners-Handbook-to-Fences-and-Wildlife_2nd-Edition_-lo-res.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/FladryManual.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/installing-turbo-fladry-guide-ib.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/carcus-mgt.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/01/Review-of-livestock-compensation-programs-052620.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/10/Extension-Education-Breck-Wolf-behavior-and-mgt.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/10/Extension-Education-Collins-WLA_RCPP_Extension.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/10/Extension-Education-Collins-WLA_WWC_4Cs.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/10/Extension-Education-CPW-Wolf-Livestock-Compensation.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/10/Extension-Education-Gary-Cattle-records.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/10/Extension-Education-Hoag-Economic-value-of-wolves.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/10/Extension-Education-Stanko-Rancher-Community-dialogs.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/10/Extension-Education-Vail-Rocky-Mt-Wolf-Project.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2023/10/Extension-Education-Wilson-Blackfoot-Challenge.pdf\n",
        "https://sam.extension.colostate.edu/wp-content/uploads/sites/2/2017/04/sam-irr-guide.pdf\n",
        "https://extension.colostate.edu/docs/pubs/insect/ihm-bee.pdf\n",
        "https://extension.colostate.edu/wp-content/uploads/2022/10/Fire-Blight_2.907-2__OConnor-Zentz_10-22.pdf\n",
        "https://extension.colostate.edu/docs/pubs/spanish/07001.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/FrontRangeSm.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/FrontRange.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/MountainsSm.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/Mountains.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/PrairieSm.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/Prairie.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/SESm.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/SE.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/WestSlopeSm.pdf\n",
        "https://extension.colostate.edu/docs/pubs/native/WestSlope.pdf\n",
        "\"\"\""
      ],
      "metadata": {
        "id": "tn_NUpE_FyPF"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "###Downloads PDFs from a list of URLs and saves them locally"
      ],
      "metadata": {
        "id": "RWLgGcEXkO50"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "\n",
        "def download_pdfs(pdf_urls):\n",
        "  for url in pdf_urls:\n",
        "    response = requests.get(url)\n",
        "    filename = url.split('/')[-1]\n",
        "    with open(filename, 'wb') as f:\n",
        "      f.write(response.content)\n",
        "\n",
        "pdf_urls = pdf_urls_string.splitlines()\n",
        "download_pdfs(pdf_urls)\n"
      ],
      "metadata": {
        "id": "CornsGwikcqg"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from PyPDF2 import PdfReader\n",
        "\n",
        "def extract_text(filename):\n",
        "  with open(filename, 'rb') as f:\n",
        "    pdf_reader = PdfReader(f)\n",
        "    page_text = \"\"\n",
        "    for page in pdf_reader.pages:\n",
        "      page_text += page.extract_text()\n",
        "  return page_text\n"
      ],
      "metadata": {
        "id": "o4erXHG0oZxR"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "###Save the PDF file to a separate TXT file (optional)"
      ],
      "metadata": {
        "id": "yJjytUrIn8AR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from urllib.request import urlretrieve\n",
        "from PyPDF2 import PdfReader\n",
        "\n",
        "def download_and_extract_text(pdf_url, filename='ihm-bee.txt'):\n",
        "  urlretrieve(pdf_url, filename)\n",
        "  with open(filename, 'rb') as pdf_file:\n",
        "    pdf_reader = PdfReader(pdf_file)\n",
        "    text = \"\"\n",
        "    for page in pdf_reader.pages:\n",
        "      text += page.extract_text()\n",
        "  return text\n",
        "\n",
        "pdf_url = \"https://extension.colostate.edu/docs/pubs/insect/ihm-bee.pdf\"\n",
        "ihm_bee_text = download_and_extract_text(pdf_url)\n",
        "with open('ihm_bee_content.txt', 'w') as f:\n",
        "  f.write(ihm_bee_text)\n"
      ],
      "metadata": {
        "id": "GAUDC-lvk93h"
      },
      "execution_count": 12,
      "outputs": []
    }
  ]
}