diff --git a/README.rst b/README.rst index 542bd413..39f5b5a8 100644 --- a/README.rst +++ b/README.rst @@ -2,31 +2,31 @@ Energy Language Model (ELM) *************************** -.. image:: https://github.com/NREL/elm/workflows/Documentation/badge.svg - :target: https://nrel.github.io/sup3r/ +.. image:: https://github.com/NatLabRockies/elm/workflows/Documentation/badge.svg + :target: https://natlabrockies.github.io/elm/ -.. image:: https://github.com/NREL/elm/workflows/pytests/badge.svg - :target: https://github.com/NREL/elm/actions?query=workflow%3A%22pytests%22 +.. image:: https://github.com/NatLabRockies/elm/workflows/pytests/badge.svg + :target: https://github.com/NatLabRockies/elm/actions?query=workflow%3A%22pytests%22 -.. image:: https://github.com/NREL/elm/workflows/Lint%20Code%20Base/badge.svg - :target: https://github.com/NREL/elm/actions?query=workflow%3A%22Lint+Code+Base%22 +.. image:: https://github.com/NatLabRockies/elm/workflows/Lint%20Code%20Base/badge.svg + :target: https://github.com/NatLabRockies/elm/actions?query=workflow%3A%22Lint+Code+Base%22 -.. image:: https://img.shields.io/pypi/pyversions/NREL-elm.svg - :target: https://pypi.org/project/NREL-elm/ +.. image:: https://img.shields.io/pypi/pyversions/NLR-elm.svg + :target: https://pypi.org/project/NLR-elm/ -.. image:: https://badge.fury.io/py/NREL-elm.svg - :target: https://badge.fury.io/py/NREL-elm +.. image:: https://badge.fury.io/py/NLR-elm.svg + :target: https://badge.fury.io/py/NLR-elm .. image:: https://zenodo.org/badge/690793778.svg :target: https://zenodo.org/doi/10.5281/zenodo.10070538 The Energy Language Model (ELM) software provides interfaces to apply Large Language Models (LLMs) like ChatGPT and GPT-4 to energy research. For example, you might be interested in: -- `Converting PDFs into a text database `_ -- `Chunking text documents and embedding into a vector database `_ -- `Performing recursive document summarization `_ -- `Building an automated data extraction workflow using decision trees `_ -- `Building a chatbot app that interfaces with reports from OSTI `_ +- `Converting PDFs into a text database `_ +- `Chunking text documents and embedding into a vector database `_ +- `Performing recursive document summarization `_ +- `Building an automated data extraction workflow using decision trees `_ +- `Building a chatbot app that interfaces with reports from OSTI `_ Installing ELM ============== @@ -34,15 +34,15 @@ Installing ELM .. inclusion-install NOTE: If you are installing ELM to run ordinance scraping and extraction, -see the `ordinance-specific installation instructions `_. +see the `ordinance-specific installation instructions `_. Option #1 (basic usage): -#. ``pip install NREL-elm`` +#. ``pip install NLR-elm`` Option #2 (developer install): -#. from home dir, ``git clone git@github.com:NREL/elm.git`` +#. from home dir, ``git clone git@github.com:NatLabRockies/elm.git`` #. Create ``elm`` environment and install package a) Create a conda env: ``conda create -n elm`` b) Run the command: ``conda activate elm`` @@ -58,4 +58,4 @@ Option #2 (developer install): Acknowledgments =============== -This work was authored by the National Renewable Energy Laboratory, operated by Alliance for Sustainable Energy, LLC, for the U.S. Department of Energy (DOE) under Contract No. DE-AC36-08GO28308. Funding provided by the DOE Wind Energy Technologies Office (WETO), the DOE Solar Energy Technologies Office (SETO), and internal research funds at the National Renewable Energy Laboratory. The views expressed in the article do not necessarily represent the views of the DOE or the U.S. Government. The U.S. Government retains and the publisher, by accepting the article for publication, acknowledges that the U.S. Government retains a nonexclusive, paid-up, irrevocable, worldwide license to publish or reproduce the published form of this work, or allow others to do so, for U.S. Government purposes. +This work was authored by the National Laboratory of the Rockies, operated by Alliance for Energy Innovation, LLC, for the U.S. Department of Energy (DOE) under Contract No. DE-AC36-08GO28308. Funding provided by the DOE Wind Energy Technologies Office (WETO), the DOE Solar Energy Technologies Office (SETO), and internal research funds at the National Laboratory of the Rockies. The views expressed in the article do not necessarily represent the views of the DOE or the U.S. Government. The U.S. Government retains and the publisher, by accepting the article for publication, acknowledges that the U.S. Government retains a nonexclusive, paid-up, irrevocable, worldwide license to publish or reproduce the published form of this work, or allow others to do so, for U.S. Government purposes. diff --git a/docs/source/conf.py b/docs/source/conf.py index 86843733..f305ea82 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -115,7 +115,7 @@ html_context = { "display_github": True, - "github_user": "nrel", + "github_user": "nlr", "github_repo": "elm", "github_version": "main", "conf_py_path": "/docs/source/", diff --git a/docs/source/dev/ords_architecture.rst b/docs/source/dev/ords_architecture.rst index 39fa6ea9..d97af80a 100644 --- a/docs/source/dev/ords_architecture.rst +++ b/docs/source/dev/ords_architecture.rst @@ -326,11 +326,11 @@ resources, we can use ``Services`` to monitor their use as well! Let's look at a There are several other services provided out of the box - see the -`documentation `_ for details +`documentation `_ for details Alternatively, we provide two base classes that you can extend to get similar functionality: -`ThreadedService `_ +`ThreadedService `_ for threaded tasks and -`ProcessPoolService `_ +`ProcessPoolService `_ for multiprocessing tasks. **4.2 Key Classes** @@ -763,9 +763,9 @@ We give a rough breakdown of the following call: from elm.web.search import web_search_links_as_docs QUERIES = [ - "NREL wiki", + "NLR wiki", "National Renewable Energy Laboratory director", - "NREL leadership wikipedia", + "NLR leadership wikipedia", ] async def main(): diff --git a/elm/__init__.py b/elm/__init__.py index b28c2c6d..8398c070 100644 --- a/elm/__init__.py +++ b/elm/__init__.py @@ -16,7 +16,7 @@ from elm.web.osti import OstiRecord, OstiList __author__ = """Grant Buster""" -__email__ = "Grant.Buster@nrel.gov" +__email__ = "Grant.Buster@nlr.gov" ELM_DIR = os.path.dirname(os.path.realpath(__file__)) TEST_DATA_DIR = os.path.join(os.path.dirname(ELM_DIR), 'tests', 'data') diff --git a/elm/ords/README.md b/elm/ords/README.md index e8032415..9cc0167e 100644 --- a/elm/ords/README.md +++ b/elm/ords/README.md @@ -16,7 +16,7 @@ Then, install `pdftotext`: $ pip install pytesseract pdf2image -At this point, you can install ELM per the [front-page README](https://github.com/NREL/elm/blob/main/README.rst) instructions, e.g.: +At this point, you can install ELM per the [front-page README](https://github.com/NatLabRockies/elm/blob/main/README.rst) instructions, e.g.: $ pip install -e . @@ -25,11 +25,11 @@ To do so, simply run: $ rebrowser_playwright install -Now you are ready to run ordinance retrieval and extraction. See the [example](https://github.com/NREL/elm/blob/main/examples/ordinance_gpt/README.rst) to get started. If you get additional import errors, just install additional packages as necessary, e.g.: +Now you are ready to run ordinance retrieval and extraction. See the [example](https://github.com/NatLabRockies/elm/blob/main/examples/ordinance_gpt/README.rst) to get started. If you get additional import errors, just install additional packages as necessary, e.g.: $ pip install beautifulsoup4 html5lib ## Architecture -For information on the architectural design of this code, see the [design document](https://nrel.github.io/elm/dev/ords_architecture.html). +For information on the architectural design of this code, see the [design document](https://natlabrockies.github.io/elm/dev/ords_architecture.html). diff --git a/elm/version.py b/elm/version.py index b42fce51..f9963abf 100644 --- a/elm/version.py +++ b/elm/version.py @@ -2,4 +2,4 @@ ELM version number """ -__version__ = "0.0.34" +__version__ = "0.0.35" diff --git a/elm/web/rhub.py b/elm/web/rhub.py index 7b88503c..5eef93d9 100644 --- a/elm/web/rhub.py +++ b/elm/web/rhub.py @@ -17,8 +17,8 @@ class ProfilesRecord(dict): """Class to handle a single profiles as dictionary data. This class requires setting an 'RHUB_API_KEY' environment variable to access the Pure Web Service. The API key can be - obtained by contacting an NREL library representative: - Library@nrel.gov. + obtained by contacting an NLR library representative: + Library@nlr.gov. """ def __init__(self, record): """ @@ -333,8 +333,8 @@ class ProfilesList(list): """Class to retrieve and handle multiple profiles from an API URL. This class requires setting an 'RHUB_API_KEY' environment variable to access the Pure Web Service. The API key can be - obtained by contacting an NREL library representative: - Library@nrel.gov. + obtained by contacting an NLR library representative: + Library@nlr.gov. """ def __init__(self, url, n_pages=1): """ @@ -497,8 +497,8 @@ class PublicationsRecord(dict): """Class to handle a single publication as dictionary data. This class requires setting an 'RHUB_API_KEY' environment variable to access the Pure Web Service. The API key can be - obtained by contacting an NREL library representative: - Library@nrel.gov. + obtained by contacting an NLR library representative: + Library@nlr.gov. """ def __init__(self, record): """ @@ -736,8 +736,8 @@ class PublicationsList(list): """Class to retrieve and handle multiple publications from an API URL. This class requires setting an 'RHUB_API_KEY' environment variable to access the Pure Web Service. The API key can be - obtained by contacting an NREL library representative: - Library@nrel.gov. + obtained by contacting an NLR library representative: + Library@nlr.gov. """ def __init__(self, url, n_pages=1): """ diff --git a/examples/energy_wizard/README.rst b/examples/energy_wizard/README.rst index 4c0e8a1e..0aa8c28f 100644 --- a/examples/energy_wizard/README.rst +++ b/examples/energy_wizard/README.rst @@ -17,7 +17,7 @@ Notes: Downloading and Embedding PDFs ============================== -Run ``python ./retrieve_docs.py`` to retrieve 20 of the latest NREL technical +Run ``python ./retrieve_docs.py`` to retrieve 20 of the latest NLR technical reports from OSTI. The script then converts the PDFs to text and then runs the text through the OpenAI embedding model. diff --git a/examples/energy_wizard/retrieve_docs.py b/examples/energy_wizard/retrieve_docs.py index 3968fa41..4f708448 100644 --- a/examples/energy_wizard/retrieve_docs.py +++ b/examples/energy_wizard/retrieve_docs.py @@ -17,7 +17,7 @@ init_logger('elm', log_level='INFO') -# NREL-Azure endpoint. You can also use just the openai endpoint. +# NLR-Azure endpoint. You can also use just the openai endpoint. # NOTE: embedding values are different between OpenAI and Azure models! openai.api_base = os.getenv("AZURE_OPENAI_ENDPOINT") openai.api_key = os.getenv("AZURE_OPENAI_KEY") diff --git a/examples/energy_wizard/run_app.py b/examples/energy_wizard/run_app.py index cd18e488..8c6e6312 100644 --- a/examples/energy_wizard/run_app.py +++ b/examples/energy_wizard/run_app.py @@ -11,7 +11,7 @@ model = 'gpt-4' -# NREL-Azure endpoint. You can also use just the openai endpoint. +# NLR-Azure endpoint. You can also use just the openai endpoint. # NOTE: embedding values are different between OpenAI and Azure models! openai.api_base = os.getenv("AZURE_OPENAI_ENDPOINT") openai.api_key = os.getenv("AZURE_OPENAI_KEY") diff --git a/examples/ordinance_gpt/README.rst b/examples/ordinance_gpt/README.rst index 78e95575..2e969306 100644 --- a/examples/ordinance_gpt/README.rst +++ b/examples/ordinance_gpt/README.rst @@ -7,12 +7,12 @@ This example folder contains supporting documents, results, and code for the Ord Prerequisites ============= We recommend installing the pytesseract module to allow PDF retrieval for scanned documents. -See the `ordinance-specific installation instructions `_ +See the `ordinance-specific installation instructions `_ for more details. Running from Python =================== -This instruction set presents a simplified example to extract ordinance data from a ordinance document on disk. This corresponds with the ordinance data extraction from PDF results in `Buster et al., 2024 `_. +This instruction set presents a simplified example to extract ordinance data from a ordinance document on disk. This corresponds with the ordinance data extraction from PDF results in `Buster et al., 2024 `_. To run this, first download one or more ordinance documents from `the Box folder `_. @@ -23,17 +23,17 @@ After downloading the ordinance document(s), set the relevant path for the ``fp_ Running from the Command Line Utility ===================================== -This instruction set is an experimental process to use LLMs to search the internet for relevant ordinance documents, download those documents, and then extract the relevant ordinance data. +This instruction set is an experimental process to use LLMs to search the internet for relevant ordinance documents, download those documents, and then extract the relevant ordinance data. There are a few key things you need to set up in order to run ordinance retrieval and extraction. First, you must specify which counties you want to process. You can do this by setting up a CSV file with a ``County`` and a ``State`` column. Each row in the CSV file then represents a single county to process. -See the `example CSV `_ +See the `example CSV `_ file for reference. Once you have set up the county CSV, you can fill out the -`template JSON config `_. -See the documentation for the `"process_counties_with_openai" function `_ +`template JSON config `_. +See the documentation for the `"process_counties_with_openai" function `_ for an explanation of all the allowed inputs to the configuration file. Some notable inputs here are the ``azure*`` keys, which should be configured to match your Azure OpenAI API deployment (unless it's defined in your environment with the ``AZURE_OPENAI_API_KEY``, ``AZURE_OPENAI_VERSION``, @@ -60,7 +60,7 @@ Debugging --------- Not sure why things aren't working? No error messages? Make sure you run the CLI call with a ``-v`` flag for "verbose" logging (e.g., ``$ elm ords -c config.json -v``) -Errors on import statements? Trouble importing ``pdftotext`` with cryptic error messages like ``symbol not found in flat namespace``? Follow the `ordinance-specific install instructions `_ *exactly*. +Errors on import statements? Trouble importing ``pdftotext`` with cryptic error messages like ``symbol not found in flat namespace``? Follow the `ordinance-specific install instructions `_ *exactly*. Source Ordinance Documents ========================== @@ -71,10 +71,10 @@ The ordinance documents downloaded using (an older version of) this example code Extension to Other Technologies =============================== Extending this functionality to other technologies is possible but requires deeper understanding of the underlying processes. -We recommend you start out by examining the decision tree queries in `graphs.py `_ -as well as how they are applied in `parse.py `_. Once you +We recommend you start out by examining the decision tree queries in `graphs.py `_ +as well as how they are applied in `parse.py `_. Once you have a firm understanding of these two modules, look through the -`document validation routines `_ to get a better sense of how to +`document validation routines `_ to get a better sense of how to adjust the web-scraping portion of the code to your technology. When you have set up the validation and parsing for your -technology, put it all together by adjusting the `"process_counties_with_openai" function `_ +technology, put it all together by adjusting the `"process_counties_with_openai" function `_ to call your new routines. diff --git a/examples/research_hub/retrieve_docs.py b/examples/research_hub/retrieve_docs.py index 999e757d..7b3e18f3 100644 --- a/examples/research_hub/retrieve_docs.py +++ b/examples/research_hub/retrieve_docs.py @@ -21,7 +21,7 @@ init_logger('elm', log_level='INFO') -# NREL-Azure endpoint. You can also use just the openai endpoint. +# NLR-Azure endpoint. You can also use just the openai endpoint. # NOTE: embedding values are different between OpenAI and Azure models! openai.api_base = os.getenv("AZURE_OPENAI_ENDPOINT") openai.api_key = os.getenv("AZURE_OPENAI_KEY") diff --git a/examples/research_hub/run_app.py b/examples/research_hub/run_app.py index 8ba814f6..c2a14da6 100644 --- a/examples/research_hub/run_app.py +++ b/examples/research_hub/run_app.py @@ -11,7 +11,7 @@ model = 'gpt-4' -# NREL-Azure endpoint. You can also use just the openai endpoint. +# NLR-Azure endpoint. You can also use just the openai endpoint. # NOTE: embedding values are different between OpenAI and Azure models! openai.api_base = os.getenv("AZURE_OPENAI_ENDPOINT") openai.api_key = os.getenv("AZURE_OPENAI_KEY") diff --git a/examples/web_information_retrieval/README.md b/examples/web_information_retrieval/README.md index a87bdf6e..8fb19406 100644 --- a/examples/web_information_retrieval/README.md +++ b/examples/web_information_retrieval/README.md @@ -5,7 +5,7 @@ In this directory, we provide a set of examples that demonstrate how to use ELM - [`Custom web information retrieval using search engines`](./example_search_retrieval_wiki.ipynb) In this example, we demonstrate how to set up your own end-to-end web and information retrieval pipeline. - Specifically, we set up a procedure to extract the name of NREL's current director using only Wikipedia articles. + Specifically, we set up a procedure to extract the name of NLR's current director using only Wikipedia articles. > [!NOTE] > Due to the non-deterministic nature of several pipeline components (Google search, LLM), you may get diff --git a/examples/web_information_retrieval/example_search_retrieval_wiki.ipynb b/examples/web_information_retrieval/example_search_retrieval_wiki.ipynb index 043fea7b..88b9ea19 100644 --- a/examples/web_information_retrieval/example_search_retrieval_wiki.ipynb +++ b/examples/web_information_retrieval/example_search_retrieval_wiki.ipynb @@ -7,8 +7,8 @@ "# Custom web information retrieval using search engines\n", "\n", "In this demo, we will build a custom search-engine-based information extraction pipeline based on the ordinance web scraping tool. \n", - "See the [ordinance example](https://github.com/NREL/elm/tree/main/examples/ordinance_gpt) if you are interested in that particular\n", - "example. For this exercise, we will set up a pipeline to extract the name of the current director of NREL based on Wikipedia articles. \n", + "See the [ordinance example](https://github.com/NatLabRockies/elm/tree/main/examples/ordinance_gpt) if you are interested in that particular\n", + "example. For this exercise, we will set up a pipeline to extract the name of the current director of NLR based on Wikipedia articles. \n", "\n", "## General pipeline structure\n", "\n", @@ -24,16 +24,16 @@ "## Scraping text from Google search results\n", "\n", "We will begin by setting up the Google search. To do this, we must come up with one or more relevant queries. \n", - "Since we are interested in looking up the director of NREL using Wikipedia articles about NREL, we can use the following search queries:" + "Since we are interested in looking up the director of NLR using Wikipedia articles about NLR, we can use the following search queries:" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "QUERIES = [\"NREL wiki\", \"National Renewable Energy Laboratory director\"]" + "QUERIES = [\"NLR wiki\", \"National Laboratory of the Rockies director\"]" ] }, { @@ -74,23 +74,9 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[,\n", - " ,\n", - " ,\n", - " ]" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "docs" ] @@ -107,18 +93,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'source': 'https://en.wikipedia.org/?title=National_Renewable_Energy_Lab&redirect=no'}\n", - "{'source': 'https://www.energy.gov/person/dr-martin-keller#:~:text=Martin%20Keller-,Dr.,Alliance%20for%20Sustainable%20Energy%2C%20LLC.'}\n", - "{'source': 'https://www2.nrel.gov/about/leadership'}\n", - "{'source': 'https://www.linkedin.com/in/martin-keller-a09b016'}\n" - ] - } - ], + "outputs": [], "source": [ "for d in docs:\n", " print(d.attrs)" @@ -128,9 +103,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Excellent, it looks like we definitely have some Wikipedia articles about NREL among the results.\n", + "Excellent, it looks like we definitely have some Wikipedia articles about NLR among the results.\n", "\n", - "However, we also have some documents from links in which we are not interested (e.g., we would like to ignore the official `nrel.gov` page for this exercise). This will generally be true for every analysis, since Google search results can vary broadly. What we need to do next, then, is filter the results down to only include the sources we are interested in.\n", + "However, we also have some documents from links in which we are not interested (e.g., we would like to ignore the official `nlr.gov` page for this exercise). This will generally be true for every analysis, since Google search results can vary broadly. What we need to do next, then, is filter the results down to only include the sources we are interested in.\n", "\n", "\n", "## Filtering results\n", @@ -142,7 +117,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -163,15 +138,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'source': 'https://en.wikipedia.org/wiki/National_Renewable_Energy_Laboratory'}\n" - ] - } - ], + "outputs": [], "source": [ "from elm.web.utilities import filter_documents\n", "\n", @@ -193,7 +160,7 @@ "## Extracting relevant text\n", "Unfortunately, just because we filtered down to the documents we are interested in does not usually mean we can dive right into extracting values. Often, the documents we are examining contain a **lot** of text, most of which is **not** relevant to the question at hand (e.g., ordinance documents can be hundreds of pages long, and often we are just interested in the information found in one small section). \n", "\n", - "To get around this, we leverage the LLM to parse the text and extract only the text we are interested in. Let's write another function to call the LLM on chunks of the document text and determine whether the text contains information about NREL's director.\n", + "To get around this, we leverage the LLM to parse the text and extract only the text we are interested in. Let's write another function to call the LLM on chunks of the document text and determine whether the text contains information about NLR's director.\n", "After parsing all of the text chunks, we will stitch back together the relevant chunks to give us only the relevant text:" ] }, @@ -214,10 +181,10 @@ " \"of the original text directly.\"\n", ")\n", "INSTRUCTIONS = (\n", - " \"Extract one or more direct text excerpts related to leadership at NREL. \"\n", + " \"Extract one or more direct text excerpts related to leadership at NLR. \"\n", " \"Be sure to include any relevant names and position titles. Include \"\n", " \"section headers (if any) for the text excerpts. If there is no text \"\n", - " \"related to leadership at NREL, simply say: \"\n", + " \"related to leadership at NLR, simply say: \"\n", " '\"No relevant text.\"'\n", ")\n", "\n", @@ -280,7 +247,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -305,7 +272,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -336,43 +303,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "SOURCE: https://en.wikipedia.org/wiki/National_Renewable_Energy_Laboratory\n", - "================================\n", - "## History\n", - "\n", - "[edit]\n", - "\n", - "Martin Keller became NREL's ninth director in November 2015,[10] and currently\n", - "serves as both the director of the laboratory and the president of its\n", - "operating contractor, Alliance for Sustainable Energy, LLC.[11] He succeeded\n", - "Dan Arvizu, who retired in September 2015 after 10 years in those roles.[12]\n", - "\"Dr. Martin Keller Named Director of National Renewable Energy Laboratory\". _National Renewable Energy Laboratory_. Retrieved June 27, 2017.\n", - "\n", - "\"Dr. Martin Keller – Laboratory Director\". Retrieved January 30, 2017.\n", - "\n", - "SOURCE: https://en.wikipedia.org/wiki/United_States_Department_of_Energy_National_Laboratories\n", - "================================\n", - "\"National Renewable Energy Laboratory (NREL)\n", - "\n", - "Golden, Colorado, 1977\n", - "\n", - "Operating organization:\n", - "\n", - "Alliance for Sustainable Energy, LLC (since 2008)[11]\n", - "\n", - "Number of employees/ Annual budget (FY2021):\n", - "\n", - "2685 \n", - "US$393,000,000\"\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "for d in docs:\n", " print(\"SOURCE:\", d.attrs[\"source\"])\n", @@ -396,7 +327,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -408,8 +339,8 @@ " G.add_node(\n", " \"init\",\n", " prompt=(\n", - " \"Does the following text mention the National Renewable Energy \"\n", - " \"Laboratory (NREL)? Begin your response with either 'Yes' or \"\n", + " \"Does the following text mention the National Laboratory of the \"\n", + " \"Rockies (NLR)? Begin your response with either 'Yes' or \"\n", " \"'No' and justify your answer.\"\n", " '\\n\\n\"\"\"\\n{text}\\n\"\"\"'\n", " ),\n", @@ -423,7 +354,7 @@ " \"leadership\",\n", " prompt=(\n", " \"Does the text mention who the current director of the National \"\n", - " \"Renewable Energy Laboratory (NREL) is? Begin your response with \"\n", + " \"Laboratory of the Rockies (NLR) is? Begin your response with \"\n", " \"either 'Yes' or 'No' and justify your answer.\"\n", " ),\n", " )\n", @@ -435,7 +366,7 @@ " \"name\",\n", " prompt=(\n", " \"Based on the text, who is the current director of the National \"\n", - " \"Renewable Energy Laboratory (NREL)?\"\n", + " \"Laboratory of the Rockies (NLR)?\"\n", " ),\n", " )\n", " G.add_edge(\"name\", \"final\") # no condition - always go to the end\n", @@ -446,7 +377,7 @@ " \"answer in JSON format (not markdown). Your JSON file must \"\n", " 'include exactly two keys. The keys are \"director\" and '\n", " '\"explanation\". The value of the \"director\" key should '\n", - " \"be a string containing the name of the current director of NREL \"\n", + " \"be a string containing the name of the current director of NLR \"\n", " 'as mentioned in the text. The value of the \"explanation\" '\n", " \"key should be a string containing a short explanation for your \"\n", " \"answer.\"\n", @@ -511,48 +442,9 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "None of the edge conditions from \"leadership\" were satisfied: [{'condition': . at 0x1387f2020>}]\n", - "Ran into an exception when traversing tree. Last message from LLM is printed below. See debug logs for more detail. \n", - "Last message: \n", - "\"\"\"\n", - "No, the text does not mention who the current director of the National Renewable Energy Laboratory (NREL) is. It only provides information on the operating organization, location, establishment year, number of employees, and annual budget for FY2021.\n", - "\"\"\"\n", - "None of the edge conditions from \"leadership\" were satisfied: [{'condition': . at 0x1387f2020>}]\n", - "Traceback (most recent call last):\n", - " File \"/Users/rolson2/GitHub/rolson2/elm/elm/ords/extraction/tree.py\", line 109, in async_run\n", - " out = await self.async_call_node(node0)\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - " File \"/Users/rolson2/GitHub/rolson2/elm/elm/ords/extraction/tree.py\", line 89, in async_call_node\n", - " return self._parse_graph_output(node0, out)\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - " File \"/Users/rolson2/GitHub/rolson2/elm/elm/tree.py\", line 174, in _parse_graph_output\n", - " raise AttributeError(msg)\n", - "AttributeError: None of the edge conditions from \"leadership\" were satisfied: [{'condition': . at 0x1387f2020>}]\n" - ] - }, - { - "data": { - "text/plain": [ - "[{'director': 'Martin Keller',\n", - " 'explanation': 'The text indicates that Martin Keller became the ninth director of the National Renewable Energy Laboratory (NREL) in November 2015 and continues to serve in that capacity.',\n", - " 'source': 'https://en.wikipedia.org/wiki/National_Renewable_Energy_Laboratory',\n", - " 'relevant_text': '## History\\n\\n[edit]\\n\\nMartin Keller became NREL\\'s ninth director in November 2015,[10] and currently\\nserves as both the director of the laboratory and the president of its\\noperating contractor, Alliance for Sustainable Energy, LLC.[11] He succeeded\\nDan Arvizu, who retired in September 2015 after 10 years in those roles.[12]\\n\"Dr. Martin Keller Named Director of National Renewable Energy Laboratory\". _National Renewable Energy Laboratory_. Retrieved June 27, 2017.\\n\\n\"Dr. Martin Keller – Laboratory Director\". Retrieved January 30, 2017.'},\n", - " {'source': 'https://en.wikipedia.org/wiki/United_States_Department_of_Energy_National_Laboratories',\n", - " 'relevant_text': '\"National Renewable Energy Laboratory (NREL)\\n\\nGolden, Colorado, 1977\\n\\nOperating organization:\\n\\nAlliance for Sustainable Energy, LLC (since 2008)[11]\\n\\nNumber of employees/ Annual budget (FY2021):\\n\\n2685 \\nUS$393,000,000\"'}]" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "async with RunningAsyncServices(services):\n", " tasks = [\n", @@ -574,74 +466,9 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
directorexplanationsourcerelevant_text
0Martin KellerThe text indicates that Martin Keller became t...https://en.wikipedia.org/wiki/National_Renewab...## History\\n\\n[edit]\\n\\nMartin Keller became N...
1NaNNaNhttps://en.wikipedia.org/wiki/United_States_De...\"National Renewable Energy Laboratory (NREL)\\n...
\n", - "
" - ], - "text/plain": [ - " director explanation \\\n", - "0 Martin Keller The text indicates that Martin Keller became t... \n", - "1 NaN NaN \n", - "\n", - " source \\\n", - "0 https://en.wikipedia.org/wiki/National_Renewab... \n", - "1 https://en.wikipedia.org/wiki/United_States_De... \n", - "\n", - " relevant_text \n", - "0 ## History\\n\\n[edit]\\n\\nMartin Keller became N... \n", - "1 \"National Renewable Energy Laboratory (NREL)\\n... " - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import pandas as pd\n", "\n", @@ -669,7 +496,7 @@ ], "metadata": { "kernelspec": { - "display_name": "dev", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -683,7 +510,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.8" + "version": "3.12.11" } }, "nbformat": 4, diff --git a/examples/web_information_retrieval/example_website_retrieval_atb.ipynb b/examples/web_information_retrieval/example_website_retrieval_atb.ipynb index cefc2b65..b84f11c4 100644 --- a/examples/web_information_retrieval/example_website_retrieval_atb.ipynb +++ b/examples/web_information_retrieval/example_website_retrieval_atb.ipynb @@ -7,7 +7,7 @@ "# Website information retrieval\n", "\n", "In this demo, we will put together a website crawler to retrieve information from a particular website. \n", - "In particular, we will collection information from NREL's Annual Technology Baseline (ATB) website: https://atb.nrel.gov/.\n", + "In particular, we will collection information from NLR's Annual Technology Baseline (ATB) website: https://atb.nrel.gov/.\n", "The information collecting using this technique can be used in downstream model, such as e.g. a chatbot with a knowledge base attuned to the ATB.\n", "\n", "## Website Crawling\n", @@ -37,7 +37,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -55,20 +55,9 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "init_logger(\"elm\", log_level=\"DEBUG\")" ] @@ -185,7 +174,7 @@ "source": [ "Now we instantiate the crawler using our ``empty_link_scorer`` and \n", "``empty_validator`` functions. We'll also specify ``{\"verify_ssl\": False}`` \n", - "as ``file_loader_kwargs`` to avoid some SSL issues over the NREL VPN and \n", + "as ``file_loader_kwargs`` to avoid some SSL issues over the NLR VPN and \n", "also set ``include_external=False`` (which is actually already the default) \n", "to prevent the crawl from collecting information from external domains.\n", "We'll add our own filter that we implemented above using ``extra_url_filters=[ATBFilter()]``,\n", @@ -228,672 +217,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "DEBUG - 2025-07-02 23:19:14,482 [website_crawl.py:518] : Crawled https://atb.nrel.gov/\n", - "DEBUG - 2025-07-02 23:19:14,487 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:16,047 [website_crawl.py:518] : Crawled https://atb.nrel.gov/archive\n", - "DEBUG - 2025-07-02 23:19:16,051 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:25,061 [website_crawl.py:518] : Crawled https://atb.nrel.gov/contact\n", - "DEBUG - 2025-07-02 23:19:25,070 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:25,321 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/data\n", - "DEBUG - 2025-07-02 23:19:25,326 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:25,530 [website_crawl.py:518] : Crawled https://atb.nrel.gov/search\n", - "DEBUG - 2025-07-02 23:19:25,532 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:25,700 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/data\n", - "DEBUG - 2025-07-02 23:19:25,704 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:25,710 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/about\n", - "DEBUG - 2025-07-02 23:19:25,729 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:25,903 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/technologies\n", - "DEBUG - 2025-07-02 23:19:25,915 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:26,120 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index\n", - "DEBUG - 2025-07-02 23:19:26,130 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:26,452 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/about\n", - "DEBUG - 2025-07-02 23:19:26,467 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:28,236 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/index\n", - "DEBUG - 2025-07-02 23:19:28,240 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:30,029 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies\n", - "DEBUG - 2025-07-02 23:19:30,032 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:31,470 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/acronyms\n", - "DEBUG - 2025-07-02 23:19:31,472 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:40,295 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/biopower_(eia)\n", - "DEBUG - 2025-07-02 23:19:40,308 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:40,479 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/changes_in_2024\n", - "DEBUG - 2025-07-02 23:19:40,523 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:40,659 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/approach_&_methodology\n", - "DEBUG - 2025-07-02 23:19:40,668 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:40,928 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/concentrating_solar_power\n", - "DEBUG - 2025-07-02 23:19:40,939 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:41,239 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/commercial_pv\n", - "DEBUG - 2025-07-02 23:19:41,272 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:41,460 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/commercial_battery_storage\n", - "DEBUG - 2025-07-02 23:19:41,474 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:43,163 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/data/disclaimer\n", - "DEBUG - 2025-07-02 23:19:43,168 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:51,548 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/data/technical_limitations\n", - "DEBUG - 2025-07-02 23:19:51,551 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:52,066 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/distributed_wind\n", - "DEBUG - 2025-07-02 23:19:52,078 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:54,624 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/hydropower\n", - "DEBUG - 2025-07-02 23:19:54,643 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:54,918 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/fossil_energy_technologies\n", - "DEBUG - 2025-07-02 23:19:54,943 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:55,677 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/geothermal\n", - "DEBUG - 2025-07-02 23:19:55,699 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:55,734 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/financial_cases_&_methods\n", - "DEBUG - 2025-07-02 23:19:55,754 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:55,990 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/data/errata\n", - "DEBUG - 2025-07-02 23:19:55,993 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:55,995 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/equations_&_variables\n", - "DEBUG - 2025-07-02 23:19:55,999 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:56,103 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions\n", - "DEBUG - 2025-07-02 23:19:56,110 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:19:57,693 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/approach_&_methodology\n", - "DEBUG - 2025-07-02 23:19:57,700 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:07,412 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/technical_limitations\n", - "DEBUG - 2025-07-02 23:20:07,415 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:08,035 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/references\n", - "DEBUG - 2025-07-02 23:20:08,050 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:09,266 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/data\n", - "DEBUG - 2025-07-02 23:20:09,278 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:09,299 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/disclaimer\n", - "DEBUG - 2025-07-02 23:20:09,319 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:09,753 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/land-based_wind\n", - "DEBUG - 2025-07-02 23:20:09,774 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:09,805 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions\n", - "DEBUG - 2025-07-02 23:20:09,820 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:10,153 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/financial_cases_&_methods\n", - "DEBUG - 2025-07-02 23:20:10,165 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:10,319 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/land-based_wind\n", - "DEBUG - 2025-07-02 23:20:10,325 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:10,503 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/nuclear\n", - "DEBUG - 2025-07-02 23:20:10,515 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:16,244 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/offshore_wind\n", - "DEBUG - 2025-07-02 23:20:16,281 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:19,979 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/acronyms\n", - "DEBUG - 2025-07-02 23:20:19,982 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:22,233 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/references\n", - "DEBUG - 2025-07-02 23:20:22,251 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:22,414 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/technologies/index\n", - "DEBUG - 2025-07-02 23:20:22,421 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:22,755 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/utility-scale_battery_storage\n", - "DEBUG - 2025-07-02 23:20:22,779 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:23,131 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/pumped_storage_hydropower\n", - "DEBUG - 2025-07-02 23:20:23,137 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:23,475 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/utility-scale_pv-plus-battery\n", - "DEBUG - 2025-07-02 23:20:23,488 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:23,799 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/residential_battery_storage\n", - "DEBUG - 2025-07-02 23:20:23,809 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:24,159 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/utility-scale_pv\n", - "DEBUG - 2025-07-02 23:20:24,175 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:24,210 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/residential_pv\n", - "DEBUG - 2025-07-02 23:20:24,220 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:25,726 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/assumptions\n", - "DEBUG - 2025-07-02 23:20:25,730 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:30,889 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/data/disclaimer\n", - "DEBUG - 2025-07-02 23:20:30,900 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:32,151 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/data/references\n", - "DEBUG - 2025-07-02 23:20:32,155 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:32,376 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/data/technical_limitations\n", - "DEBUG - 2025-07-02 23:20:32,384 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:32,519 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/aviation\n", - "DEBUG - 2025-07-02 23:20:32,527 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:32,732 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/fuels\n", - "DEBUG - 2025-07-02 23:20:32,737 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:32,767 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions\n", - "DEBUG - 2025-07-02 23:20:32,821 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:32,986 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/changes_in_2024\n", - "DEBUG - 2025-07-02 23:20:32,995 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:34,441 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/index/about\n", - "DEBUG - 2025-07-02 23:20:34,445 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:36,887 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/index/technologies\n", - "DEBUG - 2025-07-02 23:20:36,890 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:37,062 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/index/data\n", - "DEBUG - 2025-07-02 23:20:37,070 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:38,507 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/light-duty_vehicles\n", - "DEBUG - 2025-07-02 23:20:38,510 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:39,996 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/medium-_and_heavy-duty_vehicles\n", - "DEBUG - 2025-07-02 23:20:40,000 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:40,274 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/references\n", - "DEBUG - 2025-07-02 23:20:40,288 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:43,727 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Aviation_Fuels\n", - "DEBUG - 2025-07-02 23:20:43,763 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:47,955 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Hydrogen\n", - "DEBUG - 2025-07-02 23:20:47,959 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:48,302 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Natural_Gas\n", - "DEBUG - 2025-07-02 23:20:48,317 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:48,621 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Marine_Fuels\n", - "DEBUG - 2025-07-02 23:20:48,641 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:48,672 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/data\n", - "DEBUG - 2025-07-02 23:20:48,699 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:48,962 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/comparison_of_ld_vehicles\n", - "DEBUG - 2025-07-02 23:20:48,987 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:49,063 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Electricity\n", - "DEBUG - 2025-07-02 23:20:49,070 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:49,199 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Petro-_and_Bio-Based_Diesel_Fuel\n", - "DEBUG - 2025-07-02 23:20:49,208 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:51,163 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/fuels\n", - "DEBUG - 2025-07-02 23:20:51,171 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:52,713 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/gasoline_and_ethanol\n", - "DEBUG - 2025-07-02 23:20:52,728 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:53,582 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/references\n", - "DEBUG - 2025-07-02 23:20:53,585 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:20:55,433 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/approach_&_methodology/data\n", - "DEBUG - 2025-07-02 23:20:55,438 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:00,570 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/biopower_(eia)/references\n", - "DEBUG - 2025-07-02 23:21:00,574 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:00,743 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/approach_&_methodology/references\n", - "DEBUG - 2025-07-02 23:21:00,753 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:00,998 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/biopower_(eia)/financial_cases_&_methods\n", - "DEBUG - 2025-07-02 23:21:01,017 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:01,307 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/changes_in_2024/equations_&_variables\n", - "DEBUG - 2025-07-02 23:21:01,310 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:07,698 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/changes_in_2024/financial_cases_&_methods\n", - "DEBUG - 2025-07-02 23:21:07,735 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:13,182 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/commercial_pv/pv-ac-dc\n", - "DEBUG - 2025-07-02 23:21:13,211 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:14,405 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/commercial_battery_storage/references\n", - "DEBUG - 2025-07-02 23:21:14,411 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:15,810 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/changes_in_2024/references\n", - "DEBUG - 2025-07-02 23:21:15,841 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:16,624 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/commercial_pv/references\n", - "DEBUG - 2025-07-02 23:21:16,632 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:16,824 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/commercial_battery_storage/data\n", - "DEBUG - 2025-07-02 23:21:16,831 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:17,240 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/commercial_battery_storage/utility-scale_battery_storage\n", - "DEBUG - 2025-07-02 23:21:17,268 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:17,577 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/commercial_battery_storage/residential_battery_storage\n", - "DEBUG - 2025-07-02 23:21:17,585 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:17,592 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/commercial_pv/approach_&_methodology\n", - "DEBUG - 2025-07-02 23:21:17,602 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:17,753 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/commercial_pv/residential_pv\n", - "DEBUG - 2025-07-02 23:21:17,764 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:22,882 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/commercial_pv/utility-scale_pv\n", - "DEBUG - 2025-07-02 23:21:22,909 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:26,279 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/concentrating_solar_power/2023/data\n", - "DEBUG - 2025-07-02 23:21:26,282 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:26,747 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/Distributed_Wind\n", - "DEBUG - 2025-07-02 23:21:26,753 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:27,176 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/concentrating_solar_power/references\n", - "DEBUG - 2025-07-02 23:21:27,197 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:27,368 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/Land-Based_Wind\n", - "DEBUG - 2025-07-02 23:21:27,383 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:27,660 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/Offshore_Wind\n", - "DEBUG - 2025-07-02 23:21:27,675 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:27,676 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/concentrating_solar_power/definitions\n", - "DEBUG - 2025-07-02 23:21:27,689 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:32,975 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/biopower_(eia)\n", - "DEBUG - 2025-07-02 23:21:32,990 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:41,678 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/financial_cases_&_methods\n", - "DEBUG - 2025-07-02 23:21:41,693 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:43,971 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/concentrating_solar_power\n", - "DEBUG - 2025-07-02 23:21:43,991 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:44,305 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/data\n", - "DEBUG - 2025-07-02 23:21:44,320 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:44,322 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/commercial_battery_storage\n", - "DEBUG - 2025-07-02 23:21:44,336 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:44,860 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/disclaimer\n", - "DEBUG - 2025-07-02 23:21:44,876 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:45,303 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/commercial_pv\n", - "DEBUG - 2025-07-02 23:21:45,314 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:45,315 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/equations_&_variables\n", - "DEBUG - 2025-07-02 23:21:45,319 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:45,531 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/geothermal\n", - "DEBUG - 2025-07-02 23:21:45,541 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:45,542 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/fossil_energy_technologies\n", - "DEBUG - 2025-07-02 23:21:45,551 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:51,938 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/hydropower\n", - "DEBUG - 2025-07-02 23:21:51,961 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:21:58,740 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/references\n", - "DEBUG - 2025-07-02 23:21:58,745 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:01,965 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/utility-scale_pv-plus-battery\n", - "DEBUG - 2025-07-02 23:22:01,974 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:02,285 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/nuclear\n", - "DEBUG - 2025-07-02 23:22:02,306 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:02,549 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/technical_limitations\n", - "DEBUG - 2025-07-02 23:22:02,555 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:02,842 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/pumped_storage_hydropower\n", - "DEBUG - 2025-07-02 23:22:02,858 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:02,987 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/utility-scale_battery_storage\n", - "DEBUG - 2025-07-02 23:22:03,019 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:03,197 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/utility-scale_pv\n", - "DEBUG - 2025-07-02 23:22:03,209 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:03,385 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/residential_pv\n", - "DEBUG - 2025-07-02 23:22:03,397 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:03,535 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/definitions/residential_battery_storage\n", - "DEBUG - 2025-07-02 23:22:03,544 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:05,826 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/distributed_wind/references\n", - "DEBUG - 2025-07-02 23:22:05,830 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:07,818 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/equations_&_variables/references\n", - "DEBUG - 2025-07-02 23:22:07,830 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:08,000 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/equations_&_variables/data\n", - "DEBUG - 2025-07-02 23:22:08,010 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:09,665 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/financial_cases_&_methods/references\n", - "DEBUG - 2025-07-02 23:22:09,669 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:11,016 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/fossil_energy_technologies/references\n", - "DEBUG - 2025-07-02 23:22:11,021 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:13,043 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/geothermal/references\n", - "DEBUG - 2025-07-02 23:22:13,048 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:15,831 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/hydropower/references\n", - "DEBUG - 2025-07-02 23:22:15,834 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:16,013 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/hydropower/data\n", - "DEBUG - 2025-07-02 23:22:16,022 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:17,778 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/approach_&_methodology/data\n", - "DEBUG - 2025-07-02 23:22:17,783 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:25,460 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/commercial_battery_storage\n", - "DEBUG - 2025-07-02 23:22:25,463 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:25,464 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/data/disclaimer\n", - "DEBUG - 2025-07-02 23:22:25,466 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:25,740 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/Land-Based_Wind\n", - "DEBUG - 2025-07-02 23:22:25,744 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:26,072 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/Distributed_Wind\n", - "DEBUG - 2025-07-02 23:22:26,076 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:26,217 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/data/errata\n", - "DEBUG - 2025-07-02 23:22:26,221 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:26,223 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/biopower_(eia)\n", - "DEBUG - 2025-07-02 23:22:26,227 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:26,229 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/Offshore_Wind\n", - "DEBUG - 2025-07-02 23:22:26,233 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:26,336 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/data/technical_limitations\n", - "DEBUG - 2025-07-02 23:22:26,341 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:26,343 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/approach_&_methodology/references\n", - "DEBUG - 2025-07-02 23:22:26,349 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:27,467 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/commercial_pv\n", - "DEBUG - 2025-07-02 23:22:27,473 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:33,483 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/concentrating_solar_power\n", - "DEBUG - 2025-07-02 23:22:33,486 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:33,935 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/financial_cases_&_methods\n", - "DEBUG - 2025-07-02 23:22:33,939 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:34,223 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/data\n", - "DEBUG - 2025-07-02 23:22:34,228 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:34,230 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/equations_&_variables\n", - "DEBUG - 2025-07-02 23:22:34,234 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:34,400 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/geothermal\n", - "DEBUG - 2025-07-02 23:22:34,403 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:34,404 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/disclaimer\n", - "DEBUG - 2025-07-02 23:22:34,406 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:34,408 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/nuclear\n", - "DEBUG - 2025-07-02 23:22:34,412 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:34,413 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/hydropower\n", - "DEBUG - 2025-07-02 23:22:34,416 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:34,518 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/fossil_energy_technologies\n", - "DEBUG - 2025-07-02 23:22:34,521 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:35,718 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/pumped_storage_hydropower\n", - "DEBUG - 2025-07-02 23:22:35,722 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:42,892 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/land-based_wind/references\n", - "DEBUG - 2025-07-02 23:22:42,895 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:43,440 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/utility-scale_pv\n", - "DEBUG - 2025-07-02 23:22:43,444 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:43,446 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/technical_limitations\n", - "DEBUG - 2025-07-02 23:22:43,450 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:43,597 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/utility-scale_pv-plus-battery\n", - "DEBUG - 2025-07-02 23:22:43,601 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:43,752 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/financial_cases_&_methods/references\n", - "DEBUG - 2025-07-02 23:22:43,757 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:43,758 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/utility-scale_battery_storage\n", - "DEBUG - 2025-07-02 23:22:43,763 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:43,766 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/references\n", - "DEBUG - 2025-07-02 23:22:43,773 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:43,774 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/residential_pv\n", - "DEBUG - 2025-07-02 23:22:43,779 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:43,882 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index/definitions/residential_battery_storage\n", - "DEBUG - 2025-07-02 23:22:43,886 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:45,599 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/land-based_wind/references\n", - "DEBUG - 2025-07-02 23:22:45,602 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:47,297 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/nuclear/references\n", - "DEBUG - 2025-07-02 23:22:47,300 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:50,824 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/offshore_wind/land-based_wind\n", - "DEBUG - 2025-07-02 23:22:50,842 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:51,669 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/offshore_wind/references\n", - "DEBUG - 2025-07-02 23:22:51,673 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:53,485 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/pumped_storage_hydropower/references\n", - "DEBUG - 2025-07-02 23:22:53,488 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:56,893 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/residential_battery_storage/commercial_battery_storage\n", - "DEBUG - 2025-07-02 23:22:56,900 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:58,373 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/residential_battery_storage/utility-scale_battery_storage\n", - "DEBUG - 2025-07-02 23:22:58,385 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:22:58,864 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/residential_battery_storage/data\n", - "DEBUG - 2025-07-02 23:22:58,872 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:00,473 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/residential_battery_storage/references\n", - "DEBUG - 2025-07-02 23:23:00,476 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:02,318 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/residential_pv/approach_&_methodology\n", - "DEBUG - 2025-07-02 23:23:02,328 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:11,743 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/residential_pv/commercial_pv\n", - "DEBUG - 2025-07-02 23:23:11,761 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:14,302 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/technologies/index/land-based_wind\n", - "DEBUG - 2025-07-02 23:23:14,310 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:15,335 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/residential_pv/references\n", - "DEBUG - 2025-07-02 23:23:15,367 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:15,756 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/technologies/index/disclaimer\n", - "DEBUG - 2025-07-02 23:23:15,765 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:15,966 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/technologies/index/data\n", - "DEBUG - 2025-07-02 23:23:15,970 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:15,973 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/technologies/index/definitions\n", - "DEBUG - 2025-07-02 23:23:15,982 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:15,984 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/residential_pv/pv-ac-dc\n", - "DEBUG - 2025-07-02 23:23:15,992 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:15,993 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/technologies/index/approach_&_methodology\n", - "DEBUG - 2025-07-02 23:23:15,998 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:16,111 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/technologies/index/financial_cases_&_methods\n", - "DEBUG - 2025-07-02 23:23:16,116 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:16,393 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/residential_pv/utility-scale_pv\n", - "DEBUG - 2025-07-02 23:23:16,412 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:17,863 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/technologies/index/references\n", - "DEBUG - 2025-07-02 23:23:17,866 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:19,365 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/technologies/index/technical_limitations\n", - "DEBUG - 2025-07-02 23:23:19,368 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:22,740 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/utility-scale_battery_storage/commercial_battery_storage\n", - "DEBUG - 2025-07-02 23:23:22,747 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:23,777 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/utility-scale_battery_storage/data\n", - "DEBUG - 2025-07-02 23:23:23,781 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:23,782 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/utility-scale_battery_storage/references\n", - "DEBUG - 2025-07-02 23:23:23,786 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:26,374 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/utility-scale_battery_storage/residential_battery_storage\n", - "DEBUG - 2025-07-02 23:23:26,378 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:29,656 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/utility-scale_pv-plus-battery/financial_cases_&_methods\n", - "DEBUG - 2025-07-02 23:23:29,676 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:34,294 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/utility-scale_pv-plus-battery/references\n", - "DEBUG - 2025-07-02 23:23:34,310 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:35,370 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/utility-scale_pv/approach_&_methodology\n", - "DEBUG - 2025-07-02 23:23:35,382 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:35,536 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/assumptions/battery_electric_vehicle_assumptions\n", - "DEBUG - 2025-07-02 23:23:35,541 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:35,542 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/assumptions/fuel_cell_electric_vehicle_assumptions\n", - "DEBUG - 2025-07-02 23:23:35,545 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:35,713 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/utility-scale_pv/references\n", - "DEBUG - 2025-07-02 23:23:35,719 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:35,978 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/utility-scale_pv/residential_pv\n", - "DEBUG - 2025-07-02 23:23:35,987 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:36,122 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/utility-scale_pv/commercial_pv\n", - "DEBUG - 2025-07-02 23:23:36,134 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:37,561 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/assumptions/fuel_economy_assumptions\n", - "DEBUG - 2025-07-02 23:23:37,564 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:44,925 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/assumptions/vehicle_range_assumptions\n", - "DEBUG - 2025-07-02 23:23:44,928 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:44,929 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/aviation/references\n", - "DEBUG - 2025-07-02 23:23:44,932 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:45,229 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/assumptions/sustainable_aviation_fuel_assumptions\n", - "DEBUG - 2025-07-02 23:23:45,240 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:45,477 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/assumptions/vehicle_size_class_assumptions\n", - "DEBUG - 2025-07-02 23:23:45,481 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:45,483 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/assumptions/levelized_cost_of_driving_assumptions\n", - "DEBUG - 2025-07-02 23:23:45,489 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:45,493 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/assumptions/lowest_co2e_emissions_fuel_assumptions\n", - "DEBUG - 2025-07-02 23:23:45,498 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:45,639 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/aviation_fuels\n", - "DEBUG - 2025-07-02 23:23:45,664 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:45,766 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/assumptions/production_volume_assumptions\n", - "DEBUG - 2025-07-02 23:23:45,774 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:47,729 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/battery_electric\n", - "DEBUG - 2025-07-02 23:23:47,746 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:49,827 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/battery_electric_mdhd\n", - "DEBUG - 2025-07-02 23:23:49,846 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:51,560 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/blendstocks\n", - "DEBUG - 2025-07-02 23:23:51,576 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:54,002 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/changes_in_2024/levelized_cost_of_driving_assumptions\n", - "DEBUG - 2025-07-02 23:23:54,004 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:23:55,776 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/changes_in_2024/references\n", - "DEBUG - 2025-07-02 23:23:55,779 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:03,150 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions/battery_electric_vehicle_assumptions\n", - "DEBUG - 2025-07-02 23:24:03,164 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:03,513 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/comparison_of_mdhd_vehicles\n", - "DEBUG - 2025-07-02 23:24:03,568 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:03,935 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/comparison_of_ld_vehicles\n", - "DEBUG - 2025-07-02 23:24:03,979 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:04,218 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions/diesel\n", - "DEBUG - 2025-07-02 23:24:04,237 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:04,238 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions/electricity\n", - "DEBUG - 2025-07-02 23:24:04,247 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:04,357 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions/bob\n", - "DEBUG - 2025-07-02 23:24:04,368 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:06,875 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions/ethanol\n", - "DEBUG - 2025-07-02 23:24:06,892 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:16,393 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions/gasoline_hybrid\n", - "DEBUG - 2025-07-02 23:24:16,414 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:16,416 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions/lowest_co2e_emissions_fuel_assumptions\n", - "DEBUG - 2025-07-02 23:24:16,421 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:16,698 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions/gasoline\n", - "DEBUG - 2025-07-02 23:24:16,716 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:16,901 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions/fuels\n", - "DEBUG - 2025-07-02 23:24:16,917 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:16,919 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions/hydrogen\n", - "DEBUG - 2025-07-02 23:24:16,924 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:17,176 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions/levelized_cost_of_driving_assumptions\n", - "DEBUG - 2025-07-02 23:24:17,180 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:17,285 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions/fuel_cell_electric_vehicle_assumptions\n", - "DEBUG - 2025-07-02 23:24:17,287 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:17,288 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions/fuel_economy_assumptions\n", - "DEBUG - 2025-07-02 23:24:17,297 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:17,399 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions/gasoline_and_ethanol\n", - "DEBUG - 2025-07-02 23:24:17,409 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:21,036 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions/natural_gas\n", - "DEBUG - 2025-07-02 23:24:21,087 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:28,299 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions/vehicle_range_assumptions\n", - "DEBUG - 2025-07-02 23:24:28,315 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:28,636 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions/natural_gas_fuel\n", - "DEBUG - 2025-07-02 23:24:28,654 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:28,914 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions/references\n", - "DEBUG - 2025-07-02 23:24:28,933 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:29,176 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions/vehicle_size_class_assumptions\n", - "DEBUG - 2025-07-02 23:24:29,191 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:29,193 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/diesel\n", - "DEBUG - 2025-07-02 23:24:29,231 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:29,424 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions/production_volume_assumptions\n", - "DEBUG - 2025-07-02 23:24:29,435 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:29,443 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/definitions/sustainable_aviation_fuel_assumptions\n", - "DEBUG - 2025-07-02 23:24:29,452 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:33,241 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/diesel_hybrid_mdhd\n", - "DEBUG - 2025-07-02 23:24:33,277 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:35,934 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/fuel_cell\n", - "DEBUG - 2025-07-02 23:24:35,955 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:36,164 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/fuel_cell_mdhd\n", - "DEBUG - 2025-07-02 23:24:36,181 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:36,387 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/electricity\n", - "DEBUG - 2025-07-02 23:24:36,397 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:36,528 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/diesel_mdhd\n", - "DEBUG - 2025-07-02 23:24:36,569 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:39,792 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/fuels/Diesel_Bio-Based_Blendstock\n", - "DEBUG - 2025-07-02 23:24:39,807 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:47,645 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/fuels/aviation_fuels\n", - "DEBUG - 2025-07-02 23:24:47,659 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:47,927 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/fuels/Gasoline_and_Ethanol\n", - "DEBUG - 2025-07-02 23:24:47,954 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:47,956 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/fuels/Marine_Fuels\n", - "DEBUG - 2025-07-02 23:24:47,977 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:48,689 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/fuels/electricity\n", - "DEBUG - 2025-07-02 23:24:48,696 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:48,697 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/fuels/on-road_fuels\n", - "DEBUG - 2025-07-02 23:24:48,707 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:48,910 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/fuels/Petro-_and_Bio-Based_Diesel_Fuel\n", - "DEBUG - 2025-07-02 23:24:48,918 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:48,919 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/fuels/hydrogen\n", - "DEBUG - 2025-07-02 23:24:48,924 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:48,925 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/fuels/blendstocks\n", - "DEBUG - 2025-07-02 23:24:48,930 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:48,931 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/fuels/Ethanol\n", - "DEBUG - 2025-07-02 23:24:48,948 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:50,368 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/fuels/references\n", - "DEBUG - 2025-07-02 23:24:50,371 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:55,847 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/gasoline_and_ethanol\n", - "DEBUG - 2025-07-02 23:24:55,863 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:56,019 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/gasoline\n", - "DEBUG - 2025-07-02 23:24:56,057 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:24:56,160 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/gasoline_hybrid\n", - "DEBUG - 2025-07-02 23:24:56,184 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:01,499 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/hydrogen\n", - "DEBUG - 2025-07-02 23:25:01,503 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:02,198 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/index/data/technical_limitations\n", - "DEBUG - 2025-07-02 23:25:02,203 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:02,489 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/index/technologies/Aviation_Fuels\n", - "DEBUG - 2025-07-02 23:25:02,501 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:02,614 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/index/data/disclaimer\n", - "DEBUG - 2025-07-02 23:25:02,618 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:02,619 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/index/data/references\n", - "DEBUG - 2025-07-02 23:25:02,622 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:02,623 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/index/technologies/Electricity\n", - "DEBUG - 2025-07-02 23:25:02,626 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:03,907 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/index/technologies/Hydrogen\n", - "DEBUG - 2025-07-02 23:25:03,910 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:10,777 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/index/technologies/fuels\n", - "DEBUG - 2025-07-02 23:25:10,780 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:11,007 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/index/technologies/Natural_Gas\n", - "DEBUG - 2025-07-02 23:25:11,011 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:11,014 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/index/technologies/Petro-_and_Bio-Based_Diesel_Fuel\n", - "DEBUG - 2025-07-02 23:25:11,025 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:11,027 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/index/technologies/gasoline_and_ethanol\n", - "DEBUG - 2025-07-02 23:25:11,031 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:11,034 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/index/technologies/Marine_Fuels\n", - "DEBUG - 2025-07-02 23:25:11,045 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:11,296 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/index/technologies/comparison_of_ld_vehicles\n", - "DEBUG - 2025-07-02 23:25:11,310 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:11,312 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/index/technologies/data\n", - "DEBUG - 2025-07-02 23:25:11,332 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:11,434 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/index/technologies/references\n", - "DEBUG - 2025-07-02 23:25:11,438 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:13,782 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/light-duty_vehicles/comparison_of_ld_vehicles\n", - "DEBUG - 2025-07-02 23:25:13,814 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:19,209 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/light-duty_vehicles/medium-_and_heavy-duty_vehicles\n", - "DEBUG - 2025-07-02 23:25:19,216 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:19,481 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/literature_context_for_mdhd_vehicles\n", - "DEBUG - 2025-07-02 23:25:19,491 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:19,670 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/marine_fuels\n", - "DEBUG - 2025-07-02 23:25:19,677 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:19,678 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/literature_context_for_ld_vehicles\n", - "DEBUG - 2025-07-02 23:25:19,684 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:21,982 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/medium-_and_heavy-duty_vehicles/comparison_of_mdhd_vehicles\n", - "DEBUG - 2025-07-02 23:25:21,998 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:23,437 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/medium-_and_heavy-duty_vehicles/light-duty_vehicles\n", - "DEBUG - 2025-07-02 23:25:23,440 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:27,531 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/on-road_fuels\n", - "DEBUG - 2025-07-02 23:25:27,549 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:27,768 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/natural_gas_fuel\n", - "DEBUG - 2025-07-02 23:25:27,779 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:28,066 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/natural_gas\n", - "DEBUG - 2025-07-02 23:25:28,077 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:30,540 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/petro-_and_bio-based_diesel_fuel\n", - "DEBUG - 2025-07-02 23:25:30,556 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:32,717 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/plug-in_hybrid\n", - "DEBUG - 2025-07-02 23:25:32,738 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:34,419 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/plug-in_hybrid_mdhd\n", - "DEBUG - 2025-07-02 23:25:34,434 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:35,670 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Aviation_Fuels/data\n", - "DEBUG - 2025-07-02 23:25:35,672 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:43,677 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Electricity/natural_gas_fuel\n", - "DEBUG - 2025-07-02 23:25:43,680 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:43,681 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Aviation_Fuels/references\n", - "DEBUG - 2025-07-02 23:25:43,685 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:43,927 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Natural_Gas/Fuels\n", - "DEBUG - 2025-07-02 23:25:43,930 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:43,931 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Electricity/references\n", - "DEBUG - 2025-07-02 23:25:43,936 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:44,120 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Electricity/data\n", - "DEBUG - 2025-07-02 23:25:44,125 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:44,134 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Marine_Fuels/references\n", - "DEBUG - 2025-07-02 23:25:44,139 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:44,268 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Electricity/hydrogen\n", - "DEBUG - 2025-07-02 23:25:44,273 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:44,274 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Hydrogen/data\n", - "DEBUG - 2025-07-02 23:25:44,285 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:44,387 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Hydrogen/references\n", - "DEBUG - 2025-07-02 23:25:44,391 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:45,738 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Natural_Gas/Natural_Gas_Fuel\n", - "DEBUG - 2025-07-02 23:25:45,740 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:52,550 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Petro-_and_Bio-Based_Diesel_Fuel/diesel_bio-based_blendstock\n", - "DEBUG - 2025-07-02 23:25:52,553 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:53,090 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Natural_Gas/gasoline_and_ethanol\n", - "DEBUG - 2025-07-02 23:25:53,096 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:53,318 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Natural_Gas/natural_gas_fuel\n", - "DEBUG - 2025-07-02 23:25:53,323 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:53,325 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Petro-_and_Bio-Based_Diesel_Fuel/data\n", - "DEBUG - 2025-07-02 23:25:53,329 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:53,551 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Natural_Gas/comparison_of_ld_vehicles\n", - "DEBUG - 2025-07-02 23:25:53,556 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:53,668 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Petro-_and_Bio-Based_Diesel_Fuel/Blendstocks\n", - "DEBUG - 2025-07-02 23:25:53,671 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:53,672 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Petro-_and_Bio-Based_Diesel_Fuel/references\n", - "DEBUG - 2025-07-02 23:25:53,677 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:53,678 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Natural_Gas/data\n", - "DEBUG - 2025-07-02 23:25:53,682 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:53,683 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/Natural_Gas/references\n", - "DEBUG - 2025-07-02 23:25:53,689 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:25:55,101 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/comparison_of_ld_vehicles/Blendstocks\n", - "DEBUG - 2025-07-02 23:25:55,106 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:00,584 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/data/disclaimer\n", - "DEBUG - 2025-07-02 23:26:00,588 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:00,763 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/comparison_of_ld_vehicles/fuels\n", - "DEBUG - 2025-07-02 23:26:00,766 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:01,056 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/comparison_of_ld_vehicles/electricity\n", - "DEBUG - 2025-07-02 23:26:01,059 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:01,185 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/comparison_of_ld_vehicles/references\n", - "DEBUG - 2025-07-02 23:26:01,189 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:01,366 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/comparison_of_ld_vehicles/data\n", - "DEBUG - 2025-07-02 23:26:01,369 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:01,370 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/comparison_of_ld_vehicles/gasoline_and_ethanol\n", - "DEBUG - 2025-07-02 23:26:01,373 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:01,374 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/comparison_of_ld_vehicles/diesel\n", - "DEBUG - 2025-07-02 23:26:01,377 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:01,479 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/comparison_of_ld_vehicles/Fuels\n", - "DEBUG - 2025-07-02 23:26:01,482 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:01,584 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/comparison_of_ld_vehicles/technologies\n", - "DEBUG - 2025-07-02 23:26:01,587 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:03,207 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/data/references\n", - "DEBUG - 2025-07-02 23:26:03,211 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:09,004 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/fuels/Diesel_Bio-Based_Blendstock\n", - "DEBUG - 2025-07-02 23:26:09,007 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:09,009 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/fuels/electricity\n", - "DEBUG - 2025-07-02 23:26:09,012 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:09,217 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/fuels/Ethanol\n", - "DEBUG - 2025-07-02 23:26:09,230 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:09,232 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/data/technical_limitations\n", - "DEBUG - 2025-07-02 23:26:09,236 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:09,240 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/fuels/blendstocks\n", - "DEBUG - 2025-07-02 23:26:09,254 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:09,256 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/fuels/Marine_Fuels\n", - "DEBUG - 2025-07-02 23:26:09,260 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:09,373 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/fuels/Gasoline_and_Ethanol\n", - "DEBUG - 2025-07-02 23:26:09,378 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:09,380 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/fuels/Petro-_and_Bio-Based_Diesel_Fuel\n", - "DEBUG - 2025-07-02 23:26:09,390 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:09,492 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/fuels/aviation_fuels\n", - "DEBUG - 2025-07-02 23:26:09,497 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:11,058 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/fuels/hydrogen\n", - "DEBUG - 2025-07-02 23:26:11,061 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:22,270 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/gasoline_and_ethanol/ethanol\n", - "DEBUG - 2025-07-02 23:26:22,273 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:22,541 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/gasoline_and_ethanol/BOB\n", - "DEBUG - 2025-07-02 23:26:22,546 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:22,548 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/gasoline_and_ethanol/Blendstocks\n", - "DEBUG - 2025-07-02 23:26:22,552 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:22,827 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/gasoline_and_ethanol/bob\n", - "DEBUG - 2025-07-02 23:26:22,839 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:22,840 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/fuels/references\n", - "DEBUG - 2025-07-02 23:26:22,845 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:23,023 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/gasoline_and_ethanol/data\n", - "DEBUG - 2025-07-02 23:26:23,028 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:23,030 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/gasoline_and_ethanol/Ethanol\n", - "DEBUG - 2025-07-02 23:26:23,040 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:23,041 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/fuels/on-road_fuels\n", - "DEBUG - 2025-07-02 23:26:23,047 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:23,149 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/gasoline_and_ethanol/definitions\n", - "DEBUG - 2025-07-02 23:26:23,152 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-02 23:26:25,059 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/technologies/gasoline_and_ethanol/references\n", - "DEBUG - 2025-07-02 23:26:25,062 [website_crawl.py:533] : Document passed validation check\n", - "INFO - 2025-07-02 23:26:25,437 [website_crawl.py:540] : Crawled 324 pages\n", - "INFO - 2025-07-02 23:26:25,438 [website_crawl.py:541] : Found 324 potential documents\n", - "DEBUG - 2025-07-02 23:26:25,440 [website_crawl.py:542] : Average score: 1.00\n", - "DEBUG - 2025-07-02 23:26:25,442 [website_crawl.py:549] : Pages crawled by depth:\n", - "DEBUG - 2025-07-02 23:26:25,444 [website_crawl.py:551] : Depth 0: 1 pages\n", - "DEBUG - 2025-07-02 23:26:25,445 [website_crawl.py:551] : Depth 1: 11 pages\n", - "DEBUG - 2025-07-02 23:26:25,446 [website_crawl.py:551] : Depth 2: 62 pages\n", - "DEBUG - 2025-07-02 23:26:25,447 [website_crawl.py:551] : Depth 3: 250 pages\n" - ] - } - ], + "outputs": [], "source": [ "atb_info = await crawler.run(\"https://atb.nrel.gov/\",\n", " termination_callback=empty_terminator)" @@ -908,56 +234,18 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "324" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "len(atb_info)" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "HTMLDocument with 1 pages\n", - "Attrs:\n", - " source:\thttps://atb.nrel.gov/archive\n", - " website_link_relevance_score:\t1\n", - "HTMLDocument with 1 pages\n", - "Attrs:\n", - " source:\thttps://atb.nrel.gov/contact\n", - " website_link_relevance_score:\t1\n", - "HTMLDocument with 1 pages\n", - "Attrs:\n", - " source:\thttps://atb.nrel.gov/transportation/2024/data\n", - " website_link_relevance_score:\t1\n", - "HTMLDocument with 1 pages\n", - "Attrs:\n", - " source:\thttps://atb.nrel.gov/search\n", - " website_link_relevance_score:\t1\n", - "HTMLDocument with 1 pages\n", - "Attrs:\n", - " source:\thttps://atb.nrel.gov/electricity/2024/data\n", - " website_link_relevance_score:\t1\n" - ] - } - ], + "outputs": [], "source": [ "for doc in atb_info[:5]:\n", " print(doc)" @@ -965,20 +253,9 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "https://atb.nrel.gov/electricity/2024/distributed_wind\n", - "[Skip to main content](https://atb.nrel.gov/electricity/2024/distributed_wind#content) Contribute to enhancing the Electricity ATB! Share your feedback in this 5-minute [ questionnaire](https://forms.office.com/g/j9Ua4Hsayx), or signup for general [ email updates regarding the ATB](https://atb.nrel.gov/register). [ Sign up for general email updates regarding the ATB ](https://atb.nrel.gov/register) The 2024 Electricity ATB is live! Join the webinar to learn what's new. [ Register to attend](https://nrel.zoomgov.com/webinar/register/WN_mDY0z3ahTyyG_Y86xnZIaA) or [ sign up for general email updates](https://atb.nrel.gov/register). [ ](http://www.nrel.gov \"Visit the National Renewable Energy Lab Homepage\") [Annual Technology Baseline](https://atb.nrel.gov/) Menu * [ Electricity](https://atb.nrel.gov/electricity/2024/distributed_wind) * [ Electricity](https://atb.nrel.gov/electricity/2024/index) * [About](https://atb.nrel.gov/electricity/2024/about) * [Technologies](https://atb.nrel.gov/electricity/2024/technologies) * [Data](https://atb.nrel.gov/electricity/2024/data) * [ Transportation](https://atb.nrel.gov/electricity/2024/distributed_wind) * [ Transportation](https://atb.nrel.gov/transportation/2024/index) * [About](https://atb.nrel.gov/transportation/2024/about) * [Technologies](https://atb.nrel.gov/transportation/2024/technologies) * [Data](https://atb.nrel.gov/transportation/2024/data) * [ Contact](https://atb.nrel.gov/electricity/2024/distributed_wind) * [Contact Us](https://atb.nrel.gov/contact) * [Register for Updates](https://atb.nrel.gov/register) * [Archive](https://atb.nrel.gov/archive) * [ ](https://atb.nrel.gov/search) * [Login](https://atb.nrel.gov/login) # Distributed Wind ![](https://atb.nrel.gov/img/other/tableau-animated-loader.png) ## Resource Categorization The U.S. Department of Energy's (DOE's) Wind Energy Technologies Office [defines distributed wind](https://www.energy.gov/eere/wind/distributed-wind) in terms of technology application, based on a wind plant's location relative to end-use and power distribution infrastructure, rather than technology or project size. The following wind system attributes are used by the office to characterize them as distributed: * **Proximity to End Use:** wind turbines installed at or near the point of end use to meet on-site energy demand or support the operation of the existing distribution grid * **Point of Interconnection:** wind turbines connected on the customer side of the meter (also known as behind-the-meter), are connected directly to the distribution grid, or are off-grid in a remote location. Distributed wind energy systems are commonly installed on, but are not limited to, residential, agricultural, commercial, industrial, and community sites and can range in size from a 1-kilowatt (kW) turbine at a home to multimegawatt turbines at a manufacturing facility or connected to a local distribution system. Distributed wind project performance and cost are represented using four turbine technology classes: residential, commercial, midsize, and large. When used in the context of wind turbine technology, these labels refer specifically to the size of the turbines. Notably, any turbine size may be installed at the consumer’s site, independent of sector (residential, commercial, or industrial), with the applicability of a specific turbine being governed by both the ability to site a given machine on a specific parcel of land and the consumer’s load [(Lantz et al., 2016)](https://atb.nrel.gov/electricity/2024/distributed_wind#ICJXFBVE). ![](https://atb.nrel.gov/img/electricity/2022/p137/v1/dw_scales.png) Turbine Technology Classes **Turbine Technology Class** | **Size Range (kW)** | **Hub Height (meters [m])** \\---|---|--- Residential | <20 | 35 Commercial | 21–100 | 45 Midsize | 101–999 | 55 Large | >1,000 | 80 For distributed wind, similar to land-based utility-scale wind, each of the potential wind sites characterized in the ATB is associated with 1 of 10 wind speed classes. The following table shows each resource class as well as the resulting mean wind speed ranges that define each class. Wind Classes 1–10 sweep an exhaustive range of average annual wind speeds that distributed wind projects can be expected to have in the contiguous United States. Wind Speed Class 1 suggests a resource-rich wind site that is most attractive for wind project development, and Wind Speed Class 10 represents a less favorable wind resource site. Wind Class | Average Annual Wind Speed at 110 m Aboveground | Shear | Weibull K Factor \\---|---|---|--- 1 | 9.52 | 0.14 | 2 2 | 8.87 | 0.14 | 2 3 | 8.66 | 0.14 | 2 4 | 8.45 | 0.14 | 2 5 | 8.2 | 0.14 | 2 6 | 7.84 | 0.14 | 2 7 | 7.36 | 0.14 | 2 8 | 6.8 | 0.14 | 2 9 | 6.21 | 0.14 | 2 10 | 5.13 | 0.14 | 2 ## Scenario Descriptions Given the highly localized nature of the techno-economic performance of a distributed wind project, estimating future capital costs and other trends is challenging. The current status of the industry coupled with substantial uncertainty in terms of future demand and deployment at the global level add to the challenge. The costs modeled here reflect an initial assessment but also a rigorous analytical estimate. Given the uncertainty in this domain, a broad range of values is created to support scenario analysis. For large turbines, we rely on DOE's Low Wind Cost levelized cost of energy (LCOE) trajectory detailed by Wind Vision 2015 and in the National Renewable Energy Laboratory's (NREL's) ATB. This independent LCOE trajectory is used for large wind turbines, which have seen significant public and private sector research and development investment over the past 3 decades and are believed to have captured a more substantial portion of their long-term cost reduction potential in today’s modern technology. This trajectory results in substantially less change in 2050 LCOE than the various trajectories applied to the other distributed wind size classes. Notably, although we use the LCOE trajectory from Wind Vision 2015 for the large turbine sizes, we do not attempt to use the specific LCOE input values (e.g., capital cost, operation and maintenance [O&M;], and capacity factors) detailed in DOE’s Wind Vision (2015). Rather, we use the characterizations noted previously and calculated the capital cost required to achieve the specified LCOE reduction. DOE’s Low Wind Cost trajectory [(DOE, 2015)](https://atb.nrel.gov/electricity/2024/distributed_wind#9J2RJMVW) is applied for two reasons: It is more consistent with subsequent analysis and research on future wind turbine cost trajectories [(Wiser and Bolinger, 2016)](https://atb.nrel.gov/electricity/2024/distributed_wind#QSTSTRKJ), and it is assumed distributed wind applications have opportunities for innovation and efficiency in development and balance-of-station costs that may increase their probability of relatively greater cost reduction than modern multimegawatt wind power plants [(Bhaskar and Stehly, 2021)](https://atb.nrel.gov/electricity/2024/distributed_wind#ABU9QBBN). A summary of the capital expenditures (CAPEX) reductions between 2022 and 2030 is provided in the table below (Summary of Technology Innovation by Scenario). Summary of Technology Innovation by Scenario **Scenario** | **Innovations** \\---|--- Conservative Scenario | Limited adoption of technology innovation Moderate Scenario | Moderate adoption of longer wind turbine blades Standardized zoning, permitting, interconnection, and incentives Moderate volume of turbine manufacturing leading to lower overhead charged per wind turbine Advanced Scenario | Advancements in blade design that enable full adoption of long wind turbine blades Fully integrated zoning, permitting, interconnection, and incentives increase efficiencies and streamline processes High volume of turbine manufacturing significantly reduces overhead charged per wind turbine ### Scenario Assumptions The cost reduction pathways for the distributed wind scenarios are developed from a combination of published literature, prior techno-economic analyses, and utility-scale land-based wind technology. Primarily, the reductions are derived from land-based wind median learning rate, projected global growth rates, short-term cost estimates, and the Wind Vision study cost reduction assumptions [(DOE, 2015)](https://atb.nrel.gov/electricity/2024/distributed_wind#9J2RJMVW). Summary of Cost Reduction Assumptions by Scenario **Scenario** | **Description** \\---|--- Conservative Scenario | * 34% reduction in capital costs by 2030 and 63% reduction by 2050 * 3% reduction in O&M; costs by 2030 and 9% reduction by 2050 Moderate Scenario | * 53% reduction in capital costs by 2030 and 73% reduction by 2050 * 3% reduction in O&M; costs by 2030 and 9% reduction by 2050 Advanced Scenario | * 65% reduction in capital costs by 2030 and 83% reduction by 2050 * 3% reduction in O&M; costs by 2030 and 9% reduction by 2050 Values presented in the table are for the commercial-scale wind turbine; different levels of cost reduction are assumed for the other distributed wind turbine scales (i.e., residential, midsize, and large). ## Representative Technology Analysis results for distributed wind detailed in the ATB are contingent on a detailed characterization or representation of distributed wind technologies, and estimates of addressable resource potential require knowledge of potential hub heights and energy generation for wind turbines ranging from kilowatt- to megawatt-scale. The technology characterizations, such as turbine rating, hub height, and rotor diameter for the baseline scenarios, are summarized in the following table. These characteristics represent recent (2019) technology, as informed by empirical data and certified wind turbine equipment [(Bhaskar and Stehly, 2021)](https://atb.nrel.gov/electricity/2024/distributed_wind#ABU9QBBN). To obtain current and future cost and performance estimates, the technology representations for the Base Year (2022) and 2030 (future) are defined. For a detailed breakdown of current and future distributed wind turbine performance, including their representative power curves (current and future), see Appendix B of the DWFS 2022 [(McCabe et al., 2022)](https://atb.nrel.gov/electricity/2024/distributed_wind#E3EZRW5Y). The turbine performance improvements through 2050 remain the same across all three scenarios (Conservative, Moderate, Advanced). Existing literature on the future performance improvements of distributed wind turbines does not break down the performance improvements by various scenarios, resulting in single performance improvement trajectories for all four turbine classes [(DOE, 2015)](https://atb.nrel.gov/electricity/2024/distributed_wind#9J2RJMVW) and [(Lantz et al., 2016)](https://atb.nrel.gov/electricity/2024/distributed_wind#ICJXFBVE). In contrast to land-based utility-scale wind, distributed wind projects tend to be single-turbine projects (though projects can have more than one turbine). The technological advances seen in multiturbine utility-scale projects primarily revolve around advances in plant-level improvements in energy production owing to plant-level design and optimization. As a result, most technological innovations in utility-scale wind do not translate to distributed wind projects. Characteristics of Representative Technology: Current (2022) Project Scale Classification | Turbine Rating (kW) | Hub Height (m) | Rotor Diameter (m) | Specific Power | Max Coefficient of Performance (Cp) | Max Tip Speed (meters per second [m/s]) | Max Tip-Speed Ratio \\---|---|---|---|---|---|---|--- Residential | 20 | 30 | 12.4 | 166 | 0.4 | 95 | 9.7 Commercial | 100 | 40 | 27.6 | 167 | 0.5 | 75 | 8 Midsize | 650 | 60 | 70 | 169 | 0.5 | 70 | 8 Large | 1,500 | 80 | 107 | 167 | 0.5 | 82 | 8 Characteristics of Representative Technology: Future (2030 and Beyond) Project Scale Classification | Turbine Rating (kW) | Hub Height (m) | Rotor Diameter (m) | Specific Power | Max Coefficient of Performance (Cp) | Max Tip Speed (m/s) | Max Tip-Speed Ratio \\---|---|---|---|---|---|---|--- Residential | 20 | 35 | 13 | 150 | 0.4 | 95 | 9.7 Commercial | 100 | 45 | 29.1 | 150 | 0.5 | 95 | 9.7 Midsize | 650 | 65 | 74.3 | 150 | 0.5 | 95 | 9.7 Large | 1,500 | 85 | 112.8 | 150 | 0.5 | 95 | 9.7 ## Methodology This section describes the methodology to estimate current and future CAPEX, O&M;, and capacity factor. The current and future cost and performance estimates assume a single-turbine project across all four turbine technology classes, which is consistent with current project sizes seen in literature [(Bhaskar and Stehly, 2021)](https://atb.nrel.gov/electricity/2024/distributed_wind#ABU9QBBN). To resolve the substantial uncertainty in future capital cost estimates for distributed wind applications, we develop an analytical method that forms the basis for our estimates. For residential, commercial, and midsize turbine classes, we use a combination of technology learning rates and estimated global growth rates to identify potential changes in estimated LCOE [(Lantz et al., 2016)](https://atb.nrel.gov/electricity/2024/distributed_wind#ICJXFBVE). The resulting LCOE trajectories are used in conjunction with future performance and O&M; characterizations and with constant financing terms to calculate capital costs [(McCabe et al., 2022)](https://atb.nrel.gov/electricity/2024/distributed_wind#E3EZRW5Y). ### Capital Expenditures (CAPEX) For the future CAPEX projections of distributed wind projects, we use the same approach to modeling future cost projections of distributed wind under the various scenarios discussed in [(Lantz et al., 2016)](https://atb.nrel.gov/electricity/2024/distributed_wind#ICJXFBVE) but for updated 2022 costs obtained from NREL’s 2022 Cost of Wind Energy study [(Stehly et al., 2023)](https://atb.nrel.gov/electricity/2024/distributed_wind#TZV8XQ9Q) and the Distributed Wind Futures Study [(McCabe et al., 2022)](https://atb.nrel.gov/electricity/2024/distributed_wind#E3EZRW5Y). In 2023, CAPEX increases from 2022 because of supply chain stress, high commodity prices, and increased logistics costs. These near-term factors are accounted for by applying an average CAPEX multiplier of 10% in 2023 and 5% in 2024, relative to 2022 costs. These inflationary impacts are assumed to recover around 2025 ([(IEA, 2022)](https://atb.nrel.gov/electricity/2024/distributed_wind#T7AZBPFR), [(Vestas, 2022)](https://atb.nrel.gov/electricity/2024/distributed_wind#WB9UGKZP)). Components of the CAPEX for a representative distributed wind project of any scale are summarized in [(McCabe et al., 2022)](https://atb.nrel.gov/electricity/2024/distributed_wind#E3EZRW5Y) and [(Bhaskar and Stehly, 2021)](https://atb.nrel.gov/electricity/2024/distributed_wind#ABU9QBBN). Components of the CAPEX Cost Type | Cost Component \\---|--- Turbine CAPEX | Turbine (price to installer) Tower (price to installer) Other turbine equipment costs Original equipment manufacturer's extended warranty Transport to customer Balance of System CAPEX | Zoning, permitting, interconnection, and incentives Engineering and design Management (e.g., project and construction management) Turbine foundation Site preparation Grid connection Collection system Turbine erection and installation Development costs In the ATB, CAPEX reflects typical plants and does not include differences in regional costs associated with labor, materials, taxes, or system requirements. The CAPEX estimations calculated here are for single-turbine projects across all four project scales. Distributed wind projects range from 1 kW in size to multimegawatt turbines. Accordingly, distributed wind projects can have multiple turbines within the same project, although the likelihood of multiple turbines in a project is higher for midsize and large-scale projects (as opposed to residential and commercial-scale projects). Estimating the cost of multiturbine distributed wind projects is beyond the scope of this ATB analysis. For a detailed assessment of multiturbine midsize and large projects, refer to [(Bhaskar and Stehly, 2021)](https://atb.nrel.gov/electricity/2024/distributed_wind#ABU9QBBN). Balance of System (BOS) CAPEX ![](https://atb.nrel.gov/img/electricity/2024/p196/v1/screenshot_2024-06-07_at_8.54.19%E2%80%AFam.png) **Source****:**[(Bhaskar and Stehly, 2021)](https://atb.nrel.gov/electricity/2024/distributed_wind#ABU9QBBN) ### Operation and Maintenance (O&M;) Costs **Definition:** O&M; costs represent the all-in fixed and variable expenditures required to operate and maintain a distributed wind project. For a detailed breakdown of the components of the O&M; cost bucket, review the [O&M; section of the land-based wind page of the ATB](https://atb.nrel.gov/electricity/2024/land-based_wind#operation_and_maintenance_\\\\(o&m;\\\\)_costs). **Base Year** : The all-in O&M; costs of $39/kW-yr in the Base Year for all four project scales are reported in the _2022 Cost of Wind Energy Review_ [(Stehly et al., 2023)](https://atb.nrel.gov/electricity/2024/distributed_wind#TZV8XQ9Q). **Future Years** : O&M; costs are assumed to decline consistently across turbine sizes by a cumulative 10% (an annual rate of approximately 0.3% per year) between 2022 and 2050, consistent with the Wind Vision [(DOE, 2015)](https://atb.nrel.gov/electricity/2024/distributed_wind#9J2RJMVW), for all scenarios that assume a change in distributed wind O&M.; ### Capacity Factor **Definition** : Similar to land-based wind, the capacity factor for a distributed wind project is influenced by the project's generation profile, expected downtime, and energy losses within the wind plant. The specific power (i.e., ratio of machine rating to rotor-swept area), hub height, and use of stall-regulated or pitch-regulated machine are design choices that influence the capacity factor. To calculate the Base Year and future capacity factors, an idealized power curve is developed for each current and future [Representative Technology](https://atb.nrel.gov/electricity/2024/distributed_wind#representative_technology) in the System Advisor Model ([SAM](https://sam.nrel.gov)) and run for each of the weighted average wind speeds in each wind speed class. The capacity factors are calculated at the representative turbine hub height by extrapolating the wind speed up or down from the referenced 110-m, above-ground-level, long-term average hourly wind resource data from the [Wind Integration National Dataset (WIND) Toolkit](https://www.nrel.gov/grid/wind-toolkit.html). The following chart shows a range of capacity factors based on variation in the resource for distributed wind projects in the contiguous United States and the future capacity factor estimates for the Conservative, Moderate, and Advanced Scenarios. ![](https://atb.nrel.gov/img/other/tableau-animated-loader.png) **Future Trends** Future performance of distributed wind technologies is characterized using two primary assumptions. First, we assume turbine-specific power will continue to decline to levels of approximately 150 watts per square meter (W/m2). Although future specific power estimates will ultimately be determined by the relative cost of additional blade length and rotor growth to additional generator capacity along with the anticipated time spent operating at maximum power, the 150-W/m2 endpoint is supported by a few known concepts or prototype machines designed for the distributed wind space. In addition, there is an increased incentive to go to lower specific power in the lower-wind-quality locations where distributed wind often finds itself (i.e., a different design optimum). Also, some existing distributed wind turbine platforms may have more robust design margins in other portions of the turbine (e.g., the turbine mainframe) as a function of more limited means to have unique custom components that could further facilitate a move to lower specific power. Improvements to BOS costs (site preparation, wind turbine foundation, electrical, and tower) are also considered. We assume performance converges over time as technology advancements are adopted within and across each of the respective size classes. This assumption is grounded in the absence of technical constraints that would preclude convergence. As a simplifying assumption, capacity factor improvements are equivalent across all modeled scenarios and analysis that include performance improvements. To characterize the temporal rate of change in performance, we assume improvements will be fairly sizable in the near term and innovation will have diminishing returns over time so that by 2050 continued annual improvements in capacity factor will be well below 1% per year. Resulting capacity factors are projected to 2050 by turbine size class and based on the methods described previously [(Lantz et al., 2016)](https://atb.nrel.gov/electricity/2024/distributed_wind#ICJXFBVE). ## References The following references are specific to this page; for all references in this ATB, see [References](https://atb.nrel.gov/electricity/2024/references). Bhaskar, Parangat, and Tyler Stehly. “Technology Innovation Pathways for Distributed Wind Balance-of-System Cost Reduction.” Golden, CO: National Renewable Energy Laboratory, April 6, 2021. . DOE. “Wind Vision: A New Era for Wind Power in the United States.” Washington, D.C.: U.S. Department of Energy, 2015. . IEA. “Impact of High Commodity Price Scenario on Forecast Total Investment Costs and CAPEX, Onshore Wind and Utility-Scale PV, 2015-2026,” October 26, 2022. . Lantz, Eric, Benjamin Sigrin, Michael Gleason, Robert Preus, and Ian Baring-Gould. “Assessing the Future of Distributed Wind: Opportunities for Behind-the-Meter Projects.” Golden, CO: National Renewable Energy Laboratory, November 1, 2016. . McCabe, Kevin, Ashreeta Prasanna, Jane Lockshin, Parangat Bhaskar, Thomas Bowen, Ruth Baranowski, Ben Sigrin, and Eric Lantz. “Distributed Wind Energy Futures Study.” Golden, CO: National Renewable Energy Laboratory, May 2022. . Stehly, Tyler, Patrick Duffy, and Daniel Mulas Hernando. “2022 Cost of Wind Energy Review.” December 2023. . Vestas. “Vestas - Q3 2022 Investor Presentation.” November 2022. . Wiser, Ryan, and Mark Bolinger. “2015 Wind Technologies Market Report,” 2016. . * [About](https://atb.nrel.gov/electricity/2024/about) * [Technologies](https://atb.nrel.gov/electricity/2024/technologies) * * [ Land-Based Wind ](https://atb.nrel.gov/electricity/2024/land-based_wind) * [ Offshore Wind ](https://atb.nrel.gov/electricity/2024/offshore_wind) * [ Distributed Wind ](https://atb.nrel.gov/electricity/2024/distributed_wind) * [ Utility-Scale PV ](https://atb.nrel.gov/electricity/2024/utility-scale_pv) * [ Commercial PV ](https://atb.nrel.gov/electricity/2024/commercial_pv) * [ Residential PV ](https://atb.nrel.gov/electricity/2024/residential_pv) * [ Concentrating Solar Power ](https://atb.nrel.gov/electricity/2024/concentrating_solar_power) * [ Geothermal ](https://atb.nrel.gov/electricity/2024/geothermal) * [ Hydropower ](https://atb.nrel.gov/electricity/2024/hydropower) * [ Utility-Scale PV-Plus-Battery ](https://atb.nrel.gov/electricity/2024/utility-scale_pv-plus-battery) * [ Utility-Scale Battery Storage ](https://atb.nrel.gov/electricity/2024/utility-scale_battery_storage) * [ Commercial Battery Storage ](https://atb.nrel.gov/electricity/2024/commercial_battery_storage) * [ Residential Battery Storage ](https://atb.nrel.gov/electricity/2024/residential_battery_storage) * [ Pumped Storage Hydropower ](https://atb.nrel.gov/electricity/2024/pumped_storage_hydropower) * [ Fossil Energy Technologies ](https://atb.nrel.gov/electricity/2024/fossil_energy_technologies) * [ Nuclear ](https://atb.nrel.gov/electricity/2024/nuclear) * [ Biopower (EIA) ](https://atb.nrel.gov/electricity/2024/biopower_\\\\(eia\\\\)) * [Data](https://atb.nrel.gov/electricity/2024/data) * * * Developed with funding from the U.S. Department of Energy’s Office of Energy Efficiency and Renewable Energy. * * * [**National Renewable Energy Laboratory**](https://www.nrel.gov/index.html) [About](https://www.nrel.gov/about/index.html) [Research](https://www.nrel.gov/research/index.html) [Work With Us](https://www.nrel.gov/index.html) [News](https://www.nrel.gov/news/index.html) [Careers](https://www.nrel.gov/careers/index.html) [Contact Us](https://atb.nrel.gov/contact) [Visit](https://www.nrel.gov/about/visiting-nrel.html) [Subscribe to NREL](https://www.nrel.gov/news/subscribe.html) [Accessibility](https://www.nrel.gov/accessibility.html) [Disclaimer](https://www.nrel.gov/disclaimer.html) [Security and Privacy](https://www.nrel.gov/security.html) [Site Feedback](https://www.nrel.gov/webmaster.html) [Developers](https://developer.nrel.gov/) [Employees](https://thesource.nrel.gov/) * * * [ ![Alliance for Sustainable Energy, LLC](https://atb.nrel.gov/client/img/alliance-logo_black.png) ](https://www.allianceforsustainableenergy.org/) [ ![U.S. Department of Energy](https://atb.nrel.gov/client/img/DOE-Logo-TM-Black.svg) ](https://www.energy.gov) The National Renewable Energy Laboratory is a national laboratory of the [U.S. Department of Energy](https://www.energy.gov/), [Office of Energy Efficiency and Renewable Energy](https://www.energy.gov/eere/office-energy-efficiency-renewable-energy), operated by the [Alliance for Sustainable Energy LLC](https://www.allianceforsustainableenergy.org/). Section Issue Type Problem Text Suggestion * * *\n", - "\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "print(atb_info[20].attrs[\"source\"])\n", "print(atb_info[20].text)" @@ -1013,7 +290,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1059,7 +336,7 @@ "from elm.ords.services.openai import OpenAIService\n", "\n", "SYSTEM_MESSAGE = (\n", - " \"You examine text scraped form the NREL ATB website to determine if it \"\n", + " \"You examine text scraped form the NLR ATB website to determine if it \"\n", " \"contains information relevant to the user's query. Base your responses \"\n", " \"only off of the content of the webpage and **not** on any auxiliary \"\n", " \"information.\"\n", @@ -1089,7 +366,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1135,7 +412,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1158,42 +435,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "DEBUG - 2025-07-03 00:02:54,377 [provider.py:137] : Initializing Service: OpenAIService\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "DEBUG - 2025-07-03 00:02:57,569 [website_crawl.py:518] : Crawled https://atb.nrel.gov/\n", - "DEBUG - 2025-07-03 00:03:00,471 [website_crawl.py:518] : Crawled https://atb.nrel.gov/archive\n", - "DEBUG - 2025-07-03 00:03:08,155 [website_crawl.py:518] : Crawled https://atb.nrel.gov/contact\n", - "DEBUG - 2025-07-03 00:03:10,733 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/index\n", - "DEBUG - 2025-07-03 00:03:12,260 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/data\n", - "DEBUG - 2025-07-03 00:03:13,532 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/data\n", - "DEBUG - 2025-07-03 00:03:14,594 [website_crawl.py:518] : Crawled https://atb.nrel.gov/transportation/2024/about\n", - "DEBUG - 2025-07-03 00:03:15,675 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/technologies\n", - "DEBUG - 2025-07-03 00:03:16,669 [website_crawl.py:518] : Crawled https://atb.nrel.gov/search\n", - "DEBUG - 2025-07-03 00:03:17,547 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/about\n", - "DEBUG - 2025-07-03 00:03:22,573 [website_crawl.py:518] : Crawled https://atb.nrel.gov/electricity/2024/geothermal\n", - "DEBUG - 2025-07-03 00:03:25,089 [website_crawl.py:533] : Document passed validation check\n", - "DEBUG - 2025-07-03 00:03:25,089 [website_crawl.py:537] : Exiting crawl early\n", - "INFO - 2025-07-03 00:03:25,304 [website_crawl.py:540] : Crawled 11 pages\n", - "INFO - 2025-07-03 00:03:25,304 [website_crawl.py:541] : Found 1 potential documents\n", - "DEBUG - 2025-07-03 00:03:25,305 [website_crawl.py:542] : Average score: 1.00\n", - "DEBUG - 2025-07-03 00:03:25,305 [website_crawl.py:549] : Pages crawled by depth:\n", - "DEBUG - 2025-07-03 00:03:25,306 [website_crawl.py:551] : Depth 0: 1 pages\n", - "DEBUG - 2025-07-03 00:03:25,307 [website_crawl.py:551] : Depth 1: 9 pages\n", - "DEBUG - 2025-07-03 00:03:25,307 [website_crawl.py:551] : Depth 2: 1 pages\n", - "DEBUG - 2025-07-03 00:03:25,308 [provider.py:151] : Tearing down Service: OpenAIService\n" - ] - } - ], + "outputs": [], "source": [ "import openai\n", "from elm.utilities import validate_azure_api_params\n", @@ -1214,23 +456,9 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[HTMLDocument with 1 pages\n", - " Attrs:\n", - " source:\thttps://atb.nrel.gov/electricity/2024/geothermal\n", - " website_link_relevance_score:\t2]" - ] - }, - "execution_count": 49, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "geo_info" ] @@ -1251,7 +479,7 @@ ], "metadata": { "kernelspec": { - "display_name": "dev", + "display_name": "Python 3", "language": "python", "name": "python3" }, diff --git a/requirements.txt b/requirements.txt index 907e39c2..5d1f3252 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ langchain-text-splitters lxml matplotlib networkx -nrel-rex +nlr-rex nltk numpy pandas diff --git a/setup.py b/setup.py index 9f564f19..d9eb11e2 100644 --- a/setup.py +++ b/setup.py @@ -25,13 +25,13 @@ description = "Energy Language Model" setup( - name="nrel_elm", + name="nlr_elm", version=version, description=description, long_description=readme, author="Grant Buster", - author_email="grant.buster@nrel.gov", - url="https://github.com/NREL/elm", + author_email="grant.buster@nlr.gov", + url="https://github.com/NatLabRockies/elm", packages=find_packages(), package_dir={"elm": "elm"}, license="BSD 3-Clause", diff --git a/tests/ords/test_integrated.py b/tests/ords/test_integrated.py index 71051427..c8b03523 100644 --- a/tests/ords/test_integrated.py +++ b/tests/ords/test_integrated.py @@ -2,6 +2,7 @@ # pylint: disable=missing-class-docstring,missing-function-docstring # pylint: disable=unused-argument """ELM Ordinance integration tests""" +import os import time import logging import asyncio @@ -142,6 +143,8 @@ async def _test_response(*args, **kwargs): } +@pytest.mark.skipif(os.getenv("GITHUB_ACTIONS") == "true", + reason="Flaky in GHA") @pytest.mark.asyncio async def test_search_with_logging(tmp_path): """Test searching for some counties with logging""" diff --git a/tests/ords/utilities/test_utilities_exceptions.py b/tests/ords/utilities/test_utilities_exceptions.py index ded4e385..390bfc80 100644 --- a/tests/ords/utilities/test_utilities_exceptions.py +++ b/tests/ords/utilities/test_utilities_exceptions.py @@ -2,7 +2,7 @@ """Test ELM Ordinance exception types. Most exception logic + tests pulled from GAPs -(https://github.com/NREL/gaps) +(https://github.com/NatLabRockies/gaps) """ from pathlib import Path diff --git a/tests/web/search/test_web_search.py b/tests/web/search/test_web_search.py index 04b6e01a..001df274 100644 --- a/tests/web/search/test_web_search.py +++ b/tests/web/search/test_web_search.py @@ -14,6 +14,9 @@ import elm.web.search.bing import elm.web.search.yahoo +pytestmark = pytest.mark.skipif(os.getenv("GITHUB_ACTIONS") == "true", + reason="Flaky in GHA") + SE_TO_TEST = [(elm.web.search.bing.PlaywrightBingLinkSearch, {}), (elm.web.search.yahoo.PlaywrightYahooLinkSearch, {})] if CSE_ID := os.getenv("GOOGLE_CSE_ID"): diff --git a/tests/web/search/test_web_search_api.py b/tests/web/search/test_web_search_api.py index 2eae1a05..76030773 100644 --- a/tests/web/search/test_web_search_api.py +++ b/tests/web/search/test_web_search_api.py @@ -45,8 +45,9 @@ async def _search(self, *__, **___): @pytest.mark.skipif(os.getenv("GITHUB_ACTIONS") == "true", reason="Fails in GHA due to rate limiting") -@pytest.mark.parametrize("queries", [['1. "NREL elm"'], - ['1. "NREL elm"', "NREL reV"],]) +@pytest.mark.parametrize("queries", [['1. "NatLabRockies elm"'], + ['1. "NatLabRockies elm"', + "NatLabRockies reV"],]) @pytest.mark.parametrize("se", SE_API_TO_TEST) @pytest.mark.asyncio async def test_basic_search_query(queries, se):