From 78edd5cfb214910ca67a64bc9a66dfa1457c304f Mon Sep 17 00:00:00 2001 From: Sujee Maniyam Date: Tue, 5 Nov 2024 23:06:35 -0800 Subject: [PATCH 01/31] crawling with DPK connector Signed-off-by: Sujee Maniyam --- examples/notebooks/html-processing/.gitignore | 2 + .../html-processing/1_download_site.ipynb | 214 ++++++++++++++++++ examples/notebooks/html-processing/README.md | 28 +++ .../notebooks/html-processing/my_config.py | 20 ++ .../notebooks/html-processing/my_utils.py | 102 +++++++++ .../html-processing/requirements.txt | 15 ++ 6 files changed, 381 insertions(+) create mode 100644 examples/notebooks/html-processing/.gitignore create mode 100644 examples/notebooks/html-processing/1_download_site.ipynb create mode 100644 examples/notebooks/html-processing/README.md create mode 100644 examples/notebooks/html-processing/my_config.py create mode 100644 examples/notebooks/html-processing/my_utils.py create mode 100644 examples/notebooks/html-processing/requirements.txt diff --git a/examples/notebooks/html-processing/.gitignore b/examples/notebooks/html-processing/.gitignore new file mode 100644 index 000000000..ebbfa428f --- /dev/null +++ b/examples/notebooks/html-processing/.gitignore @@ -0,0 +1,2 @@ +input/* +output*/ \ No newline at end of file diff --git a/examples/notebooks/html-processing/1_download_site.ipynb b/examples/notebooks/html-processing/1_download_site.ipynb new file mode 100644 index 000000000..cc7bffe11 --- /dev/null +++ b/examples/notebooks/html-processing/1_download_site.ipynb @@ -0,0 +1,214 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from my_config import MY_CONFIG" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ Cleared output directory\n" + ] + } + ], + "source": [ + "import os, sys\n", + "import shutil\n", + "\n", + "if not os.path.exists(MY_CONFIG.INPUT_DIR ):\n", + " shutil.os.makedirs(MY_CONFIG.INPUT_DIR, exist_ok=True)\n", + "\n", + "## clear output folder\n", + "shutil.rmtree(MY_CONFIG.OUTPUT_DIR, ignore_errors=True)\n", + "shutil.os.makedirs(MY_CONFIG.OUTPUT_DIR, exist_ok=True)\n", + "\n", + "print (\"✅ Cleared output directory\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Download a Website\n", + "\n", + "We will use `dpk-connector` utility to download a some HTML pages from a site " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Visited url: https://thealliance.ai/our-work\n", + "input/our-work\n", + "Saved contents of url: https://thealliance.ai/our-work\n", + "Visited url: https://thealliance.ai/affiliated-projects/quansight_ragna\n", + "input/quansight_ragna\n", + "Saved contents of url: https://thealliance.ai/affiliated-projects/quansight_ragna\n", + "Visited url: https://thealliance.ai/affiliated-projects/cornell_generative\n", + "input/cornell_generative\n", + "Saved contents of url: https://thealliance.ai/affiliated-projects/cornell_generative\n", + "Visited url: https://thealliance.ai/core-projects/trust-and-safety-evaluations\n", + "input/trust-and-safety-evaluations\n", + "Saved contents of url: https://thealliance.ai/core-projects/trust-and-safety-evaluations\n", + "Visited url: https://thealliance.ai/affiliated-projects/mbzuai_llm\n", + "input/mbzuai_llm\n", + "Saved contents of url: https://thealliance.ai/affiliated-projects/mbzuai_llm\n", + "Visited url: https://thealliance.ai/affiliated-projects/trusty_ai\n", + "input/trusty_ai\n", + "Saved contents of url: https://thealliance.ai/affiliated-projects/trusty_ai\n", + "Visited url: https://thealliance.ai/core-projects/time-series-data-and-model-initiative\n", + "input/time-series-data-and-model-initiative\n", + "Saved contents of url: https://thealliance.ai/core-projects/time-series-data-and-model-initiative\n", + "Visited url: https://thealliance.ai/core-projects/open-trusted-data-initiative\n", + "input/open-trusted-data-initiative\n", + "Saved contents of url: https://thealliance.ai/core-projects/open-trusted-data-initiative\n", + "Visited url: https://thealliance.ai/core-projects/safety-priorities-ranking-by-domain\n", + "input/safety-priorities-ranking-by-domain\n", + "Saved contents of url: https://thealliance.ai/core-projects/safety-priorities-ranking-by-domain\n", + "Visited url: https://thealliance.ai/core-projects/ntia_request\n", + "input/ntia_request\n", + "Saved contents of url: https://thealliance.ai/core-projects/ntia_request\n", + "input/aiconfig_editor\n", + "Saved contents of url: https://thealliance.ai/affiliated-projects/aiconfig_editor\n", + "input/ibm_granite\n", + "Saved contents of url: https://thealliance.ai/affiliated-projects/ibm_granite\n", + "input/the-living-guide-to-applying-ai\n", + "Saved contents of url: https://thealliance.ai/core-projects/the-living-guide-to-applying-ai\n", + "input/trusted-evals\n", + "Saved contents of url: https://thealliance.ai/core-projects/trusted-evals\n", + "input/sb1047\n", + "Saved contents of url: https://thealliance.ai/core-projects/sb1047\n", + "input/ai-accelerator-software-ecosystem-guide\n", + "Saved contents of url: https://thealliance.ai/core-projects/ai-accelerator-software-ecosystem-guide\n", + "input/trust-user-guide\n", + "Saved contents of url: https://thealliance.ai/core-projects/trust-user-guide\n", + "input/contact\n", + "Saved contents of url: https://thealliance.ai/contact\n", + "input/industry-open-fms-initiative\n", + "Saved contents of url: https://thealliance.ai/core-projects/industry-open-fms-initiative\n", + "input/become-a-collaborator\n", + "Saved contents of url: https://thealliance.ai/become-a-collaborator\n", + "input/affiliated-projects\n", + "input/advocacy\n", + "input/community\n", + "input/foundation-models-datasets\n", + "input/hardware-enablement\n", + "input/foundation-models\n", + "input/contribute\n", + "input/applications-and-tools\n", + "input/aia-members\n", + "input/about-aia\n", + "input/skills-education\n", + "input/trust-and-safety\n", + "input/blog\n", + "input/core-projects\n" + ] + }, + { + "data": { + "text/plain": [ + "'Crawl is done'" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from dpk_connector import crawl, shutdown\n", + "import nest_asyncio\n", + "import os\n", + "from my_utils import get_mime_type, get_filename_from_url\n", + "from dpk_connector.core.utils import validate_url\n", + "\n", + "# Use nest_asyncio to enable a nested event loop run for the crawler inside the Jupyter notebook\n", + "nest_asyncio.apply()\n", + "\n", + "# Initialize counter\n", + "retrieved_pages = 0\n", + "saved_pages = 0\n", + "\n", + "# Define a callback function to be executed at the retrieval of each page during a crawl\n", + "def on_downloaded(url: str, body: bytes, headers: dict) -> None:\n", + " \"\"\"\n", + " Callback function called when a page has been downloaded.\n", + " You have access to the request URL, response body and headers.\n", + " \"\"\"\n", + " global retrieved_pages, saved_pages\n", + " retrieved_pages+=1\n", + "\n", + " if saved_pages < MY_CONFIG.CRAWL_MAX_DOWNLOADS:\n", + " print(f\"Visited url: {url}\")\n", + "\n", + " # Get mime_type of retrieved page\n", + " mime_type = get_mime_type(body)\n", + " \n", + " # Save the page if it is a PDF to only download research papers\n", + " if MY_CONFIG.CRAWL_MIME_TYPE in mime_type.lower():\n", + " filename = get_filename_from_url(url)\n", + " local_file_path = os.path.join(MY_CONFIG.INPUT_DIR, filename)\n", + " print (local_file_path)\n", + " \n", + " with open(local_file_path, 'wb') as f:\n", + " f.write(body)\n", + " \n", + " if saved_pages < MY_CONFIG.CRAWL_MAX_DOWNLOADS :\n", + " print(f\"Saved contents of url: {url}\")\n", + " saved_pages+=1\n", + " \n", + "# Define a user agent to provide information about the client making the request\n", + "user_agent = \"dpk-connector\"\n", + "\n", + "async def run_my_crawl():\n", + " crawl([MY_CONFIG.CRAWL_URL_BASE], \n", + " on_downloaded, \n", + " user_agent=user_agent, \n", + " depth_limit = MY_CONFIG.CRAWL_MAX_DEPTH, \n", + " path_focus = True, \n", + " download_limit = MY_CONFIG.CRAWL_MAX_DOWNLOADS)\n", + " return \"Crawl is done\"\n", + "\n", + "# Now run the configured crawl\n", + "await run_my_crawl()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dpk-3-r022dev2-py311", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/notebooks/html-processing/README.md b/examples/notebooks/html-processing/README.md new file mode 100644 index 000000000..425689790 --- /dev/null +++ b/examples/notebooks/html-processing/README.md @@ -0,0 +1,28 @@ +# Using Data Prep Kit to Process HTML files + +## Step-1: Setup Python Env + +See [project README](../../../README.md#create-a-virtual-environment). + +Here is a quick setup guide + +```bash +conda create -n dpk-r022dev2-py311 python=3.11 +conda activate dpk-r022dev2-py311 +``` + +If on linux do this + +```bash +conda install gcc_linux-64 +conda install gxx_linux-64 +``` + +Install modules + +```bash +pip install -r requirements.txt +``` + + +## Download HTML pages \ No newline at end of file diff --git a/examples/notebooks/html-processing/my_config.py b/examples/notebooks/html-processing/my_config.py new file mode 100644 index 000000000..3993352c2 --- /dev/null +++ b/examples/notebooks/html-processing/my_config.py @@ -0,0 +1,20 @@ +import os + +## Configuration +class MyConfig: + pass + +MY_CONFIG = MyConfig () + +## Crawl settings +# MY_CONFIG.CRAWL_URL_BASE = 'https://thealliance.ai/' +MY_CONFIG.CRAWL_URL_BASE = 'https://thealliance.ai/our-work' +MY_CONFIG.CRAWL_MAX_DOWNLOADS = 10 +MY_CONFIG.CRAWL_MAX_DEPTH = 2 +MY_CONFIG.CRAWL_MIME_TYPE = 'html' + +## Input Data - configure this to the folder we want to process +MY_CONFIG.INPUT_DIR = "input" +MY_CONFIG.OUTPUT_DIR = "output" +MY_CONFIG.OUTPUT_DIR_FINAL = os.path.join(MY_CONFIG.OUTPUT_DIR , "output_final") +### ------------------------------- diff --git a/examples/notebooks/html-processing/my_utils.py b/examples/notebooks/html-processing/my_utils.py new file mode 100644 index 000000000..acea186c4 --- /dev/null +++ b/examples/notebooks/html-processing/my_utils.py @@ -0,0 +1,102 @@ +import os +import requests +from humanfriendly import format_size +import pandas as pd +import glob +import magic +from dpk_connector.core.utils import ( + urlparse_cached +) +from urllib.parse import unquote + + +## Reads parquet files in a folder into a pandas dataframe +def read_parquet_files_as_df (parquet_dir): + parquet_files = glob.glob(f'{parquet_dir}/*.parquet') + + # read each parquet file into a DataFrame and store in a list + dfs = [pd.read_parquet (f) for f in parquet_files] + + # Concatenate all DataFrames into a single DataFrame + data_df = pd.concat(dfs, ignore_index=True) + return data_df + + +def download_file(url, local_file, chunk_size=1024*1024): + """ + Downloads a remote URL to a local file. + + Args: + url (str): The remote URL. + local_filename (str): The name of the local file to save the downloaded content. + chunk_size (int): The size in bytes of each chunk. Defaults to 1024. + + Returns: + None + + Example usage: + download_file('http://example.com/file.txt', 'file.txt', chunk_size=1024*1024) # Download in chunks of 1MB + """ + # Check if the local file already exists + if os.path.exists(local_file): + file_size = format_size(os.path.getsize(local_file)) + print(f"Local file '{local_file}' ({file_size}) already exists. Skipping download.") + return + + # Create the directory if it doesn't exist + os.makedirs(os.path.dirname(local_file), exist_ok=True) + + # Stream the file download + with requests.get(url, stream=True) as r: + r.raise_for_status() + with open(local_file, 'wb') as f: + for chunk in r.iter_content(chunk_size=chunk_size): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + print() + file_size = format_size(os.path.getsize(local_file)) + print(f"{local_file} ({file_size}) downloaded successfully.") +## --- end: download_file ------ + +def get_mime_type(byte_data: bytes) -> str: + """ + Obtain the MIME type for provided byte data using the magic library. + + Args: + byte_data: bytes: Bytes data to identify mimetype for. + + Returns: + str: Mimetype for given bytes data. + + Example: + >>> byte_data = b'\n...' + >>> get_mime_type(byte_data) + 'text/html' + """ + + # Validate input type + if not isinstance(byte_data, bytes): + raise TypeError("Input must be of type 'bytes'") + + # Initialize a magic object for MIME type detection + mime = magic.Magic(mime=True) + + # Return MIME type from the byte data + return mime.from_buffer(byte_data) + + +def get_extension(url: str) -> str: + parsed = urlparse_cached(url) + path = parsed.path + filename = unquote(os.path.basename(path)) + ext = os.path.splitext(filename)[1] + if 16 < len(ext): + ext = ext[:16] + return ext + + +def get_filename_from_url(url: str) -> str: + parsed = urlparse_cached(url) + basename = unquote(os.path.splitext(os.path.basename(parsed.path))[0]) + ext = get_extension(url) + return basename + ext \ No newline at end of file diff --git a/examples/notebooks/html-processing/requirements.txt b/examples/notebooks/html-processing/requirements.txt new file mode 100644 index 000000000..750e5af4b --- /dev/null +++ b/examples/notebooks/html-processing/requirements.txt @@ -0,0 +1,15 @@ +data-prep-toolkit[ray]==0.2.2.dev2 +data-prep-toolkit-transforms[ray,all]==0.2.2.dev2 + +data-prep-connector==0.2.2 + +## utils +humanfriendly +pandas +mimetypes-magic +nest-asyncio + +## jupyter and friends +jupyterlab +ipykernel +ipywidgets \ No newline at end of file From 48a0d6cdba2d042cdd1fd0d2efffe3e09f8c751f Mon Sep 17 00:00:00 2001 From: Sujee Maniyam Date: Tue, 5 Nov 2024 23:18:48 -0800 Subject: [PATCH 02/31] tweaked steps Signed-off-by: Sujee Maniyam --- .../html-processing/1_download_site.ipynb | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/examples/notebooks/html-processing/1_download_site.ipynb b/examples/notebooks/html-processing/1_download_site.ipynb index cc7bffe11..fcee2f2af 100644 --- a/examples/notebooks/html-processing/1_download_site.ipynb +++ b/examples/notebooks/html-processing/1_download_site.ipynb @@ -1,5 +1,24 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Crawl a website \n", + "\n", + "We will use `DPK-Connector` module\n", + "\n", + "References\n", + "- https://github.com/IBM/data-prep-kit/tree/dev/data-connector-lib\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step-1: Config" + ] + }, { "cell_type": "code", "execution_count": 1, @@ -9,6 +28,13 @@ "from my_config import MY_CONFIG" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step-2: Setup Directories" + ] + }, { "cell_type": "code", "execution_count": 2, @@ -40,7 +66,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Download a Website\n", + "# Step-3: Crawl the website\n", "\n", "We will use `dpk-connector` utility to download a some HTML pages from a site " ] From 4cb96a99650f5ad04c72665a725750255972a5fd Mon Sep 17 00:00:00 2001 From: Sujee Maniyam Date: Thu, 7 Nov 2024 21:43:23 -0800 Subject: [PATCH 03/31] html processing getting started --- ...download_site.ipynb => 1_crawl_site.ipynb} | 114 +++++++--------- .../2_process_html_python.ipynb | 123 ++++++++++++++++++ examples/notebooks/html-processing/README.md | 17 ++- .../input2/thealliance.ai/about-aia.html | 25 ++++ .../input2/thealliance.ai/aia-members.html | 25 ++++ .../thealliance.ai/become-a-collaborator.html | 25 ++++ .../input2/thealliance.ai/blog.html | 25 ++++ .../input2/thealliance.ai/community.html | 25 ++++ .../input2/thealliance.ai/contact.html | 25 ++++ .../input2/thealliance.ai/contribute.html | 25 ++++ .../core-projects/ntia_request.html | 25 ++++ .../thealliance.ai/core-projects/sb1047.html | 25 ++++ .../core-projects/trusted-evals.html | 25 ++++ .../thealliance.ai/focus-areas/advocacy.html | 25 ++++ .../focus-areas/applications-and-tools.html | 25 ++++ .../foundation-models-datasets.html | 25 ++++ .../focus-areas/hardware-enablement.html | 25 ++++ .../focus-areas/skills-education.html | 25 ++++ .../focus-areas/trust-and-safety.html | 25 ++++ .../input2/thealliance.ai/index.html | 25 ++++ .../input2/thealliance.ai/our-work.html | 25 ++++ .../input2/thealliance.ai/robots.txt | 3 + .../notebooks/html-processing/my_config.py | 6 +- .../html-processing/requirements.txt | 9 +- examples/notebooks/html-processing/test.ipynb | 49 +++++++ examples/notebooks/html-processing/test.py | 27 ++++ 26 files changed, 719 insertions(+), 79 deletions(-) rename examples/notebooks/html-processing/{1_download_site.ipynb => 1_crawl_site.ipynb} (52%) create mode 100644 examples/notebooks/html-processing/2_process_html_python.ipynb create mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/about-aia.html create mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/aia-members.html create mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/become-a-collaborator.html create mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/blog.html create mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/community.html create mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/contact.html create mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/contribute.html create mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/core-projects/ntia_request.html create mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/core-projects/sb1047.html create mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/core-projects/trusted-evals.html create mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/advocacy.html create mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/applications-and-tools.html create mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/foundation-models-datasets.html create mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/hardware-enablement.html create mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/skills-education.html create mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/trust-and-safety.html create mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/index.html create mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/our-work.html create mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/robots.txt create mode 100644 examples/notebooks/html-processing/test.ipynb create mode 100644 examples/notebooks/html-processing/test.py diff --git a/examples/notebooks/html-processing/1_download_site.ipynb b/examples/notebooks/html-processing/1_crawl_site.ipynb similarity index 52% rename from examples/notebooks/html-processing/1_download_site.ipynb rename to examples/notebooks/html-processing/1_crawl_site.ipynb index fcee2f2af..6c99c3681 100644 --- a/examples/notebooks/html-processing/1_download_site.ipynb +++ b/examples/notebooks/html-processing/1_crawl_site.ipynb @@ -73,77 +73,42 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Visited url: https://thealliance.ai/our-work\n", - "input/our-work\n", - "Saved contents of url: https://thealliance.ai/our-work\n", - "Visited url: https://thealliance.ai/affiliated-projects/quansight_ragna\n", - "input/quansight_ragna\n", - "Saved contents of url: https://thealliance.ai/affiliated-projects/quansight_ragna\n", - "Visited url: https://thealliance.ai/affiliated-projects/cornell_generative\n", - "input/cornell_generative\n", - "Saved contents of url: https://thealliance.ai/affiliated-projects/cornell_generative\n", - "Visited url: https://thealliance.ai/core-projects/trust-and-safety-evaluations\n", - "input/trust-and-safety-evaluations\n", - "Saved contents of url: https://thealliance.ai/core-projects/trust-and-safety-evaluations\n", - "Visited url: https://thealliance.ai/affiliated-projects/mbzuai_llm\n", - "input/mbzuai_llm\n", - "Saved contents of url: https://thealliance.ai/affiliated-projects/mbzuai_llm\n", - "Visited url: https://thealliance.ai/affiliated-projects/trusty_ai\n", - "input/trusty_ai\n", - "Saved contents of url: https://thealliance.ai/affiliated-projects/trusty_ai\n", - "Visited url: https://thealliance.ai/core-projects/time-series-data-and-model-initiative\n", - "input/time-series-data-and-model-initiative\n", - "Saved contents of url: https://thealliance.ai/core-projects/time-series-data-and-model-initiative\n", - "Visited url: https://thealliance.ai/core-projects/open-trusted-data-initiative\n", - "input/open-trusted-data-initiative\n", - "Saved contents of url: https://thealliance.ai/core-projects/open-trusted-data-initiative\n", - "Visited url: https://thealliance.ai/core-projects/safety-priorities-ranking-by-domain\n", - "input/safety-priorities-ranking-by-domain\n", - "Saved contents of url: https://thealliance.ai/core-projects/safety-priorities-ranking-by-domain\n", - "Visited url: https://thealliance.ai/core-projects/ntia_request\n", - "input/ntia_request\n", - "Saved contents of url: https://thealliance.ai/core-projects/ntia_request\n", - "input/aiconfig_editor\n", - "Saved contents of url: https://thealliance.ai/affiliated-projects/aiconfig_editor\n", - "input/ibm_granite\n", - "Saved contents of url: https://thealliance.ai/affiliated-projects/ibm_granite\n", - "input/the-living-guide-to-applying-ai\n", - "Saved contents of url: https://thealliance.ai/core-projects/the-living-guide-to-applying-ai\n", - "input/trusted-evals\n", - "Saved contents of url: https://thealliance.ai/core-projects/trusted-evals\n", - "input/sb1047\n", - "Saved contents of url: https://thealliance.ai/core-projects/sb1047\n", - "input/ai-accelerator-software-ecosystem-guide\n", - "Saved contents of url: https://thealliance.ai/core-projects/ai-accelerator-software-ecosystem-guide\n", - "input/trust-user-guide\n", - "Saved contents of url: https://thealliance.ai/core-projects/trust-user-guide\n", - "input/contact\n", - "Saved contents of url: https://thealliance.ai/contact\n", - "input/industry-open-fms-initiative\n", - "Saved contents of url: https://thealliance.ai/core-projects/industry-open-fms-initiative\n", - "input/become-a-collaborator\n", - "Saved contents of url: https://thealliance.ai/become-a-collaborator\n", - "input/affiliated-projects\n", - "input/advocacy\n", - "input/community\n", - "input/foundation-models-datasets\n", - "input/hardware-enablement\n", - "input/foundation-models\n", - "input/contribute\n", - "input/applications-and-tools\n", - "input/aia-members\n", - "input/about-aia\n", - "input/skills-education\n", - "input/trust-and-safety\n", - "input/blog\n", - "input/core-projects\n" + "url: https://thealliance.ai/\n", + "url: https://thealliance.ai/core-projects/ntia_request\n", + "url: https://thealliance.ai/contact\n", + "url: https://thealliance.ai/blog/open-source-ai-demo-night-sf-2024\n", + "url: https://thealliance.ai/core-projects/trusted-evals\n", + "url: https://thealliance.ai/core-projects/sb1047\n", + "url: https://thealliance.ai/focus-areas/hardware-enablement\n", + "url: https://thealliance.ai/focus-areas/applications-and-tools\n", + "url: https://thealliance.ai/focus-areas/trust-and-safety\n", + "url: https://thealliance.ai/community\n", + "url: https://thealliance.ai/focus-areas/advocacy\n", + "url: https://thealliance.ai/focus-areas/skills-education\n", + "url: https://thealliance.ai/contribute\n", + "url: https://thealliance.ai/become-a-collaborator\n", + "url: https://thealliance.ai/focus-areas/foundation-models-datasets\n", + "url: https://thealliance.ai/aia-members\n", + "url: https://thealliance.ai/our-work\n", + "url: https://thealliance.ai/about-aia\n", + "url: https://thealliance.ai/core-projects/the-living-guide-to-applying-ai\n", + "url: https://thealliance.ai/core-projects/trust-and-safety-evaluations\n", + "url: https://thealliance.ai/core-projects/safety-priorities-ranking-by-domain\n", + "url: https://thealliance.ai/core-projects/trust-user-guide\n", + "url: https://thealliance.ai/blog/ai-alliance-expands-with-seven-new-members-from-in\n", + "url: https://thealliance.ai/blog/ai-for-drug-discovery-open-innovation-forum\n", + "url: https://thealliance.ai/blog/unesco-language-translator\n", + "url: https://thealliance.ai/blog/advancing-domain-specific-qa-the-ai-alliances-guid\n", + "url: https://thealliance.ai/core-projects/ai-accelerator-software-ecosystem-guide\n", + "url: https://thealliance.ai/blog\n", + "url: https://thealliance.ai/blog/evaluation-of-generative-ai---whats-ultimately-our\n" ] }, { @@ -179,15 +144,19 @@ " \"\"\"\n", " global retrieved_pages, saved_pages\n", " retrieved_pages+=1\n", + " \n", + " print (f'on_downloaded (url={url})')\n", "\n", " if saved_pages < MY_CONFIG.CRAWL_MAX_DOWNLOADS:\n", " print(f\"Visited url: {url}\")\n", "\n", " # Get mime_type of retrieved page\n", " mime_type = get_mime_type(body)\n", + " print (f'mime_type={mime_type}')\n", " \n", " # Save the page if it is a PDF to only download research papers\n", " if MY_CONFIG.CRAWL_MIME_TYPE in mime_type.lower():\n", + " print ('hi')\n", " filename = get_filename_from_url(url)\n", " local_file_path = os.path.join(MY_CONFIG.INPUT_DIR, filename)\n", " print (local_file_path)\n", @@ -200,14 +169,21 @@ " saved_pages+=1\n", " \n", "# Define a user agent to provide information about the client making the request\n", - "user_agent = \"dpk-connector\"\n", + "# user_agent = \"dpk-connector\"\n", + "user_agent = \"Mozilla/5.0 (X11; Linux i686; rv:125.0) Gecko/20100101 Firefox/125.0\"\n", + "\n", + "\n", + "def on_downloaded2(url: str, body: bytes, headers: dict) -> None:\n", + " # print(f\"url: {url}, headers: {headers}, body: {body[:64]}\")\n", + " print(f\"url: {url}\")\n", "\n", "async def run_my_crawl():\n", " crawl([MY_CONFIG.CRAWL_URL_BASE], \n", - " on_downloaded, \n", + " on_downloaded2, \n", " user_agent=user_agent, \n", " depth_limit = MY_CONFIG.CRAWL_MAX_DEPTH, \n", - " path_focus = True, \n", + " subdomain_focus=True,\n", + " # path_focus = True, \n", " download_limit = MY_CONFIG.CRAWL_MAX_DOWNLOADS)\n", " return \"Crawl is done\"\n", "\n", @@ -218,7 +194,7 @@ ], "metadata": { "kernelspec": { - "display_name": "dpk-3-r022dev2-py311", + "display_name": "dpk-6-basic-022dev2-py311", "language": "python", "name": "python3" }, diff --git a/examples/notebooks/html-processing/2_process_html_python.ipynb b/examples/notebooks/html-processing/2_process_html_python.ipynb new file mode 100644 index 000000000..456e21272 --- /dev/null +++ b/examples/notebooks/html-processing/2_process_html_python.ipynb @@ -0,0 +1,123 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Processing HTML Files\n", + "\n", + "We will be using **html2parquet transform**\n", + "\n", + "References\n", + "- [html2parquet](https://github.com/IBM/data-prep-kit/tree/dev/transforms/language/html2parquet/python)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step-1: Data\n", + "\n", + "We will process data that is downloaded using [1_crawl_site.ipynb](1_crawl_site.ipynb).\n", + "\n", + "We have a couple of crawled HTML files in `input*` directory. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step-2: Configuration" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "## All config is defined here\n", + "from my_config import MY_CONFIG" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step-3: HTML2Parquet" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "ename": "AttributeError", + "evalue": "'Html2ParquetTransform' object has no attribute 'get_name'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m:20\u001b[0m\n", + "File \u001b[0;32m~/apps/anaconda3/envs/dpk-html-processing-py311/lib/python3.11/site-packages/data_processing/runtime/pure_python/transform_launcher.py:43\u001b[0m, in \u001b[0;36mPythonTransformLauncher.__init__\u001b[0;34m(self, runtime_config, data_access_factory)\u001b[0m\n\u001b[1;32m 33\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\n\u001b[1;32m 34\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 35\u001b[0m runtime_config: PythonTransformRuntimeConfiguration,\n\u001b[1;32m 36\u001b[0m data_access_factory: DataAccessFactoryBase \u001b[38;5;241m=\u001b[39m DataAccessFactory(),\n\u001b[1;32m 37\u001b[0m ):\n\u001b[1;32m 38\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 39\u001b[0m \u001b[38;5;124;03m Creates driver\u001b[39;00m\n\u001b[1;32m 40\u001b[0m \u001b[38;5;124;03m :param runtime_config: transform runtime factory\u001b[39;00m\n\u001b[1;32m 41\u001b[0m \u001b[38;5;124;03m :param data_access_factory: the factory to create DataAccess instances.\u001b[39;00m\n\u001b[1;32m 42\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m---> 43\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mruntime_config\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata_access_factory\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 44\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexecution_config \u001b[38;5;241m=\u001b[39m PythonTransformExecutionConfiguration(name\u001b[38;5;241m=\u001b[39mruntime_config\u001b[38;5;241m.\u001b[39mget_name())\n", + "File \u001b[0;32m~/apps/anaconda3/envs/dpk-html-processing-py311/lib/python3.11/site-packages/data_processing/runtime/transform_launcher.py:37\u001b[0m, in \u001b[0;36mAbstractTransformLauncher.__init__\u001b[0;34m(self, runtime_config, data_access_factory)\u001b[0m\n\u001b[1;32m 31\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 32\u001b[0m \u001b[38;5;124;03mCreates driver\u001b[39;00m\n\u001b[1;32m 33\u001b[0m \u001b[38;5;124;03m:param runtime_config: transform runtime factory\u001b[39;00m\n\u001b[1;32m 34\u001b[0m \u001b[38;5;124;03m:param data_access_factory: the factory to create DataAccess instances.\u001b[39;00m\n\u001b[1;32m 35\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 36\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mruntime_config \u001b[38;5;241m=\u001b[39m runtime_config\n\u001b[0;32m---> 37\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mname \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mruntime_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_name\u001b[49m()\n\u001b[1;32m 38\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdata_access_factory \u001b[38;5;241m=\u001b[39m data_access_factory\n", + "\u001b[0;31mAttributeError\u001b[0m: 'Html2ParquetTransform' object has no attribute 'get_name'" + ] + } + ], + "source": [ + "%%time \n", + "\n", + "import ast\n", + "import os\n", + "import sys\n", + "\n", + "from html2parquet_transform import Html2ParquetTransform, Html2ParquetTransformConfiguration\n", + "from data_processing.runtime.pure_python import PythonTransformLauncher\n", + "from data_processing.utils import GB, ParamsUtils\n", + "\n", + "\n", + "html2parquet_params = {}\n", + "\n", + "params = {\n", + " \"input_folder\": MY_CONFIG.INPUT_DIR,\n", + " \"output_folder\": MY_CONFIG.OUTPUT_DIR,\n", + " \"html2parquet_params\": html2parquet_params,\n", + "}\n", + "\n", + "sys.argv = ParamsUtils.dict_to_req(d=params)\n", + "# create launcher\n", + "launcher = PythonTransformLauncher(Html2ParquetTransform(html2parquet_params))\n", + "\n", + "return_code = launcher.launch()\n", + "\n", + "if return_code == 0:\n", + " print (f\"✅ Stage:{STAGE} completed successfully\")\n", + "else:\n", + " raise Exception (\"❌ Job failed\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dpk-html-processing-py311", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/notebooks/html-processing/README.md b/examples/notebooks/html-processing/README.md index 425689790..a9ff5dc93 100644 --- a/examples/notebooks/html-processing/README.md +++ b/examples/notebooks/html-processing/README.md @@ -7,15 +7,16 @@ See [project README](../../../README.md#create-a-virtual-environment). Here is a quick setup guide ```bash -conda create -n dpk-r022dev2-py311 python=3.11 -conda activate dpk-r022dev2-py311 +conda create -n dpk-html-processing-py311 python=3.11 + +conda activate dpk-html-processing-py311 ``` If on linux do this ```bash -conda install gcc_linux-64 -conda install gxx_linux-64 +conda install -y gcc_linux-64 +conda install -y gxx_linux-64 ``` Install modules @@ -25,4 +26,10 @@ pip install -r requirements.txt ``` -## Download HTML pages \ No newline at end of file +## Download HTML pages + +TODO: 1_crawl_site.ipynb + +Shortcut : Using wget + +`wget --recursive --level=1 --adjust-extension https://thealliance.ai/` diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/about-aia.html b/examples/notebooks/html-processing/input2/thealliance.ai/about-aia.html new file mode 100644 index 000000000..4249f5693 --- /dev/null +++ b/examples/notebooks/html-processing/input2/thealliance.ai/about-aia.html @@ -0,0 +1,25 @@ +AI Alliance

About AI Alliance

Developing AI collaboratively, transparently, and with a focus on safety, ethics, and the greater good

The AI Alliance is focused on fostering an open community and enabling developers and researchers to accelerate responsible innovation in AI while ensuring scientific rigor, trust, safety, security, diversity and economic competitiveness. 

We bring together a critical mass of compute, data, tools, and talent to accelerate and advocate for open innovation in AI.

Meet our members

Frequently asked questions

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/aia-members.html b/examples/notebooks/html-processing/input2/thealliance.ai/aia-members.html new file mode 100644 index 000000000..c61ef9b27 --- /dev/null +++ b/examples/notebooks/html-processing/input2/thealliance.ai/aia-members.html @@ -0,0 +1,25 @@ +About | AI Alliance

Meet the AI Alliance

The AI Alliance is a collaborative network of companies, startups, universities, research institutions, government organizations, and non-profit foundations that are working at the forefront of AI technology, applications, and governance.

We unite top developers, scientists, academic institutions, companies, and policymakers to create the future of open, safe, and responsible AI.

Become a contributor
IBM logo
Meta logo
AMD logo
anyoneai
Core 42 logo
Databricks logo
Esade logo
FPT Software
Intel logo
KissanAI logo
MediaTek
University of Notre Dame logo
People+ai logo
Red Hat logo
Sony logo
St. Johns University
Tokyo Electron Ltd. logo
Uber logo
Wadhwani AI
Zilliz logo
\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/become-a-collaborator.html b/examples/notebooks/html-processing/input2/thealliance.ai/become-a-collaborator.html new file mode 100644 index 000000000..7d0e60bcd --- /dev/null +++ b/examples/notebooks/html-processing/input2/thealliance.ai/become-a-collaborator.html @@ -0,0 +1,25 @@ +AI Alliance

Become a collaborator

Want to get involved? Apply to join one of our working groups or major initiatives.

Our team will review your request and get back to you as soon as possible (within 30 days or fewer).

Working group application

Select which working groups or major initiatives you are interested in (select one ore more)*
Interests and contributions*

The AI Alliance is committed to protecting and respecting your privacy, and we’ll only use your personal information to contact you upon submission of this form.

By submitting the preceding form, you consent to allow IBM to store and process the personal information submitted to keep you informed about AI Alliance.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/blog.html b/examples/notebooks/html-processing/input2/thealliance.ai/blog.html new file mode 100644 index 000000000..5e883cd71 --- /dev/null +++ b/examples/notebooks/html-processing/input2/thealliance.ai/blog.html @@ -0,0 +1,25 @@ +AI Alliance

Blog & Articles

Perspectives, news, and technical reports from our community.

Articles

Spotlight on Raphaël Vienne of datacraft

In this AI Alliance member spotlight we meet Raphaël Vienne, Head of AI at datacraft.

Spotlight on Roberto Trotta of SISSA

In this AI Alliance member spotlight we meet Roberto Trotta, Professor of Theoretical Physics and Head of Data Science at SISSA, Trieste

UNESCO Language Translator, powered by Meta and Hugging Face

News

UNESCO, Meta, and Hugging Face launch an online Language Translator as part of United Nations General Assembly week, when leaders from around the world meet to discuss the world’s most pressing challenges.

Screenshot BU website reg page

AI for Drug Discovery Open Innovation Forum

News

Please join us on Wednesday, October 30, 2024 at Boston University’s Center for Computing & Data Sciences for the AI for Drug Discovery Open Innovation Forum.

Public Statement from the AI Alliance on the Possible Veto of SB 1047

Public Statement from the AI Alliance on the Possible Veto of SB 1047

News

While we support thoughtful AI regulation, SB 1047 does not strike the right balance. A
veto would allow for a more balanced approach that safeguards innovation while
addressing safety.

abstract gradient

AI Alliance Expands with Seven New Members from India

News

The AI Alliance, a global community dedicated to advancing safe and responsible AI through open innovation, has expanded its membership to include seven leading Indian organizations: AI4Bharat (IIT Madras), IIT Jodhpur, Infosys, KissanAI, People+AI, Sarvam AI, and, Wadhwani AI. This move reinforces India's growing contribution to the global AI landscape and marks a significant expansion of the Alliance. These companies are joined by IIT Bombay as one of the founding members of the Alliance.

Open Source AI Demo Night

News

The AI Alliance, in collaboration with Cerebral Valley and Ollama, hosted Open Source AI Demo Night in San Francisco, bringing together more than 200+ developers and innovators to showcase and celebrate the latest advances in open-source AI.

Spotlight on Sean Hughes of ServiceNow

In this AI Alliance member spotlight we meet Sean Hughes, AI Ecosystem Director at ServiceNow.

Spotlight on Zhuo Li of HydroX AI

Member spotlight

In this AI Alliance member spotlight we meet Zhuo Li, Founder & CEO of Hydrox AI.

Statement from AI Alliance Regarding Commerce's Final Report on Open-Weights Models

News

Commerce Department has taken a crucial step towards ensuring that the U.S. remains at the forefront of AI development by concluding that "the government should not restrict the wide availability of model weights for dual-use foundation models."

abstract office

What we're doing and why: an introduction to The AI Alliance skills and education working group

Education and skill-building is a key priority of the AI Alliance. Already, leaders are sounding alarm about an AI skills crisis, with too few people having the skills needed to work in the field, and too few opportunities for diverse populations to gain the training and experiences they need at scale, and economies standing to lose trillions.

Screen shot of the User Guide website.

Getting started with AI trust and safety

Technical Report

The AI trust and safety guide is a “living” document that provides an introduction to current trends in research and development for ensuring AI models and applications meet requirements for trustworthy results, and in particular, results that satisfy various safety criteria. Aimed at developers and leaders who are relatively new to this topic, the guide defines some common terms, provides an overview of several leading trust and safety education and technology projects, and offers recommendations for how to build-in trust and safety into your AI-based applications.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/community.html b/examples/notebooks/html-processing/input2/thealliance.ai/community.html new file mode 100644 index 000000000..347e728bf --- /dev/null +++ b/examples/notebooks/html-processing/input2/thealliance.ai/community.html @@ -0,0 +1,25 @@ +Community | AI Alliance

Join the Community

Join leading AI innovators, researchers, developers, engineers, and early enterprise adopters in our goal to advance open, safe, and responsible AI. Follow the links on this page to subscribe to our community newsletter, contribute to a working group, join The Alliance, follow us on social media, or attend one of our upcoming events.

Subscribe to our community newsletter

Get updates about everything that’s happening in the community and be the first to know about new projects, new resources, upcoming events, opportunities to collaborate and contribute, etc.

To subscribe, please fill out the newsletter sign-up form in the footer below.

Contribute code

We strive to provide a clear and simple path for anyone to join our community and contribute to AI Alliance projects, whether they are part of an existing member organization, or not. Learn more...

Become a collaborator in one of our working groups

The heartbeat of the AI Alliance is our working groups and major initiatives! Since launch, they have rolled out core projects and affiliated projects – with more to come:

  • Skills, Education, and Exploratory Research WG
  • AI Trust and Safety WG
  • Application and Tools WG
  • Diverse Hardware Enablement WG
  • Foundation Models WG
  • Policy WG

Our major initiatives currently include:

  • Open Trusted Data Initiative
  • Industry Open FMs Initiative
  • Time Series Initiative (data, benchmarks, benchmarking methods)
  • Domain Specific Trust and Safety Initiative

To join one or more, please request an invite. Thank you!

Join The Alliance

If your organization would like to join the AI Alliance as an official member, please use the contact form, and someone on our membership team will get back to you. Thanks!

Attend an AI Alliance event

Follow us on social media

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/contact.html b/examples/notebooks/html-processing/input2/thealliance.ai/contact.html new file mode 100644 index 000000000..4909ec128 --- /dev/null +++ b/examples/notebooks/html-processing/input2/thealliance.ai/contact.html @@ -0,0 +1,25 @@ +AI Alliance

Contact us

We’re here to help and answer any questions you may have regarding the AI Alliance. We look forward to hearing from you!

Please ensure to include enough detail so we can route your inquiry to the right team member. Thanks!

Keep in touch

The AI Alliance is committed to protecting and respecting your privacy, and we’ll only use your personal information to contact you upon submission of this form.

By submitting the preceding form, you consent to allow IBM to store and process the personal information submitted to keep you informed about AI Alliance.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/contribute.html b/examples/notebooks/html-processing/input2/thealliance.ai/contribute.html new file mode 100644 index 000000000..4208d2d2c --- /dev/null +++ b/examples/notebooks/html-processing/input2/thealliance.ai/contribute.html @@ -0,0 +1,25 @@ +AI Alliance

Join us in building the open future of AI

We strive to provide a clear and simple path for anyone to join our community and contribute to AI Alliance projects, whether they are part of an existing member organization, or not.

We are excited to welcome contributions to AI Alliance Core and Affiliated projects.

Core Projects
Core projects address substantial cross-community challenges and are an opportunity for individual contributors and members to collaborate, build, and make an impact on the future of AI. Core Projects are managed directly by the AI Alliance and governed as described below. You can find a list of AI Alliance Core Projects here on the AI Alliance website. 

Learn about Core Projects

Affiliated Projects
We encourage individuals to contribute to Affiliated projects run by an AI Alliance member.  Although Affiliated Projects are not managed by the AI Alliance, we highlight them as contributing to open, safe, and responsible AI. You can find a list of AI Alliance Affiliated Projects here on the AI Alliance website. Contribution to Affiliated Projects is governed as set out by the Affiliate Project, not the AI Alliance contribution process set out below.

Learn about Affiliated Projects

Making a contribution

The AI Alliance utilizes the Linux Foundation’s Developer Certificate of Origin (DCO) on a per-commit basis to enable contributions to Core Projects. The DCO does not require the committer to give away ownership of the contribution - your IP remains yours and others' IP remains theirs.

Core Project Licenses

AI Alliance Core Projects’ utilize the following licenses unless otherwise noted in the project repository:

The AI Alliance leaves open the possibility of additional terms concerning safe and responsible use for certain elements in special Core Projects, for example weights associated with certain AI models. Any decision to use any such additional terms for a Core Project must be made by the AI Alliance Steering Committee and will be clearly identified in the Core Project repository.

Anyone participating in and/or contributing to the AI Alliance community must agree to the following agreements and conditions: 

  1. AI Alliance Code of Conduct - We strive for a vibrant, diverse, and collegial community.
  2. This Developer Certificate of Origin applies to all contributions to AI Alliance Core Projects. 
  3. AI Alliance Competition Law Guidelines - The AI Alliance and its members are committed to compliance with applicable antitrust and competition laws.
Check out the AI Alliance Github for more information and ways to contribute
\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/core-projects/ntia_request.html b/examples/notebooks/html-processing/input2/thealliance.ai/core-projects/ntia_request.html new file mode 100644 index 000000000..77a5985dd --- /dev/null +++ b/examples/notebooks/html-processing/input2/thealliance.ai/core-projects/ntia_request.html @@ -0,0 +1,25 @@ +Responding to the U.S. NTIA request for comment on Dual Use Foundation Artificial Intelligence Models with Widely Available Model Weights | AI Alliance

Responding to the U.S. NTIA request for comment on Dual Use Foundation Artificial Intelligence Models with Widely Available Model Weights

Project
abstract gradient

The AI Alliance recently drafted a response to the NTIA’s request for comment on “Dual Use Foundation Artificial Intelligence Models with Widely Available Model Weights.” The request seeks public input on the potential risks, benefits, and policy approaches for AI foundation models whose weights are broadly accessible.

This initiative follows an Executive Order from President Biden on AI development and use. The public comment period is significant as it allows stakeholders to influence the policy framework, ensuring that these models are developed and used safely and ethically.  

Download our response
\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/core-projects/sb1047.html b/examples/notebooks/html-processing/input2/thealliance.ai/core-projects/sb1047.html new file mode 100644 index 000000000..89c80b88b --- /dev/null +++ b/examples/notebooks/html-processing/input2/thealliance.ai/core-projects/sb1047.html @@ -0,0 +1,25 @@ +AI Alliance Statement in Opposition to California SB 1047 | AI Alliance

A statement in opposition to California SB 1047

Project
abstract gradient

The AI Alliance, a community of technology creators, developers, and adopters collaborating to advance safe, responsible AI rooted in open innovation, respectfully opposes SB 1047, the proposed Safe and Secure Innovation for Frontier Artificial Intelligence Models Act. It would slow innovation, thwart advancements in safety and security, and undermine California’s economic growth. The bill’s technically infeasible requirements will chill innovation in the field of AI and lower access to the field’s cutting edge, thereby directly contradicting the bill’s stated purpose “...to ensure that artificial intelligence innovation . . . is accessible to academic researchers and startups, in addition to large companies.” Sec. 2(d).


The AI Alliance is a community of researchers, developers, and organizational leaders committed to supporting and enhancing open innovation across the AI technology landscape. Many members of the AI Alliance are based in California or have a nexus to the state or its residents⏤and we are deeply concerned about the potential establishment of an anti-open-source precedent. The Alliance enables developers and researchers to accelerate responsible open innovation in AI while ensuring scientific rigor, trust, safety, security, diversity, and economic competitiveness. By bringing together this community, the AI Alliance pools resources, knowledge, and best practices to address safety concerns and provides a platform for sharing and developing solutions that benefit the whole AI ecosystem.

Examples of AI Alliance contributions. AI Alliance members contribute to and rely on open AI models, with uses demonstrating the ways open models enhance innovation, economic productivity, safety, and security, while broadening access to AI. IBM, in partnership with NASA, has democratized the largest geospatial AI foundation model on Hugging Face. Stability AI is developing innovative, resource-efficient, open-access AI models. Snowflake is empowering businesses to build potent AI and machine learning applications from its enterprise data. SeedAI is crafting frameworks for co-creation, evaluation, and decision-making regarding AI technology and policies. Meta launched Llama 3, which has already been downloaded over 1 million times; over 600 derivative models have been uploaded to Hugging Face. Databricks is democratizing AI within the enterprise space, by helping thousands of organizations customize and deploy DBRX, Llama 3, and other opensource AI models for enterprise applications. ServiceNow, in collaboration with Hugging Face and the BigCode community of more than 1,200 members from 60+ countries, released its StarCoder family of models with the aim of introducing more responsibility and transparency in the development of large language models for code. AI Alliance member universities globally are leveraging open-source AI for scientific discovery, research, and innovation.

Open-source approach is safe and secure. If the bill does aim to promote “Safe and Secure Innovation for Frontier Artificial Intelligence Models,” it should facilitate an AI ecosystem that provides for the full range of options regarding form of release. Historically, open source has been a more secure approach to technology development. This remains true. The availability of model weights, for example, enables the identification and examination of model vulnerabilities, driving security enhancements for both closed and open models. Open models facilitate cross-examination and the detection of potential improvements or bugs, fostering a collaborative development environment. This perspective is shared by twenty-five leading AI experts from industry, academia, and civil society. And this sentiment is shared by over 1,200 individuals across the AI ecosystem, who, in a Joint Statement on AI Safety and Openness, declared that openness and transparency are fundamental to achieving safety, security, and accountability in AI.

The bill severely penalizes responsible open-source development. While SB 1047 is not targeting open-source development specifically, it will affect the open-source community dramatically. The bill requires developers of AI models of 10^26 FLOPS or similar performance (as determined by undefined benchmarks) to implement a full shutdown control that would halt operation of the model and all derivative models. Once a model is open sourced and subsequently downloaded by a third party, by design developers no longer have control over a model. Before such a “shutdown switch” provision is passed, we need to understand how it can be done in the context of open source; the bill does not answer that question. No models at 10^26 FLOPS are openly available today but technology is rapidly advancing and the open ecosystem could evolve alongside it. However, this legislation seems intended to freeze open-source AI development at the 2024 level.

Current exemptions are not practicable. The bill does attempt to provide some reasonable requirements for models subject to a “limited duty exemption,” including an exception to the shutdown requirement. However, few⏤if any⏤models would ever qualify for this exemption because it includes technically impracticable obligations, such as requiring developers to determine whether the exemption applies prior to training. Further, the exception is only available if the model does not enable a “hazardous capability,” but a model is considered to have a hazardous capability “even if the hazardous capability [at issue] would not manifest but for fine tuning and post training modifications performed by third-party experts intending to demonstrate those abilities.”4 As fine tuning and post training modifications could introduce almost anything to the model, this requirement would be nearly impossible to meet.

The bill needs a number of changes to make it effective and practicable. This bill will also affect smaller companies, which often rely on open-source models created by larger companies. Compliance costs may discourage open sourcing, depriving smaller companies of the models they need to build their businesses. Therefore, we recommend refocusing regulatory attention from model development to model applications (i.e., how the model is used), particularly those applications that pose high risks to public safety and security.

However, assuming the bill retains the current structure, we recommend the following changes:

  1. Amending the requirements for the limited duty exemption to make it feasible for responsible open-source developers;
  2. When a limited duty exemption does not apply, amending the obligations on developers to limit scope to instances where developers have control. Without this limitation, developers will be liable for all kinds of downstream uses that will make open sourcing models unviable;
  3. Incorporating discrete open-source exceptions throughout the bill to ensure that responsible developers can open-source models with adequate mitigations.

Download the statement
\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/core-projects/trusted-evals.html b/examples/notebooks/html-processing/input2/thealliance.ai/core-projects/trusted-evals.html new file mode 100644 index 000000000..5c0d9b8a0 --- /dev/null +++ b/examples/notebooks/html-processing/input2/thealliance.ai/core-projects/trusted-evals.html @@ -0,0 +1,25 @@ +Trusted evals request for proposals | AI Alliance

Trusted evals request for proposals

Project
abstract gradient

Context

Model evaluation, or ‘evals’, in the generative AI era is simultaneously one of the most important areas of investment but also one of the highest areas of entropy.

Our goals for the Trusted Evals working group are to:

  1. Raise awareness for community efforts around trust and safety – including, but not limited to, work happening globally in various languages and in various domains (e.g. cyber security, CBRNE, etc.). Foster and grow the academic and technical communities in Trustworthy AI. And by extension create a center of mass for domain experts that can help us push beyond where evals stand today and into areas where we don’t have good visibility; and
  2. Drive the development of comprehensive, reliable, and stable tools for model evaluation, where these tools provide repeatable, reproducible, and diverse results, and where these tools are regularly refreshed and ever evolving as we learn about new risks for generative AI and other evaluation concerns. This means, as a community, creating new benchmarks and metrics to address quality, safety, robustness and performance aspects of generative AI models. The approach is to be as broad and diverse as possible with the goal of uncovering new evals in as many domains as possible so we can learn as a community. The goal is NOT to create a standard for model evaluations, but to work closely with MLCommons to help shepherd a subset of these evals into their standardization effort.

About

Generally speaking, evals shine a light on model capabilities (we love to evaluate how well our models can reason) but they also can expose where models could present certain risks of harm. One of the major challenges as evals become more esoteric is that access is limited for experts in various fields that also intersect in GenAI. For example, at Meta there is a team of cyber security experts who also build models for coding and productivity and, in parallel, are building safeguards for things like malicious code generation. For many of CBRNE risks, these experts don’t have a central place to aggregate nor are they working with generative AI experts. Further to this, there isn’t a de facto place today for the open community to evaluate your models across a growing number of potential harms.

Request for proposal

We are seeking new perspectives in the AI evaluation domain. To foster further innovation in this area, we are pleased to invite the community to participate in the AI Alliance’s Trusted Evals working group by submitting a response to our Request for Proposal to be included in the working group’s efforts to:

  1. Raise awareness - selected proposals will be showcased through AI Alliance communications, including our newsletter, blog, whitepapers, and website; and
  2. Drive the development of comprehensive, reliable, and stable tools - The AI Alliance intends to support select project proposals with resources to help teams accelerate progress in building the foundations of safe, trusted AI.

For this RFP, we are excited to work with those in academia, big industry, startups and anyone excited to collaborate in the open and build an ecosystem around their work.

Areas of interest

  1. Cybersecurity threats
  2. Sensitive data detection including areas such as toxic content (e.g. hate speech), personally identifiable information (PII), bias, etc.
  3. Model performance including helpfulness, quality, alignment, robustness, etc., (as opposed to operational concerns like throughput, latency, scalability, etc.)
  4. Knowledge and factuality
  5. Multilingual evaluation
  6. Mediation
  7. Balancing harms and helpfulness
  8. Personal data memorization / data governance
  9. Vertical domains such as legal, financial, medical
  10. Areas related to CBRNE (chemical, biological, radiological, nuclear, and high yield explosives)
  11. Weapons acquisition specifically
  12. Measuring data set integrity when data is created by AI: label fairness, prompt generation for RLHF
  13. Effectiveness of tool use that exacerbates malicious intent
  14. Demographic representation across different countries
  15. Distributionbias
  16. Politicalbias
  17. Capability fairness
  18. Undesirable use cases
  19. Regional bias, discrimination
  20. Violence and hate
  21. Terrorism and sanctioned individuals
  22. Defamation
  23. Misinformation
  24. Guns,illegal weapons, controlled substances

Requirements

Proposals should include a link to a summary of your project, in English, explaining the area of focus, dataset description and any relevant prior work. This includes:

  • Name of project and abstract
  • Core team member bios & affiliations
  • Link to website (if applicable)
  • Link to GitHub repository
  • License
  • Links to whitepapers or publications
Submit your proposal
\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/advocacy.html b/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/advocacy.html new file mode 100644 index 000000000..7da87a218 --- /dev/null +++ b/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/advocacy.html @@ -0,0 +1,25 @@ +Advocacy | AI Alliance

Advocacy

Supporting regulatory policies that create healthy, sustainable, and open ecosystems for AI.

A thriving and competitive open innovation ecosystem for AI must be a priority for industry, civil society, and academia — policymakers should take note. Though these ecosystems are largely decentralized and self-directed, there's room for policy to aid their growth. 
    

Our work will help policymakers and governments recognize and support open innovation ecosystems for AI. We’re also bridging gaps between policy and industry, fostering responsible and ethical AI practices to benefit societies around the world.   

Working Groups

Advocacy

Open Source AI Demo Night

The Advocacy Working Group supports regulatory policies that create healthy, sustainable, and open ecosystems for AI. It works on projects like Open Source AI Demo Night, which highlighted entrepreneurs using open source AI to innovate.

Our projects

Statement from AI Alliance Regarding Commerce's Final Report on Open-Weights Models

News

Commerce Department has taken a crucial step towards ensuring that the U.S. remains at the forefront of AI development by concluding that "the government should not restrict the wide availability of model weights for dual-use foundation models."

abstract gradient

A statement in opposition to California SB 1047

Advocacy

Our perspectives and recommendations in opposition to California SB 1047, the proposed Safe and Secure Innovation for Frontier Artificial Intelligence Models Act.

abstract gradient

Responding to the U.S. NTIA request for comment on Dual Use Foundation Artificial Intelligence Models with Widely Available Model Weights

Advocacy

The request seeks public input on the potential risks, benefits, and policy approaches for AI foundation models whose weights are broadly accessible.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/applications-and-tools.html b/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/applications-and-tools.html new file mode 100644 index 000000000..eadb27fed --- /dev/null +++ b/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/applications-and-tools.html @@ -0,0 +1,25 @@ +Tools | AI Alliance

Applications and Tools

Building the most capable tools for AI model builders and GenAI application developers.

We build and promote open-source tools for model training, tuning, and inference. We are also collaborating to simplify, automate, and optimize the deployment and execution of large-scale AI workloads on Kubernetes. 

Our work

abstract gradient

Advancing Domain-Specific Q&A: The AI Alliance's Guide to Best Practices

Technical Report

The AI Alliance application and tools working group has conducted a comprehensive study on best practices for advancing domain-specific Q&A using retrieval-augmented generation (RAG) techniques. The findings of this research, provide insights and recommendations for maximizing the capabilities of Q&A AI in specialized domains.

The Living Guide to Applying AI

Applications and Tools

A "living guide" for building AI-enabled applications, this guide provides an introduction to several established design patterns for building AI systems and products, with contributions from different experts in the AI Alliance. Several common patterns, like RAG (retrieval-augmented generation), are explored from different angles, and emerging patterns, like GraphRAG (using a graph network as a source of RAG content) and agents.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/foundation-models-datasets.html b/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/foundation-models-datasets.html new file mode 100644 index 000000000..8f107d89b --- /dev/null +++ b/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/foundation-models-datasets.html @@ -0,0 +1,25 @@ +Foundation Models | AI Alliance

Open Foundation Models and Datasets

Enabling an ecosystem of open foundation models, including those with multilingual and multi-modal capabilities, and open datasets.

We are responsibly enhancing the ecosystem of open foundation models and datasets. We are embracing multilingual and multimodal models, as well as science models tackling broad societal issues like climate change and education.

To aid AI model builders and application developers, we’re collaborating to develop and promote open-source tools for model training, tuning, and inference. We are also launching programs to foster the open development of AI in safe and beneficial ways, and hosting events to explore AI use cases.

Without good datasets, model training and tuning would be impossible. We are promoting the development of open datasets with clear governance and provenance controls so they can be used without concerns for legal and other risks.

Working groups

AI for Drug Discovery

This working group aims to create a world-class research community that harnesses the potential of AI foundation models, transforms the field of drug discovery, and accelerates scientific progress by driving interdisciplinary collaboration on AI-powered drug discovery projects in the open.

Foundation Models

abstract gradient

Materials and Chemistry

We aim to curate datasets, tasks and benchmarks for materials science, build out foundation models in chemistry for prediction of properties, experimental outcomes or generation of new candidates and create a framework to foster collaboration between human experts and AI agents that will ultimately help solve global urgent challenges in sustainability and safety of materials.

Our work

These are the projects we are currently working on.

Home page for the Open Trusted Data Initiative

Open Trusted Data Initiative

Foundation Models and Datasets

Cataloging and managing trustworthy datasets.

abstract gradient

Time Series Data and Model Initiative

Foundation Models and Datasets

Time-series applications are an important target for AI. In addition to gathering high-quality and fully-governed time series datasets as part of the Open Trusted Data Initiative, Alliance members are collaborating on new and improved time series models (as part of the Industry Open FMs Initiative and benchmarks, both general-purpose and application-specific.

Please join us. We need time series and domain experts, including especially subject matter experts and use case and product owners who would like to apply emerging time series foundation models to new applications. There is an acute shortage of good, open datasets for time series and data specially benchmarks and evaluation methods for various use cases. Contributions are especially welcome here.

abstract gradient

Industry Open FMs Initiative

Foundation Models and Datasets

We have seen rapid progress in building and releasing highly-capable and open foundation models for general language, coding, scientific discovery, and multi-modal scenarios.

A key development in model strategies is a focus on building smaller, more specialized models.

More details are coming soon, but we would love for you to join us. We need both model-building and domain experts, including those outside the target domains listed above.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/hardware-enablement.html b/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/hardware-enablement.html new file mode 100644 index 000000000..93d4e8801 --- /dev/null +++ b/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/hardware-enablement.html @@ -0,0 +1,25 @@ +Hardware | AI Alliance

Hardware Enablement

Fostering a vibrant AI hardware accelerator ecosystem through enabling software technology.

We collaborate on the benchmarking, optimization, and adaptation of AI workloads to accelerate innovation in a diverse set of hardware. Our work focuses on scalability, platform adaptation, and advanced power, energy, and carbon modeling. Benchmarks and metrics developed for model inference, fine-tuning, and energy consumption of large-scale AI workloads will be contributed to the open-source community. 

Our Work

These are the projects we are currently working on.

Screen shot of the AI Accelerator Software Ecosystem Guide

AI Accelerator Software Ecosystem Guide

Hardware Enablement

The AI Accelerator Software Ecosystem Guide provides an introduction to the broad topic of software tools that support seamless execution of AI workloads on different hardware accelerators. Industry experts provide guidance on the state of the art and how to ensure success.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/skills-education.html b/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/skills-education.html new file mode 100644 index 000000000..8fdb0109e --- /dev/null +++ b/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/skills-education.html @@ -0,0 +1,25 @@ +Skills & Education | AI Alliance

Skills & Education

Supporting global AI skill-building, education, and exploratory research.

Our members partner with academics, industry leaders, researchers, and students, empowering them to develop and leverage resources to build an AI-ready workforce and engage with vital research in AI models, algorithms, and platforms.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/trust-and-safety.html b/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/trust-and-safety.html new file mode 100644 index 000000000..849aeec58 --- /dev/null +++ b/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/trust-and-safety.html @@ -0,0 +1,25 @@ +Trust | AI Alliance

Trust and Safety

Ensuring safe and trusted generative AI with benchmarks, tools, and methodologies

Our members work together to understand the landscape of AI trust and safety risks, as well as other uses of AI system evaluation. We identify use cases specific to various important domains (finance, healthcare, education, and others). We work together and with external collaborators to build tools, refine methods, and create benchmarks for detecting and mitigating those risks and performing other kinds of evaluation. We also help educate the public about responsible AI and the developer community about responsible model and application development. 

Our work

These are the projects we are currently working on.

Understanding AI Trust and Safety: A Living Guide

A major challenge for the successful use of AI is the importance of understanding potential trust and safety issues, along with their mitigation strategies. Failure to consider these issues could impact an organization's operations and the experience of its users. Concerns about safety are also a driver for current regulatory initiatives. Hence, applications built with AI must be designed and implemented with AI trust and safety in mind. This guide provides an introduction to trust and safety concerns, and offers guidance for AI projects.

Ranking AI Safety Priorities by Domain

Trust & Safety

A challenge for software development teams adopting generative AI is making sense of the safety issues that their applications must address. The AI safety ecosystem is broad and growing quickly, making it difficult for these development teams to know where they should focus their efforts. What safety concerns are most important for them to work on first?

Trust and Safety Evaluations

Trust & Safety

Much like other software, generative AI (“GenAI”) models and the AI systems that use them need to be trusted and useful to their users. The Trust and Safety Evaluations project fills gaps in the current landscape of the taxonomy of different kinds of evaluation, the tools for creating and running evaluations, and leaderboards to address particular categories of user needs.

abstract gradient

Trusted evals request for proposals

Trust & Safety

The AI Alliance Trusted Evals request for proposals is aimed at seeking new perspectives in the AI evaluation domain. We are excited to work with those in academia, industry, startups and anyone excited to collaborate in the open and build an ecosystem around their work.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/index.html b/examples/notebooks/html-processing/input2/thealliance.ai/index.html new file mode 100644 index 000000000..fd6a8217d --- /dev/null +++ b/examples/notebooks/html-processing/input2/thealliance.ai/index.html @@ -0,0 +1,25 @@ +Home | AI Alliance

Building the open future of AI

We are technology developers, researchers, industry leaders and advocates who collaborate to advance safe, responsible AI rooted in open innovation.

Advocacy

abstract gradient

Supporting regulatory policies that create healthy, sustainable, and open ecosystems for AI.

Trust & Safety

abstract image

Creating benchmarks, tools, and methodologies to evaluate and ensure safe, trusted generative AI.

Applications and Tools

abstract gradient

Building the most capable tools for AI model builders and GenAI application developers.

Foundation Models and Datasets

Foundation Models and Datasets

Enabling an ecosystem of open foundation models, including those with multilingual, multi-modal, science, and time series capabilities. Similarly, promote the development of datasets in these areas.

Hardware Enablement

abstract gradient

Fostering a vibrant AI hardware accelerator ecosystem through enabling software technology.

Skills & Education

abstract gradient

Supporting global AI skill-building, education, and exploratory research.

News & Projects

Open Source AI Demo Night

News

The AI Alliance, in collaboration with Cerebral Valley and Ollama, hosted Open Source AI Demo Night in San Francisco, bringing together more than 200+ developers and innovators to showcase and celebrate the latest advances in open-source AI.

abstract gradient

A statement in opposition to California SB 1047

Advocacy

Our perspectives and recommendations in opposition to California SB 1047, the proposed Safe and Secure Innovation for Frontier Artificial Intelligence Models Act.

abstract gradient

Trusted evals request for proposals

Trust & Safety

The AI Alliance Trusted Evals request for proposals is aimed at seeking new perspectives in the AI evaluation domain. We are excited to work with those in academia, industry, startups and anyone excited to collaborate in the open and build an ecosystem around their work.

abstract gradient

Responding to the U.S. NTIA request for comment on Dual Use Foundation Artificial Intelligence Models with Widely Available Model Weights

Advocacy

The request seeks public input on the potential risks, benefits, and policy approaches for AI foundation models whose weights are broadly accessible.

  • IBM logo
  • Meta logo
  • AMD logo
  • Core 42 logo
  • Databricks logo
  • Esade logo
  • FPT Software
  • Intel logo
  • Red Hat logo
  • Sony logo
  • Uber logo
  • University of Notre Dame logo
  • Zilliz logo

We build, enable, and advocate for open innovation across the AI technology landscape, including software, data and models, safety, security and trust, tooling, evaluation, hardware, education, open science, and advocacy.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/our-work.html b/examples/notebooks/html-processing/input2/thealliance.ai/our-work.html new file mode 100644 index 000000000..fd06a0f8a --- /dev/null +++ b/examples/notebooks/html-processing/input2/thealliance.ai/our-work.html @@ -0,0 +1,25 @@ +Our Work | AI Alliance

Accelerating responsible innovation in AI

Open and transparent innovation is essential for equipping AI researchers, developers, and users with the knowledge and tools to leverage AI advancements safely and inclusively, prioritizing diversity and economic opportunity for all. 

Through member-driven working groups, we bring together builders and experts from various fields to collaboratively and transparently address the challenges of generative AI and democratize its benefits.

Contribute to an AI Alliance project

Core projects

View all

Core projects address substantial cross-community challenges and are an opportunity for individual contributors and members to collaborate, build, and make an impact on the future of AI. Core Projects are managed directly by the AI Alliance and governed as described below.

abstract gradient

A statement in opposition to California SB 1047

Advocacy

Our perspectives and recommendations in opposition to California SB 1047, the proposed Safe and Secure Innovation for Frontier Artificial Intelligence Models Act.

abstract gradient

Trusted evals request for proposals

Trust & Safety

The AI Alliance Trusted Evals request for proposals is aimed at seeking new perspectives in the AI evaluation domain. We are excited to work with those in academia, industry, startups and anyone excited to collaborate in the open and build an ecosystem around their work.

abstract gradient

Responding to the U.S. NTIA request for comment on Dual Use Foundation Artificial Intelligence Models with Widely Available Model Weights

Advocacy

The request seeks public input on the potential risks, benefits, and policy approaches for AI foundation models whose weights are broadly accessible.

Ranking AI Safety Priorities by Domain

Trust & Safety

A challenge for software development teams adopting generative AI is making sense of the safety issues that their applications must address. The AI safety ecosystem is broad and growing quickly, making it difficult for these development teams to know where they should focus their efforts. What safety concerns are most important for them to work on first?

Home page for the Open Trusted Data Initiative

Open Trusted Data Initiative

Foundation Models and Datasets

Cataloging and managing trustworthy datasets.

abstract gradient

Time Series Data and Model Initiative

Foundation Models and Datasets

Time-series applications are an important target for AI. In addition to gathering high-quality and fully-governed time series datasets as part of the Open Trusted Data Initiative, Alliance members are collaborating on new and improved time series models (as part of the Industry Open FMs Initiative and benchmarks, both general-purpose and application-specific.

Please join us. We need time series and domain experts, including especially subject matter experts and use case and product owners who would like to apply emerging time series foundation models to new applications. There is an acute shortage of good, open datasets for time series and data specially benchmarks and evaluation methods for various use cases. Contributions are especially welcome here.

Screen shot of the AI Accelerator Software Ecosystem Guide

AI Accelerator Software Ecosystem Guide

Hardware Enablement

The AI Accelerator Software Ecosystem Guide provides an introduction to the broad topic of software tools that support seamless execution of AI workloads on different hardware accelerators. Industry experts provide guidance on the state of the art and how to ensure success.

abstract gradient

Industry Open FMs Initiative

Foundation Models and Datasets

We have seen rapid progress in building and releasing highly-capable and open foundation models for general language, coding, scientific discovery, and multi-modal scenarios.

A key development in model strategies is a focus on building smaller, more specialized models.

More details are coming soon, but we would love for you to join us. We need both model-building and domain experts, including those outside the target domains listed above.

Understanding AI Trust and Safety: A Living Guide

A major challenge for the successful use of AI is the importance of understanding potential trust and safety issues, along with their mitigation strategies. Failure to consider these issues could impact an organization's operations and the experience of its users. Concerns about safety are also a driver for current regulatory initiatives. Hence, applications built with AI must be designed and implemented with AI trust and safety in mind. This guide provides an introduction to trust and safety concerns, and offers guidance for AI projects.

The Living Guide to Applying AI

Applications and Tools

A "living guide" for building AI-enabled applications, this guide provides an introduction to several established design patterns for building AI systems and products, with contributions from different experts in the AI Alliance. Several common patterns, like RAG (retrieval-augmented generation), are explored from different angles, and emerging patterns, like GraphRAG (using a graph network as a source of RAG content) and agents.

Trust and Safety Evaluations

Trust & Safety

Much like other software, generative AI (“GenAI”) models and the AI systems that use them need to be trusted and useful to their users. The Trust and Safety Evaluations project fills gaps in the current landscape of the taxonomy of different kinds of evaluation, the tools for creating and running evaluations, and leaderboards to address particular categories of user needs.

Affiliated projects

View all

Affiliated projects are led and managed by members and identified as being aligned to the AI Alliance mission. Although Affiliated Projects are not managed by the AI Alliance, we highlight them as contributing to open, safe, and responsible AI.

IBM Granite Code Models

IBM logo

A series of code models trained by IBM licensed under Apache 2.0 license. Includes base pretrained and instruct models.

AIConfig & Editor

AIConfig is a config-based framework for building generative AI applications.

GenAI in Education: Usage Guidance

A report evaluating the feasibility, benefits, and limitations of using generative AI technologies in an educational setting and its impact on learning outcomes.

LLM 360: Amber

The Amber project includes a 7B English language model with the LLaMA architecture, an instruction-following model finetuned from LLM360-Amber, and a safety-finetuned instruction model using LLM360-AmberChat as the base.

TrustyAI

Red Hat logo

TrustyAI is an open-source toolkit designed for responsible AI.

Ragna

Ragna is an open-source RAG-based (Retrieval-Augmented Generation) AI orchestration framework designed to bridge the gap between AI research and production deployment.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/robots.txt b/examples/notebooks/html-processing/input2/thealliance.ai/robots.txt new file mode 100644 index 000000000..2333b4e75 --- /dev/null +++ b/examples/notebooks/html-processing/input2/thealliance.ai/robots.txt @@ -0,0 +1,3 @@ +User-Agent: * +Allow: / +Sitemap: https://thealliance.ai/sitemap.xml \ No newline at end of file diff --git a/examples/notebooks/html-processing/my_config.py b/examples/notebooks/html-processing/my_config.py index 3993352c2..2cf929e0a 100644 --- a/examples/notebooks/html-processing/my_config.py +++ b/examples/notebooks/html-processing/my_config.py @@ -7,11 +7,11 @@ class MyConfig: MY_CONFIG = MyConfig () ## Crawl settings -# MY_CONFIG.CRAWL_URL_BASE = 'https://thealliance.ai/' -MY_CONFIG.CRAWL_URL_BASE = 'https://thealliance.ai/our-work' +MY_CONFIG.CRAWL_URL_BASE = 'https://thealliance.ai/' +# MY_CONFIG.CRAWL_URL_BASE = 'https://thealliance.ai/our-work' MY_CONFIG.CRAWL_MAX_DOWNLOADS = 10 MY_CONFIG.CRAWL_MAX_DEPTH = 2 -MY_CONFIG.CRAWL_MIME_TYPE = 'html' +MY_CONFIG.CRAWL_MIME_TYPE = 'text/html' ## Input Data - configure this to the folder we want to process MY_CONFIG.INPUT_DIR = "input" diff --git a/examples/notebooks/html-processing/requirements.txt b/examples/notebooks/html-processing/requirements.txt index 750e5af4b..8fbb0efe6 100644 --- a/examples/notebooks/html-processing/requirements.txt +++ b/examples/notebooks/html-processing/requirements.txt @@ -1,8 +1,13 @@ -data-prep-toolkit[ray]==0.2.2.dev2 -data-prep-toolkit-transforms[ray,all]==0.2.2.dev2 +data-prep-toolkit==0.2.2.dev2 +data-prep-toolkit-transforms[all]==0.2.2.dev2 +# data-prep-toolkit[ray]==0.2.2.dev2 +# data-prep-toolkit-transforms[ray,all]==0.2.2.dev2 data-prep-connector==0.2.2 +## HTML processing +trafilatura + ## utils humanfriendly pandas diff --git a/examples/notebooks/html-processing/test.ipynb b/examples/notebooks/html-processing/test.ipynb new file mode 100644 index 000000000..964a1865a --- /dev/null +++ b/examples/notebooks/html-processing/test.ipynb @@ -0,0 +1,49 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from dpk_connector import crawl, shutdown\n", + "\n", + "def on_downloaded(url: str, body: bytes, headers: dict) -> None:\n", + " print(f\"url: {url}, headers: {headers}, body: {body[:64]}\")\n", + "\n", + "user_agent = \"Mozilla/5.0 (X11; Linux i686; rv:125.0) Gecko/20100101 Firefox/125.0\"\n", + "\n", + "crawl(\n", + " [\"https://thealliance.ai/\"],\n", + " on_downloaded,\n", + " user_agent=user_agent,\n", + " depth_limit=1,\n", + " subdomain_focus=True,\n", + " ) # blocking call\n", + "\n", + "shutdown()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dpk-6-basic-022dev2-py311", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/notebooks/html-processing/test.py b/examples/notebooks/html-processing/test.py new file mode 100644 index 000000000..49a92c9d2 --- /dev/null +++ b/examples/notebooks/html-processing/test.py @@ -0,0 +1,27 @@ +from dpk_connector import crawl, shutdown + + +def main(): + """ + An example of running crawler. + """ + + def on_downloaded(url: str, body: bytes, headers: dict) -> None: + # print(f"url: {url}, headers: {headers}, body: {body[:64]}") + print(f"url: {url}") + + user_agent = "Mozilla/5.0 (X11; Linux i686; rv:125.0) Gecko/20100101 Firefox/125.0" + + crawl( + ["https://thealliance.ai/"], + on_downloaded, + user_agent=user_agent, + depth_limit=1, + subdomain_focus=True, + ) # blocking call + + shutdown() + + +if __name__ == "__main__": + main() From 4ecb42040d4bdc4e73b94b21ca0c14803a9f0255 Mon Sep 17 00:00:00 2001 From: Sujee Maniyam Date: Thu, 14 Nov 2024 08:44:15 -0800 Subject: [PATCH 04/31] html2pq working Signed-off-by: Sujee Maniyam --- .../2_process_html_python.ipynb | 219 ++++++++++++++++-- .../notebooks/html-processing/my_config.py | 3 +- .../html-processing/requirements.txt | 3 +- 3 files changed, 204 insertions(+), 21 deletions(-) diff --git a/examples/notebooks/html-processing/2_process_html_python.ipynb b/examples/notebooks/html-processing/2_process_html_python.ipynb index 456e21272..5928cd9d9 100644 --- a/examples/notebooks/html-processing/2_process_html_python.ipynb +++ b/examples/notebooks/html-processing/2_process_html_python.ipynb @@ -53,16 +53,47 @@ "metadata": {}, "outputs": [ { - "ename": "AttributeError", - "evalue": "'Html2ParquetTransform' object has no attribute 'get_name'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", - "File \u001b[0;32m:20\u001b[0m\n", - "File \u001b[0;32m~/apps/anaconda3/envs/dpk-html-processing-py311/lib/python3.11/site-packages/data_processing/runtime/pure_python/transform_launcher.py:43\u001b[0m, in \u001b[0;36mPythonTransformLauncher.__init__\u001b[0;34m(self, runtime_config, data_access_factory)\u001b[0m\n\u001b[1;32m 33\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__init__\u001b[39m(\n\u001b[1;32m 34\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 35\u001b[0m runtime_config: PythonTransformRuntimeConfiguration,\n\u001b[1;32m 36\u001b[0m data_access_factory: DataAccessFactoryBase \u001b[38;5;241m=\u001b[39m DataAccessFactory(),\n\u001b[1;32m 37\u001b[0m ):\n\u001b[1;32m 38\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 39\u001b[0m \u001b[38;5;124;03m Creates driver\u001b[39;00m\n\u001b[1;32m 40\u001b[0m \u001b[38;5;124;03m :param runtime_config: transform runtime factory\u001b[39;00m\n\u001b[1;32m 41\u001b[0m \u001b[38;5;124;03m :param data_access_factory: the factory to create DataAccess instances.\u001b[39;00m\n\u001b[1;32m 42\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m---> 43\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mruntime_config\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata_access_factory\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 44\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexecution_config \u001b[38;5;241m=\u001b[39m PythonTransformExecutionConfiguration(name\u001b[38;5;241m=\u001b[39mruntime_config\u001b[38;5;241m.\u001b[39mget_name())\n", - "File \u001b[0;32m~/apps/anaconda3/envs/dpk-html-processing-py311/lib/python3.11/site-packages/data_processing/runtime/transform_launcher.py:37\u001b[0m, in \u001b[0;36mAbstractTransformLauncher.__init__\u001b[0;34m(self, runtime_config, data_access_factory)\u001b[0m\n\u001b[1;32m 31\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 32\u001b[0m \u001b[38;5;124;03mCreates driver\u001b[39;00m\n\u001b[1;32m 33\u001b[0m \u001b[38;5;124;03m:param runtime_config: transform runtime factory\u001b[39;00m\n\u001b[1;32m 34\u001b[0m \u001b[38;5;124;03m:param data_access_factory: the factory to create DataAccess instances.\u001b[39;00m\n\u001b[1;32m 35\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 36\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mruntime_config \u001b[38;5;241m=\u001b[39m runtime_config\n\u001b[0;32m---> 37\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mname \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mruntime_config\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_name\u001b[49m()\n\u001b[1;32m 38\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdata_access_factory \u001b[38;5;241m=\u001b[39m data_access_factory\n", - "\u001b[0;31mAttributeError\u001b[0m: 'Html2ParquetTransform' object has no attribute 'get_name'" + "name": "stderr", + "output_type": "stream", + "text": [ + "14:35:21 INFO - html2parquet parameters are : {'output_format': , 'favor_precision': , 'favor_recall': }\n", + "14:35:21 INFO - pipeline id pipeline_id\n", + "14:35:21 INFO - code location None\n", + "14:35:21 INFO - data factory data_ is using local data access: input_folder - input2/thealliance.ai/ output_folder - output\n", + "14:35:21 INFO - data factory data_ max_files -1, n_sample -1\n", + "14:35:21 INFO - data factory data_ Not using data sets, checkpointing False, max files -1, random samples -1, files to use ['.html', '.zip'], files to checkpoint ['.parquet']\n", + "14:35:21 INFO - orchestrator html2parquet started at 2024-11-13 14:35:21\n", + "14:35:21 INFO - Number of files is 18, source profile {'max_file_size': 0.2035503387451172, 'min_file_size': 0.06348037719726562, 'total_file_size': 1.7128067016601562}\n", + "14:35:21 INFO - Completed 1 files (5.56%) in 0.003 min\n", + "14:35:21 INFO - Completed 2 files (11.11%) in 0.004 min\n", + "14:35:21 INFO - Completed 3 files (16.67%) in 0.004 min\n", + "14:35:21 INFO - Completed 4 files (22.22%) in 0.004 min\n", + "14:35:21 INFO - Completed 5 files (27.78%) in 0.004 min\n", + "14:35:21 INFO - Completed 6 files (33.33%) in 0.004 min\n", + "14:35:21 INFO - Completed 7 files (38.89%) in 0.005 min\n", + "14:35:21 INFO - Completed 8 files (44.44%) in 0.005 min\n", + "14:35:21 INFO - Completed 9 files (50.0%) in 0.005 min\n", + "14:35:21 INFO - Completed 10 files (55.56%) in 0.005 min\n", + "14:35:21 INFO - Completed 11 files (61.11%) in 0.005 min\n", + "14:35:21 INFO - Completed 12 files (66.67%) in 0.006 min\n", + "14:35:21 INFO - Completed 13 files (72.22%) in 0.006 min\n", + "14:35:21 INFO - Completed 14 files (77.78%) in 0.006 min\n", + "14:35:21 INFO - Completed 15 files (83.33%) in 0.006 min\n", + "14:35:21 INFO - Completed 16 files (88.89%) in 0.006 min\n", + "14:35:21 INFO - Completed 17 files (94.44%) in 0.006 min\n", + "14:35:21 INFO - Completed 18 files (100.0%) in 0.007 min\n", + "14:35:21 INFO - Done processing 18 files, waiting for flush() completion.\n", + "14:35:21 INFO - done flushing in 0.0 sec\n", + "14:35:21 INFO - Completed execution in 0.007 min, execution result 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ Job completed successfully\n", + "CPU times: user 1.25 s, sys: 1.04 s, total: 2.29 s\n", + "Wall time: 1.34 s\n" ] } ], @@ -73,35 +104,185 @@ "import os\n", "import sys\n", "\n", - "from html2parquet_transform import Html2ParquetTransform, Html2ParquetTransformConfiguration\n", + "# from html2parquet_transform import Html2ParquetTransform, Html2ParquetTransformConfiguration\n", + "from html2parquet_transform_python import Html2ParquetPythonTransformConfiguration\n", "from data_processing.runtime.pure_python import PythonTransformLauncher\n", "from data_processing.utils import GB, ParamsUtils\n", "\n", "\n", - "html2parquet_params = {}\n", - "\n", - "params = {\n", + "local_conf = {\n", " \"input_folder\": MY_CONFIG.INPUT_DIR,\n", " \"output_folder\": MY_CONFIG.OUTPUT_DIR,\n", - " \"html2parquet_params\": html2parquet_params,\n", + "}\n", + "\n", + "params = {\n", + " \"data_files_to_use\": ast.literal_eval(\"['.html','.zip']\"),\n", + " \"html2parquet_output_format\": \"markdown\",\n", + " \"data_local_config\": ParamsUtils.convert_to_ast(local_conf),\n", "}\n", "\n", "sys.argv = ParamsUtils.dict_to_req(d=params)\n", - "# create launcher\n", - "launcher = PythonTransformLauncher(Html2ParquetTransform(html2parquet_params))\n", + "# launcher = PythonTransformLauncher(runtime_config=Html2ParquetTransformConfiguration())\n", + "launcher = PythonTransformLauncher(runtime_config=Html2ParquetPythonTransformConfiguration())\n", "\n", "return_code = launcher.launch()\n", "\n", "if return_code == 0:\n", - " print (f\"✅ Stage:{STAGE} completed successfully\")\n", + " print (f\"✅ Job completed successfully\")\n", "else:\n", " raise Exception (\"❌ Job failed\")" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Inspect the Output\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Output dimensions (rows x columns)= (9, 6)\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
titledocumentcontentsdocument_idsizedate_acquired
0about-aia.htmlabout-aia.htmlAI Alliance members plan to start or enhance p...0458ffe75d14867d5ea3c455ab4bf24bc0b6670812209a...13812024-11-13T14:35:21.120730
1become-a-collaborator.htmlbecome-a-collaborator.html# Become a collaborator\\n\\nWant to get involve...e3886adc3518259f0ecea4283b99f52a095e93525139c0...3972024-11-13T14:35:21.338032
2community.htmlcommunity.html# Join the Community\\n\\nJoin leading AI innova...eaa326274fea96c05c270dd7527faeb070c6ca704e63ae...29572024-11-13T14:35:21.361282
3blog.htmlblog.html![](https://images.prismic.io/ai-alliance/ZyPy...9907b2cab6aa23704d0ff3a475c38f33f5f2daccf6bb5a...31932024-11-13T14:35:21.348011
4contact.htmlcontact.html# Contact us\\n\\nWe’re here to help and answer ...31062192f12429eda320c8a6c5f1d2f725f5579be515a7...4622024-11-13T14:35:21.372251
\n", + "
" + ], + "text/plain": [ + " title document \\\n", + "0 about-aia.html about-aia.html \n", + "1 become-a-collaborator.html become-a-collaborator.html \n", + "2 community.html community.html \n", + "3 blog.html blog.html \n", + "4 contact.html contact.html \n", + "\n", + " contents \\\n", + "0 AI Alliance members plan to start or enhance p... \n", + "1 # Become a collaborator\\n\\nWant to get involve... \n", + "2 # Join the Community\\n\\nJoin leading AI innova... \n", + "3 ![](https://images.prismic.io/ai-alliance/ZyPy... \n", + "4 # Contact us\\n\\nWe’re here to help and answer ... \n", + "\n", + " document_id size \\\n", + "0 0458ffe75d14867d5ea3c455ab4bf24bc0b6670812209a... 1381 \n", + "1 e3886adc3518259f0ecea4283b99f52a095e93525139c0... 397 \n", + "2 eaa326274fea96c05c270dd7527faeb070c6ca704e63ae... 2957 \n", + "3 9907b2cab6aa23704d0ff3a475c38f33f5f2daccf6bb5a... 3193 \n", + "4 31062192f12429eda320c8a6c5f1d2f725f5579be515a7... 462 \n", + "\n", + " date_acquired \n", + "0 2024-11-13T14:35:21.120730 \n", + "1 2024-11-13T14:35:21.338032 \n", + "2 2024-11-13T14:35:21.361282 \n", + "3 2024-11-13T14:35:21.348011 \n", + "4 2024-11-13T14:35:21.372251 " + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from my_utils import read_parquet_files_as_df\n", + "\n", + "output_df = read_parquet_files_as_df(MY_CONFIG.OUTPUT_DIR)\n", + "\n", + "print (\"Output dimensions (rows x columns)= \", output_df.shape)\n", + "\n", + "output_df.head(5)\n", + "\n", + "## To display certain columns\n", + "#parquet_df[['column1', 'column2', 'column3']].head(5)" + ] } ], "metadata": { "kernelspec": { - "display_name": "dpk-html-processing-py311", + "display_name": "dpk-html-processing-1-py311", "language": "python", "name": "python3" }, diff --git a/examples/notebooks/html-processing/my_config.py b/examples/notebooks/html-processing/my_config.py index 2cf929e0a..663914245 100644 --- a/examples/notebooks/html-processing/my_config.py +++ b/examples/notebooks/html-processing/my_config.py @@ -14,7 +14,8 @@ class MyConfig: MY_CONFIG.CRAWL_MIME_TYPE = 'text/html' ## Input Data - configure this to the folder we want to process -MY_CONFIG.INPUT_DIR = "input" +# MY_CONFIG.INPUT_DIR = "input" +MY_CONFIG.INPUT_DIR = "input2/thealliance.ai/" MY_CONFIG.OUTPUT_DIR = "output" MY_CONFIG.OUTPUT_DIR_FINAL = os.path.join(MY_CONFIG.OUTPUT_DIR , "output_final") ### ------------------------------- diff --git a/examples/notebooks/html-processing/requirements.txt b/examples/notebooks/html-processing/requirements.txt index 8fbb0efe6..dd1ff4a7c 100644 --- a/examples/notebooks/html-processing/requirements.txt +++ b/examples/notebooks/html-processing/requirements.txt @@ -1,5 +1,6 @@ data-prep-toolkit==0.2.2.dev2 -data-prep-toolkit-transforms[all]==0.2.2.dev2 +data-prep-toolkit-transforms[html2parquet]==0.2.2.dev2 + # data-prep-toolkit[ray]==0.2.2.dev2 # data-prep-toolkit-transforms[ray,all]==0.2.2.dev2 From 4daf7a12b761b9edf438cfcd489177474da30bde Mon Sep 17 00:00:00 2001 From: Sujee Maniyam Date: Fri, 22 Nov 2024 00:09:30 -0800 Subject: [PATCH 05/31] HTML processing working example Signed-off-by: Sujee Maniyam --- .../html-processing/1_crawl_site.ipynb | 173 +++----- .../2_process_html_python.ipynb | 409 ++++++++++++++---- examples/notebooks/html-processing/README.md | 14 +- .../notebooks/html-processing/my_config.py | 6 +- .../html-processing/requirements.txt | 8 +- 5 files changed, 389 insertions(+), 221 deletions(-) diff --git a/examples/notebooks/html-processing/1_crawl_site.ipynb b/examples/notebooks/html-processing/1_crawl_site.ipynb index 6c99c3681..c65c48a54 100644 --- a/examples/notebooks/html-processing/1_crawl_site.ipynb +++ b/examples/notebooks/html-processing/1_crawl_site.ipynb @@ -44,7 +44,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "✅ Cleared output directory\n" + "✅ Cleared download directory\n" ] } ], @@ -55,146 +55,81 @@ "if not os.path.exists(MY_CONFIG.INPUT_DIR ):\n", " shutil.os.makedirs(MY_CONFIG.INPUT_DIR, exist_ok=True)\n", "\n", - "## clear output folder\n", - "shutil.rmtree(MY_CONFIG.OUTPUT_DIR, ignore_errors=True)\n", - "shutil.os.makedirs(MY_CONFIG.OUTPUT_DIR, exist_ok=True)\n", + "## clear input / output folder\n", + "shutil.rmtree(MY_CONFIG.INPUT_DIR, ignore_errors=True)\n", + "shutil.os.makedirs(MY_CONFIG.INPUT_DIR, exist_ok=True)\n", "\n", - "print (\"✅ Cleared output directory\")" + "\n", + "\n", + "print (\"✅ Cleared download directory\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "# Step-3: Crawl the website\n", + "## Step-3: Crawl the website\n", + "\n", + "We will use `dpk-connector` utility to download a some HTML pages from a site \n", + "\n", + "\n", + "**Parameters**\n", "\n", - "We will use `dpk-connector` utility to download a some HTML pages from a site " + "For configuring the crawl, users need to specify the following parameters:\n", + "\n", + "| parameter:type | Description |\n", + "| --- | --- |\n", + "| urls:list | list of seed URLs (i.e., ['https://thealliance.ai'] or ['https://www.apache.org/projects','https://www.apache.org/foundation']). The list can include any number of valid URLS that are not configured to block web crawlers |\n", + "|depth:int | control crawling depth |\n", + "| downloads:int | number of downloads that are stored to the download folder. Since the crawler operations happen asynchronously, the process can result in any 10 of the visited URLs being retrieved (i.e. consecutive runs can result in different files being downloaded) |\n", + "| folder:str | folder where downloaded files are stored. If the folder is not empty, new files are added or replace the existing ones with the same URLs |" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "url: https://thealliance.ai/\n", - "url: https://thealliance.ai/core-projects/ntia_request\n", - "url: https://thealliance.ai/contact\n", - "url: https://thealliance.ai/blog/open-source-ai-demo-night-sf-2024\n", - "url: https://thealliance.ai/core-projects/trusted-evals\n", - "url: https://thealliance.ai/core-projects/sb1047\n", - "url: https://thealliance.ai/focus-areas/hardware-enablement\n", - "url: https://thealliance.ai/focus-areas/applications-and-tools\n", - "url: https://thealliance.ai/focus-areas/trust-and-safety\n", - "url: https://thealliance.ai/community\n", - "url: https://thealliance.ai/focus-areas/advocacy\n", - "url: https://thealliance.ai/focus-areas/skills-education\n", - "url: https://thealliance.ai/contribute\n", - "url: https://thealliance.ai/become-a-collaborator\n", - "url: https://thealliance.ai/focus-areas/foundation-models-datasets\n", - "url: https://thealliance.ai/aia-members\n", - "url: https://thealliance.ai/our-work\n", - "url: https://thealliance.ai/about-aia\n", - "url: https://thealliance.ai/core-projects/the-living-guide-to-applying-ai\n", - "url: https://thealliance.ai/core-projects/trust-and-safety-evaluations\n", - "url: https://thealliance.ai/core-projects/safety-priorities-ranking-by-domain\n", - "url: https://thealliance.ai/core-projects/trust-user-guide\n", - "url: https://thealliance.ai/blog/ai-alliance-expands-with-seven-new-members-from-in\n", - "url: https://thealliance.ai/blog/ai-for-drug-discovery-open-innovation-forum\n", - "url: https://thealliance.ai/blog/unesco-language-translator\n", - "url: https://thealliance.ai/blog/advancing-domain-specific-qa-the-ai-alliances-guid\n", - "url: https://thealliance.ai/core-projects/ai-accelerator-software-ecosystem-guide\n", - "url: https://thealliance.ai/blog\n", - "url: https://thealliance.ai/blog/evaluation-of-generative-ai---whats-ultimately-our\n" - ] - }, - { - "data": { - "text/plain": [ - "'Crawl is done'" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "from dpk_connector import crawl, shutdown\n", - "import nest_asyncio\n", - "import os\n", - "from my_utils import get_mime_type, get_filename_from_url\n", - "from dpk_connector.core.utils import validate_url\n", - "\n", - "# Use nest_asyncio to enable a nested event loop run for the crawler inside the Jupyter notebook\n", - "nest_asyncio.apply()\n", - "\n", - "# Initialize counter\n", - "retrieved_pages = 0\n", - "saved_pages = 0\n", + "## must enable nested asynchronous io in a notebook as the crawler uses \n", + "## coroutine to speed up acquisition and downloads\n", "\n", - "# Define a callback function to be executed at the retrieval of each page during a crawl\n", - "def on_downloaded(url: str, body: bytes, headers: dict) -> None:\n", - " \"\"\"\n", - " Callback function called when a page has been downloaded.\n", - " You have access to the request URL, response body and headers.\n", - " \"\"\"\n", - " global retrieved_pages, saved_pages\n", - " retrieved_pages+=1\n", - " \n", - " print (f'on_downloaded (url={url})')\n", - "\n", - " if saved_pages < MY_CONFIG.CRAWL_MAX_DOWNLOADS:\n", - " print(f\"Visited url: {url}\")\n", - "\n", - " # Get mime_type of retrieved page\n", - " mime_type = get_mime_type(body)\n", - " print (f'mime_type={mime_type}')\n", - " \n", - " # Save the page if it is a PDF to only download research papers\n", - " if MY_CONFIG.CRAWL_MIME_TYPE in mime_type.lower():\n", - " print ('hi')\n", - " filename = get_filename_from_url(url)\n", - " local_file_path = os.path.join(MY_CONFIG.INPUT_DIR, filename)\n", - " print (local_file_path)\n", - " \n", - " with open(local_file_path, 'wb') as f:\n", - " f.write(body)\n", - " \n", - " if saved_pages < MY_CONFIG.CRAWL_MAX_DOWNLOADS :\n", - " print(f\"Saved contents of url: {url}\")\n", - " saved_pages+=1\n", - " \n", - "# Define a user agent to provide information about the client making the request\n", - "# user_agent = \"dpk-connector\"\n", - "user_agent = \"Mozilla/5.0 (X11; Linux i686; rv:125.0) Gecko/20100101 Firefox/125.0\"\n", - "\n", - "\n", - "def on_downloaded2(url: str, body: bytes, headers: dict) -> None:\n", - " # print(f\"url: {url}, headers: {headers}, body: {body[:64]}\")\n", - " print(f\"url: {url}\")\n", + "import nest_asyncio\n", + "nest_asyncio.apply()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "from dpk_web2parquet.transform import Web2Parquet\n", "\n", - "async def run_my_crawl():\n", - " crawl([MY_CONFIG.CRAWL_URL_BASE], \n", - " on_downloaded2, \n", - " user_agent=user_agent, \n", - " depth_limit = MY_CONFIG.CRAWL_MAX_DEPTH, \n", - " subdomain_focus=True,\n", - " # path_focus = True, \n", - " download_limit = MY_CONFIG.CRAWL_MAX_DOWNLOADS)\n", - " return \"Crawl is done\"\n", + "Web2Parquet(urls= ['https://thealliance.ai/'],\n", + " depth=2, \n", + " downloads=10,\n", + " folder='input').transform()\n", + "print (\"✅ web crawl completed\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# list some files\n", "\n", - "# Now run the configured crawl\n", - "await run_my_crawl()" + "# for item in os.listdir(MY_CONFIG.INPUT_DIR):\n", + "# print(item)" ] } ], "metadata": { "kernelspec": { - "display_name": "dpk-6-basic-022dev2-py311", + "display_name": "dpk-html-processing-2-022dev3-py312", "language": "python", "name": "python3" }, @@ -208,7 +143,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.10" + "version": "3.12.7" } }, "nbformat": 4, diff --git a/examples/notebooks/html-processing/2_process_html_python.ipynb b/examples/notebooks/html-processing/2_process_html_python.ipynb index 5928cd9d9..6dd94751b 100644 --- a/examples/notebooks/html-processing/2_process_html_python.ipynb +++ b/examples/notebooks/html-processing/2_process_html_python.ipynb @@ -40,51 +40,72 @@ "from my_config import MY_CONFIG" ] }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ Cleared output directory\n" + ] + } + ], + "source": [ + "import os, sys\n", + "import shutil\n", + "\n", + "shutil.rmtree(MY_CONFIG.OUTPUT_DIR, ignore_errors=True)\n", + "shutil.os.makedirs(MY_CONFIG.OUTPUT_DIR, exist_ok=True)\n", + "shutil.os.makedirs(MY_CONFIG.OUTPUT_DIR_HTML, exist_ok=True)\n", + "shutil.os.makedirs(MY_CONFIG.OUTPUT_DIR_MARKDOWN, exist_ok=True)\n", + "\n", + "print (\"✅ Cleared output directory\")" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Step-3: HTML2Parquet" + "## Step-3: HTML2Parquet\n", + "\n", + "Process HTML documents and extract the text in markdown format" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "14:35:21 INFO - html2parquet parameters are : {'output_format': , 'favor_precision': , 'favor_recall': }\n", - "14:35:21 INFO - pipeline id pipeline_id\n", - "14:35:21 INFO - code location None\n", - "14:35:21 INFO - data factory data_ is using local data access: input_folder - input2/thealliance.ai/ output_folder - output\n", - "14:35:21 INFO - data factory data_ max_files -1, n_sample -1\n", - "14:35:21 INFO - data factory data_ Not using data sets, checkpointing False, max files -1, random samples -1, files to use ['.html', '.zip'], files to checkpoint ['.parquet']\n", - "14:35:21 INFO - orchestrator html2parquet started at 2024-11-13 14:35:21\n", - "14:35:21 INFO - Number of files is 18, source profile {'max_file_size': 0.2035503387451172, 'min_file_size': 0.06348037719726562, 'total_file_size': 1.7128067016601562}\n", - "14:35:21 INFO - Completed 1 files (5.56%) in 0.003 min\n", - "14:35:21 INFO - Completed 2 files (11.11%) in 0.004 min\n", - "14:35:21 INFO - Completed 3 files (16.67%) in 0.004 min\n", - "14:35:21 INFO - Completed 4 files (22.22%) in 0.004 min\n", - "14:35:21 INFO - Completed 5 files (27.78%) in 0.004 min\n", - "14:35:21 INFO - Completed 6 files (33.33%) in 0.004 min\n", - "14:35:21 INFO - Completed 7 files (38.89%) in 0.005 min\n", - "14:35:21 INFO - Completed 8 files (44.44%) in 0.005 min\n", - "14:35:21 INFO - Completed 9 files (50.0%) in 0.005 min\n", - "14:35:21 INFO - Completed 10 files (55.56%) in 0.005 min\n", - "14:35:21 INFO - Completed 11 files (61.11%) in 0.005 min\n", - "14:35:21 INFO - Completed 12 files (66.67%) in 0.006 min\n", - "14:35:21 INFO - Completed 13 files (72.22%) in 0.006 min\n", - "14:35:21 INFO - Completed 14 files (77.78%) in 0.006 min\n", - "14:35:21 INFO - Completed 15 files (83.33%) in 0.006 min\n", - "14:35:21 INFO - Completed 16 files (88.89%) in 0.006 min\n", - "14:35:21 INFO - Completed 17 files (94.44%) in 0.006 min\n", - "14:35:21 INFO - Completed 18 files (100.0%) in 0.007 min\n", - "14:35:21 INFO - Done processing 18 files, waiting for flush() completion.\n", - "14:35:21 INFO - done flushing in 0.0 sec\n", - "14:35:21 INFO - Completed execution in 0.007 min, execution result 0\n" + "23:40:23 INFO - html2parquet parameters are : {'output_format': , 'favor_precision': , 'favor_recall': }\n", + "23:40:23 INFO - pipeline id pipeline_id\n", + "23:40:23 INFO - code location None\n", + "23:40:23 INFO - data factory data_ is using local data access: input_folder - input output_folder - output/1-html2parquet\n", + "23:40:23 INFO - data factory data_ max_files -1, n_sample -1\n", + "23:40:23 INFO - data factory data_ Not using data sets, checkpointing False, max files -1, random samples -1, files to use ['.html', '.zip'], files to checkpoint ['.parquet']\n", + "23:40:23 INFO - orchestrator html2parquet started at 2024-11-21 23:40:23\n", + "23:40:23 INFO - Number of files is 12, source profile {'max_file_size': 0.23389148712158203, 'min_file_size': 0.0925445556640625, 'total_file_size': 1.409872055053711}\n", + "23:40:23 INFO - Completed 1 files (8.33%) in 0.003 min\n", + "23:40:23 INFO - Completed 2 files (16.67%) in 0.004 min\n", + "23:40:23 INFO - Completed 3 files (25.0%) in 0.004 min\n", + "23:40:23 INFO - Completed 4 files (33.33%) in 0.004 min\n", + "23:40:23 INFO - Completed 5 files (41.67%) in 0.004 min\n", + "23:40:23 INFO - Completed 6 files (50.0%) in 0.004 min\n", + "23:40:23 INFO - Completed 7 files (58.33%) in 0.004 min\n", + "23:40:23 INFO - Completed 8 files (66.67%) in 0.005 min\n", + "23:40:23 INFO - Completed 9 files (75.0%) in 0.005 min\n", + "23:40:23 INFO - Completed 10 files (83.33%) in 0.005 min\n", + "23:40:23 INFO - Completed 11 files (91.67%) in 0.005 min\n", + "23:40:23 INFO - Completed 12 files (100.0%) in 0.005 min\n", + "23:40:23 INFO - Done processing 12 files, waiting for flush() completion.\n", + "23:40:23 INFO - done flushing in 0.0 sec\n", + "23:40:23 INFO - Completed execution in 0.005 min, execution result 0\n" ] }, { @@ -92,8 +113,8 @@ "output_type": "stream", "text": [ "✅ Job completed successfully\n", - "CPU times: user 1.25 s, sys: 1.04 s, total: 2.29 s\n", - "Wall time: 1.34 s\n" + "CPU times: user 1.15 s, sys: 1.01 s, total: 2.17 s\n", + "Wall time: 1.28 s\n" ] } ], @@ -112,7 +133,7 @@ "\n", "local_conf = {\n", " \"input_folder\": MY_CONFIG.INPUT_DIR,\n", - " \"output_folder\": MY_CONFIG.OUTPUT_DIR,\n", + " \"output_folder\": MY_CONFIG.OUTPUT_DIR_HTML,\n", "}\n", "\n", "params = {\n", @@ -137,19 +158,19 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Inspect the Output\n" + "## Step-4: Inspect the Output\n" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Output dimensions (rows x columns)= (9, 6)\n" + "Output dimensions (rows x columns)= (12, 6)\n" ] }, { @@ -184,84 +205,91 @@ " \n", " \n", " 0\n", - " about-aia.html\n", - " about-aia.html\n", - " AI Alliance members plan to start or enhance p...\n", - " 0458ffe75d14867d5ea3c455ab4bf24bc0b6670812209a...\n", - " 1381\n", - " 2024-11-13T14:35:21.120730\n", + " thealliance_ai_blog-open-source-ai-demo-night-...\n", + " thealliance_ai_blog-open-source-ai-demo-night-...\n", + " On August 8th, The AI Alliance, in collaborati...\n", + " 7802bb7e50653e6b21f571b28843fd9a4bcf5023eaab3a...\n", + " 3151\n", + " 2024-11-21T23:40:23.824364\n", " \n", " \n", " 1\n", - " become-a-collaborator.html\n", - " become-a-collaborator.html\n", - " # Become a collaborator\\n\\nWant to get involve...\n", - " e3886adc3518259f0ecea4283b99f52a095e93525139c0...\n", - " 397\n", - " 2024-11-13T14:35:21.338032\n", + " thealliance_ai_core-projects-sb1047_text.html\n", + " thealliance_ai_core-projects-sb1047_text.html\n", + " The AI Alliance, a community of technology cre...\n", + " bbfed07faf040c9f276df43207437f0501cf9da14ec956...\n", + " 7184\n", + " 2024-11-21T23:40:23.877930\n", " \n", " \n", " 2\n", - " community.html\n", - " community.html\n", - " # Join the Community\\n\\nJoin leading AI innova...\n", - " eaa326274fea96c05c270dd7527faeb070c6ca704e63ae...\n", - " 2957\n", - " 2024-11-13T14:35:21.361282\n", + " thealliance_ai_focus-areas-foundation-models-d...\n", + " thealliance_ai_focus-areas-foundation-models-d...\n", + " # Open Foundation Models and Datasets\\n\\n### E...\n", + " cace8c007c2c65b7a92d9f152b7e012502b1614205e7c9...\n", + " 4499\n", + " 2024-11-21T23:40:23.906316\n", " \n", " \n", " 3\n", - " blog.html\n", - " blog.html\n", - " ![](https://images.prismic.io/ai-alliance/ZyPy...\n", - " 9907b2cab6aa23704d0ff3a475c38f33f5f2daccf6bb5a...\n", - " 3193\n", - " 2024-11-13T14:35:21.348011\n", + " thealliance_ai_focus-areas-hardware-enablement...\n", + " thealliance_ai_focus-areas-hardware-enablement...\n", + " # Hardware Enablement\\n\\n### Fostering a vibra...\n", + " ddd2ecd7f5cc71f661f41f80d95525b1a6478d0fbff231...\n", + " 582\n", + " 2024-11-21T23:40:23.918782\n", " \n", " \n", " 4\n", - " contact.html\n", - " contact.html\n", - " # Contact us\\n\\nWe’re here to help and answer ...\n", - " 31062192f12429eda320c8a6c5f1d2f725f5579be515a7...\n", - " 462\n", - " 2024-11-13T14:35:21.372251\n", + " thealliance_ai_community_text.html\n", + " thealliance_ai_community_text.html\n", + " # Join the Community\\n\\nJoin leading AI innova...\n", + " eaa326274fea96c05c270dd7527faeb070c6ca704e63ae...\n", + " 2957\n", + " 2024-11-21T23:40:23.838481\n", " \n", " \n", "\n", "" ], "text/plain": [ - " title document \\\n", - "0 about-aia.html about-aia.html \n", - "1 become-a-collaborator.html become-a-collaborator.html \n", - "2 community.html community.html \n", - "3 blog.html blog.html \n", - "4 contact.html contact.html \n", + " title \\\n", + "0 thealliance_ai_blog-open-source-ai-demo-night-... \n", + "1 thealliance_ai_core-projects-sb1047_text.html \n", + "2 thealliance_ai_focus-areas-foundation-models-d... \n", + "3 thealliance_ai_focus-areas-hardware-enablement... \n", + "4 thealliance_ai_community_text.html \n", + "\n", + " document \\\n", + "0 thealliance_ai_blog-open-source-ai-demo-night-... \n", + "1 thealliance_ai_core-projects-sb1047_text.html \n", + "2 thealliance_ai_focus-areas-foundation-models-d... \n", + "3 thealliance_ai_focus-areas-hardware-enablement... \n", + "4 thealliance_ai_community_text.html \n", "\n", " contents \\\n", - "0 AI Alliance members plan to start or enhance p... \n", - "1 # Become a collaborator\\n\\nWant to get involve... \n", - "2 # Join the Community\\n\\nJoin leading AI innova... \n", - "3 ![](https://images.prismic.io/ai-alliance/ZyPy... \n", - "4 # Contact us\\n\\nWe’re here to help and answer ... \n", + "0 On August 8th, The AI Alliance, in collaborati... \n", + "1 The AI Alliance, a community of technology cre... \n", + "2 # Open Foundation Models and Datasets\\n\\n### E... \n", + "3 # Hardware Enablement\\n\\n### Fostering a vibra... \n", + "4 # Join the Community\\n\\nJoin leading AI innova... \n", "\n", " document_id size \\\n", - "0 0458ffe75d14867d5ea3c455ab4bf24bc0b6670812209a... 1381 \n", - "1 e3886adc3518259f0ecea4283b99f52a095e93525139c0... 397 \n", - "2 eaa326274fea96c05c270dd7527faeb070c6ca704e63ae... 2957 \n", - "3 9907b2cab6aa23704d0ff3a475c38f33f5f2daccf6bb5a... 3193 \n", - "4 31062192f12429eda320c8a6c5f1d2f725f5579be515a7... 462 \n", + "0 7802bb7e50653e6b21f571b28843fd9a4bcf5023eaab3a... 3151 \n", + "1 bbfed07faf040c9f276df43207437f0501cf9da14ec956... 7184 \n", + "2 cace8c007c2c65b7a92d9f152b7e012502b1614205e7c9... 4499 \n", + "3 ddd2ecd7f5cc71f661f41f80d95525b1a6478d0fbff231... 582 \n", + "4 eaa326274fea96c05c270dd7527faeb070c6ca704e63ae... 2957 \n", "\n", " date_acquired \n", - "0 2024-11-13T14:35:21.120730 \n", - "1 2024-11-13T14:35:21.338032 \n", - "2 2024-11-13T14:35:21.361282 \n", - "3 2024-11-13T14:35:21.348011 \n", - "4 2024-11-13T14:35:21.372251 " + "0 2024-11-21T23:40:23.824364 \n", + "1 2024-11-21T23:40:23.877930 \n", + "2 2024-11-21T23:40:23.906316 \n", + "3 2024-11-21T23:40:23.918782 \n", + "4 2024-11-21T23:40:23.838481 " ] }, - "execution_count": 3, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -269,7 +297,7 @@ "source": [ "from my_utils import read_parquet_files_as_df\n", "\n", - "output_df = read_parquet_files_as_df(MY_CONFIG.OUTPUT_DIR)\n", + "output_df = read_parquet_files_as_df(MY_CONFIG.OUTPUT_DIR_HTML)\n", "\n", "print (\"Output dimensions (rows x columns)= \", output_df.shape)\n", "\n", @@ -278,11 +306,208 @@ "## To display certain columns\n", "#parquet_df[['column1', 'column2', 'column3']].head(5)" ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'thealliance_ai_blog-open-source-ai-demo-night-sf-2024_text.html'" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "output_df.iloc[0,]['title']" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'thealliance_ai_blog-open-source-ai-demo-night-sf-2024_text.html'" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "output_df.iloc[0,]['document']" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "On August 8th, The AI Alliance, in collaboration with Cerebral Valley and Ollama, hosted Open Source AI Demo Night in San Francisco, bringing together more than 200+ developers and innovators to showcase and celebrate the latest advances in open-source AI. There were 7 demo teams and a panel discussion on [why open technologies and communities are essential to driving innovation in California](https://youtu.be/tOXzyHJvOKw).\n", + "\n", + "The demo teams included:\n", + "\n", + "[Ollama](https://ollama.com/)- helps developers run language models such as Llama 3.1, Mistral, Gemma 2, and others, locally on the computer or on a server cluster. Watch Michael Yang’s demo here:[Tool calling with Ollama - How an LLM accesses external information.](https://youtu.be/YWLLrgzzbj8)[Continue](https://www.continue.dev/)– a leading open-source AI code assistant that connects any models and any context to build custom autocomplete and chat experiences inside the IDE. Watch Ty Dunn’s demo here:[Using Continue to understand a brand new code library](https://youtu.be/BUq66FHVqng)[AgentOps](https://www.agentops.ai/)– an industry-leading developer platform to test and debug AI agents. Watch Alex Reibman and Ajay Poshak demo LlamaFS here:[LlamaFS: A self-organizing agentic filesystem](https://youtu.be/P3pND_JSkuQ)[CrewAI](https://www.crewai.com/)- Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.Watch João Moura’s demo here:[Build multi-agent automations with Crew.ai](https://youtu.be/5b07faElxfM).[Based Hardware](https://basedhardware.com/)– a fully open-source AI note taker that provides you with reminders, suggestions, and more; all in one simple app. Watch Nik Shevchenko’s demo here:[Friend: An AI necklace you wear which records your day](https://youtu.be/e0owdgDDP0I)[Datafog](https://www.datafog.ai/)– an open source AI/ML platform with solutions to scan unstructured content in files for PII, either annotating, anonymizing, or redacting sensitive information. Watch Sid Mohan’s demo here:[Using Open Source LLMs for PII data detection with DataFog](https://youtu.be/c1dx2bzaplk)[Semikong](https://www.semikong.ai/)- the World’s First Semiconductor Industry-Specific Large Language Model. Watch Nanda Kishore‘s demo here:[SemiKong: The Open Source Semiconductor LLM powered by Llama](https://youtu.be/zIhyFom_obM)\n", + "\n", + "\n", + "Demo Night also featured a panel discussion “[AI in the Era of Open Innovation](https://youtu.be/tOXzyHJvOKw),” moderated by CEO & Founder Aitomatic Christopher Nguyen, and featured Matt White, Executive Director of PyTorch Foundation and General Manager of AI, Linux Foundation; Charles Xie, CEO of Zilliz; and Sharon Zhou, CEO of Lamini. The panelists underscored the importance of having access to state of the art open-source AI models in building their company by fine-tuning the models to their respective company needs. Moreover, the panelists opposed California Senate Bill 1047, highlighting that it would stifle open-source AI development and have a downstream chilling effect on AI investment and expansion.\n" + ] + } + ], + "source": [ + "import pprint \n", + "\n", + "# pprint.pprint (output_df.iloc[0,]['contents'])\n", + "\n", + "print (output_df.iloc[0,]['contents'])\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "On August 8th, The AI Alliance, in collaboration with Cerebral Valley and Ollama, hosted Open Source AI Demo Night in San Francisco, bringing together more than 200+ developers and innovators to showcase and celebrate the latest advances in open-source AI. There were 7 demo teams and a panel discussion on [why open technologies and communities are essential to driving innovation in California](https://youtu.be/tOXzyHJvOKw).\n", + "\n", + "The demo teams included:\n", + "\n", + "[Ollama](https://ollama.com/)- helps developers run language models such as Llama 3.1, Mistral, Gemma 2, and others, locally on the computer or on a server cluster. Watch Michael Yang’s demo here:[Tool calling with Ollama - How an LLM accesses external information.](https://youtu.be/YWLLrgzzbj8)[Continue](https://www.continue.dev/)– a leading open-source AI code assistant that connects any models and any context to build custom autocomplete and chat experiences inside the IDE. Watch Ty Dunn’s demo here:[Using Continue to understand a brand new code library](https://youtu.be/BUq66FHVqng)[AgentOps](https://www.agentops.ai/)– an industry-leading developer platform to test and debug AI agents. Watch Alex Reibman and Ajay Poshak demo LlamaFS here:[LlamaFS: A self-organizing agentic filesystem](https://youtu.be/P3pND_JSkuQ)[CrewAI](https://www.crewai.com/)- Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.Watch João Moura’s demo here:[Build multi-agent automations with Crew.ai](https://youtu.be/5b07faElxfM).[Based Hardware](https://basedhardware.com/)– a fully open-source AI note taker that provides you with reminders, suggestions, and more; all in one simple app. Watch Nik Shevchenko’s demo here:[Friend: An AI necklace you wear which records your day](https://youtu.be/e0owdgDDP0I)[Datafog](https://www.datafog.ai/)– an open source AI/ML platform with solutions to scan unstructured content in files for PII, either annotating, anonymizing, or redacting sensitive information. Watch Sid Mohan’s demo here:[Using Open Source LLMs for PII data detection with DataFog](https://youtu.be/c1dx2bzaplk)[Semikong](https://www.semikong.ai/)- the World’s First Semiconductor Industry-Specific Large Language Model. Watch Nanda Kishore‘s demo here:[SemiKong: The Open Source Semiconductor LLM powered by Llama](https://youtu.be/zIhyFom_obM)\n", + "\n", + "\n", + "Demo Night also featured a panel discussion “[AI in the Era of Open Innovation](https://youtu.be/tOXzyHJvOKw),” moderated by CEO & Founder Aitomatic Christopher Nguyen, and featured Matt White, Executive Director of PyTorch Foundation and General Manager of AI, Linux Foundation; Charles Xie, CEO of Zilliz; and Sharon Zhou, CEO of Lamini. The panelists underscored the importance of having access to state of the art open-source AI models in building their company by fine-tuning the models to their respective company needs. Moreover, the panelists opposed California Senate Bill 1047, highlighting that it would stifle open-source AI development and have a downstream chilling effect on AI investment and expansion." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from IPython.display import Markdown\n", + "display(Markdown(output_df.iloc[0,]['contents']))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step-5: Save the markdown in a dir" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Index: 0\n", + "html_file': thealliance_ai_blog-open-source-ai-demo-night-sf-2024_text.html\n", + "base_name': thealliance_ai_blog-open-source-ai-demo-night-sf-2024_text\n", + "md_output_file': output/2-markdown/thealliance_ai_blog-open-source-ai-demo-night-sf-2024_text.md\n", + "\n", + "Index: 1\n", + "html_file': thealliance_ai_core-projects-sb1047_text.html\n", + "base_name': thealliance_ai_core-projects-sb1047_text\n", + "md_output_file': output/2-markdown/thealliance_ai_core-projects-sb1047_text.md\n", + "\n", + "Index: 2\n", + "html_file': thealliance_ai_focus-areas-foundation-models-datasets_text.html\n", + "base_name': thealliance_ai_focus-areas-foundation-models-datasets_text\n", + "md_output_file': output/2-markdown/thealliance_ai_focus-areas-foundation-models-datasets_text.md\n", + "\n", + "Index: 3\n", + "html_file': thealliance_ai_focus-areas-hardware-enablement_text.html\n", + "base_name': thealliance_ai_focus-areas-hardware-enablement_text\n", + "md_output_file': output/2-markdown/thealliance_ai_focus-areas-hardware-enablement_text.md\n", + "\n", + "Index: 4\n", + "html_file': thealliance_ai_community_text.html\n", + "base_name': thealliance_ai_community_text\n", + "md_output_file': output/2-markdown/thealliance_ai_community_text.md\n", + "\n", + "Index: 5\n", + "html_file': thealliance_ai_become-a-collaborator_text.html\n", + "base_name': thealliance_ai_become-a-collaborator_text\n", + "md_output_file': output/2-markdown/thealliance_ai_become-a-collaborator_text.md\n", + "\n", + "Index: 6\n", + "html_file': thealliance_ai_core-projects-ntia_request_text.html\n", + "base_name': thealliance_ai_core-projects-ntia_request_text\n", + "md_output_file': output/2-markdown/thealliance_ai_core-projects-ntia_request_text.md\n", + "\n", + "Index: 7\n", + "html_file': thealliance_ai_focus-areas-advocacy_text.html\n", + "base_name': thealliance_ai_focus-areas-advocacy_text\n", + "md_output_file': output/2-markdown/thealliance_ai_focus-areas-advocacy_text.md\n", + "\n", + "Index: 8\n", + "html_file': thealliance_ai_contact_text.html\n", + "base_name': thealliance_ai_contact_text\n", + "md_output_file': output/2-markdown/thealliance_ai_contact_text.md\n", + "\n", + "Index: 9\n", + "html_file': thealliance_ai_core-projects-trusted-evals_text.html\n", + "base_name': thealliance_ai_core-projects-trusted-evals_text\n", + "md_output_file': output/2-markdown/thealliance_ai_core-projects-trusted-evals_text.md\n", + "\n", + "Index: 10\n", + "html_file': thealliance_ai__text.html\n", + "base_name': thealliance_ai__text\n", + "md_output_file': output/2-markdown/thealliance_ai__text.md\n", + "\n", + "Index: 11\n", + "html_file': thealliance_ai_contribute_text.html\n", + "base_name': thealliance_ai_contribute_text\n", + "md_output_file': output/2-markdown/thealliance_ai_contribute_text.md\n", + "\n" + ] + } + ], + "source": [ + "import os\n", + "\n", + "for index, row in output_df.iterrows():\n", + " print(f\"Index: {index}\")\n", + " html_file = row['document']\n", + " base_name = os.path.splitext(os.path.basename(html_file))[0]\n", + " md_output_file = os.path.join(MY_CONFIG.OUTPUT_DIR_MARKDOWN, base_name + '.md')\n", + " print(f\"html_file': {html_file}\")\n", + " print(f\"base_name': {base_name}\")\n", + " print(f\"md_output_file': {md_output_file}\")\n", + " \n", + " with open(md_output_file, 'w') as md_output_file_handle:\n", + " md_output_file_handle.write (row['contents'])\n", + " \n", + " \n", + " print()\n", + " " + ] } ], "metadata": { "kernelspec": { - "display_name": "dpk-html-processing-1-py311", + "display_name": "dpk-html-processing-2-022dev3-py312", "language": "python", "name": "python3" }, @@ -296,7 +521,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.10" + "version": "3.12.7" } }, "nbformat": 4, diff --git a/examples/notebooks/html-processing/README.md b/examples/notebooks/html-processing/README.md index a9ff5dc93..c3189f8cd 100644 --- a/examples/notebooks/html-processing/README.md +++ b/examples/notebooks/html-processing/README.md @@ -26,10 +26,18 @@ pip install -r requirements.txt ``` -## Download HTML pages +## Step-2: Crawl a website -TODO: 1_crawl_site.ipynb +This step will crawl a site and download HTML files in `output/1-html2parquet` directory -Shortcut : Using wget +[1_crawl_site.ipynb](1_crawl_site.ipynb) + +or use `wget` `wget --recursive --level=1 --adjust-extension https://thealliance.ai/` + +## Step-3: Process HTML files + +We will process downloaded HTML files and extract the text as markdown. The output will be saved in `output/2-markdown` directory + +[2_process_html_python.ipynb](2_process_html_python.ipynb) \ No newline at end of file diff --git a/examples/notebooks/html-processing/my_config.py b/examples/notebooks/html-processing/my_config.py index 663914245..18c27aad6 100644 --- a/examples/notebooks/html-processing/my_config.py +++ b/examples/notebooks/html-processing/my_config.py @@ -14,8 +14,10 @@ class MyConfig: MY_CONFIG.CRAWL_MIME_TYPE = 'text/html' ## Input Data - configure this to the folder we want to process -# MY_CONFIG.INPUT_DIR = "input" -MY_CONFIG.INPUT_DIR = "input2/thealliance.ai/" +MY_CONFIG.INPUT_DIR = "input" +# MY_CONFIG.INPUT_DIR = "input2/thealliance.ai/" MY_CONFIG.OUTPUT_DIR = "output" +MY_CONFIG.OUTPUT_DIR_HTML = os.path.join(MY_CONFIG.OUTPUT_DIR , "1-html2parquet") +MY_CONFIG.OUTPUT_DIR_MARKDOWN = os.path.join(MY_CONFIG.OUTPUT_DIR , "2-markdown") MY_CONFIG.OUTPUT_DIR_FINAL = os.path.join(MY_CONFIG.OUTPUT_DIR , "output_final") ### ------------------------------- diff --git a/examples/notebooks/html-processing/requirements.txt b/examples/notebooks/html-processing/requirements.txt index dd1ff4a7c..7fb2be3bc 100644 --- a/examples/notebooks/html-processing/requirements.txt +++ b/examples/notebooks/html-processing/requirements.txt @@ -1,10 +1,8 @@ data-prep-toolkit==0.2.2.dev2 -data-prep-toolkit-transforms[html2parquet]==0.2.2.dev2 +data-prep-toolkit-transforms[html2parquet]==0.2.2.dev3 +data-prep-toolkit-transforms[web2parquet]==0.2.2.dev3 -# data-prep-toolkit[ray]==0.2.2.dev2 -# data-prep-toolkit-transforms[ray,all]==0.2.2.dev2 - -data-prep-connector==0.2.2 +# data-prep-connector==0.2.2 ## HTML processing trafilatura From b6e46e06585db0f853dcd6e07935c9a3cf25e36f Mon Sep 17 00:00:00 2001 From: Sujee Maniyam Date: Tue, 26 Nov 2024 00:33:27 -0800 Subject: [PATCH 06/31] html processing working example Signed-off-by: Sujee Maniyam --- .gitignore | 4 + .../html-processing/1_crawl_site.ipynb | 51 +- ...n.ipynb => 2_extract_text_from_html.ipynb} | 201 ++--- .../html-processing/3_save_to_vector_db.ipynb | 744 ++++++++++++++++++ .../notebooks/html-processing/4_query.ipynb | 246 ++++++ examples/notebooks/html-processing/README.md | 14 +- .../notebooks/html-processing/env.sample.txt | 1 + .../notebooks/html-processing/my_config.py | 22 +- .../html-processing/requirements.txt | 15 +- 9 files changed, 1150 insertions(+), 148 deletions(-) rename examples/notebooks/html-processing/{2_process_html_python.ipynb => 2_extract_text_from_html.ipynb} (72%) create mode 100644 examples/notebooks/html-processing/3_save_to_vector_db.ipynb create mode 100644 examples/notebooks/html-processing/4_query.ipynb create mode 100644 examples/notebooks/html-processing/env.sample.txt diff --git a/.gitignore b/.gitignore index 8ad2d9a1f..c704e0842 100644 --- a/.gitignore +++ b/.gitignore @@ -146,3 +146,7 @@ dmypy.json # Ignore artifacts directory created by the `make save-images` target artifacts/ + +*.lock + +*.db diff --git a/examples/notebooks/html-processing/1_crawl_site.ipynb b/examples/notebooks/html-processing/1_crawl_site.ipynb index c65c48a54..f60824eff 100644 --- a/examples/notebooks/html-processing/1_crawl_site.ipynb +++ b/examples/notebooks/html-processing/1_crawl_site.ipynb @@ -91,37 +91,54 @@ "metadata": {}, "outputs": [], "source": [ + "%%capture\n", + "\n", "## must enable nested asynchronous io in a notebook as the crawler uses \n", "## coroutine to speed up acquisition and downloads\n", "\n", "import nest_asyncio\n", - "nest_asyncio.apply()" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "%%capture\n", + "nest_asyncio.apply()\n", + "\n", "from dpk_web2parquet.transform import Web2Parquet\n", "\n", - "Web2Parquet(urls= ['https://thealliance.ai/'],\n", - " depth=2, \n", - " downloads=10,\n", - " folder='input').transform()\n", - "print (\"✅ web crawl completed\")" + "Web2Parquet(urls= [MY_CONFIG.CRAWL_URL_BASE],\n", + " depth=MY_CONFIG.CRAWL_MAX_DEPTH, \n", + " downloads=MY_CONFIG.CRAWL_MAX_DOWNLOADS,\n", + " folder=MY_CONFIG.INPUT_DIR).transform()" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Exception in callback BaseAsyncIOLoop._handle_events(8, 1)\n", + "handle: \n", + "Traceback (most recent call last):\n", + " File \"/home/sujee/apps/anaconda3/envs/dpk-html-processing-2-022dev3-py312/lib/python3.12/asyncio/events.py\", line 88, in _run\n", + " self._context.run(self._callback, *self._args)\n", + "RuntimeError: cannot enter context: <_contextvars.Context object at 0x70577a041080> is already entered\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ web crawl completed. Downloaded 20 files into 'input' directory\n" + ] + } + ], "source": [ + "\n", "# list some files\n", "\n", + "print (f\"✅ web crawl completed. Downloaded {len(os.listdir(MY_CONFIG.INPUT_DIR))} files into '{MY_CONFIG.INPUT_DIR}' directory\", )\n", + "\n", + "\n", "# for item in os.listdir(MY_CONFIG.INPUT_DIR):\n", "# print(item)" ] diff --git a/examples/notebooks/html-processing/2_process_html_python.ipynb b/examples/notebooks/html-processing/2_extract_text_from_html.ipynb similarity index 72% rename from examples/notebooks/html-processing/2_process_html_python.ipynb rename to examples/notebooks/html-processing/2_extract_text_from_html.ipynb index 6dd94751b..6dea17a74 100644 --- a/examples/notebooks/html-processing/2_process_html_python.ipynb +++ b/examples/notebooks/html-processing/2_extract_text_from_html.ipynb @@ -83,29 +83,37 @@ "name": "stderr", "output_type": "stream", "text": [ - "23:40:23 INFO - html2parquet parameters are : {'output_format': , 'favor_precision': , 'favor_recall': }\n", - "23:40:23 INFO - pipeline id pipeline_id\n", - "23:40:23 INFO - code location None\n", - "23:40:23 INFO - data factory data_ is using local data access: input_folder - input output_folder - output/1-html2parquet\n", - "23:40:23 INFO - data factory data_ max_files -1, n_sample -1\n", - "23:40:23 INFO - data factory data_ Not using data sets, checkpointing False, max files -1, random samples -1, files to use ['.html', '.zip'], files to checkpoint ['.parquet']\n", - "23:40:23 INFO - orchestrator html2parquet started at 2024-11-21 23:40:23\n", - "23:40:23 INFO - Number of files is 12, source profile {'max_file_size': 0.23389148712158203, 'min_file_size': 0.0925445556640625, 'total_file_size': 1.409872055053711}\n", - "23:40:23 INFO - Completed 1 files (8.33%) in 0.003 min\n", - "23:40:23 INFO - Completed 2 files (16.67%) in 0.004 min\n", - "23:40:23 INFO - Completed 3 files (25.0%) in 0.004 min\n", - "23:40:23 INFO - Completed 4 files (33.33%) in 0.004 min\n", - "23:40:23 INFO - Completed 5 files (41.67%) in 0.004 min\n", - "23:40:23 INFO - Completed 6 files (50.0%) in 0.004 min\n", - "23:40:23 INFO - Completed 7 files (58.33%) in 0.004 min\n", - "23:40:23 INFO - Completed 8 files (66.67%) in 0.005 min\n", - "23:40:23 INFO - Completed 9 files (75.0%) in 0.005 min\n", - "23:40:23 INFO - Completed 10 files (83.33%) in 0.005 min\n", - "23:40:23 INFO - Completed 11 files (91.67%) in 0.005 min\n", - "23:40:23 INFO - Completed 12 files (100.0%) in 0.005 min\n", - "23:40:23 INFO - Done processing 12 files, waiting for flush() completion.\n", - "23:40:23 INFO - done flushing in 0.0 sec\n", - "23:40:23 INFO - Completed execution in 0.005 min, execution result 0\n" + "00:23:09 INFO - html2parquet parameters are : {'output_format': , 'favor_precision': , 'favor_recall': }\n", + "00:23:09 INFO - pipeline id pipeline_id\n", + "00:23:09 INFO - code location None\n", + "00:23:09 INFO - data factory data_ is using local data access: input_folder - input output_folder - output/1-html2parquet\n", + "00:23:09 INFO - data factory data_ max_files -1, n_sample -1\n", + "00:23:09 INFO - data factory data_ Not using data sets, checkpointing False, max files -1, random samples -1, files to use ['.html', '.zip'], files to checkpoint ['.parquet']\n", + "00:23:09 INFO - orchestrator html2parquet started at 2024-11-26 00:23:09\n", + "00:23:09 INFO - Number of files is 20, source profile {'max_file_size': 0.23515033721923828, 'min_file_size': 0.0885457992553711, 'total_file_size': 2.5425233840942383}\n", + "00:23:10 INFO - Completed 1 files (5.0%) in 0.003 min\n", + "00:23:10 INFO - Completed 2 files (10.0%) in 0.003 min\n", + "00:23:10 INFO - Completed 3 files (15.0%) in 0.004 min\n", + "00:23:10 INFO - Completed 4 files (20.0%) in 0.004 min\n", + "00:23:10 INFO - Completed 5 files (25.0%) in 0.004 min\n", + "00:23:10 INFO - Completed 6 files (30.0%) in 0.005 min\n", + "00:23:10 INFO - Completed 7 files (35.0%) in 0.005 min\n", + "00:23:10 INFO - Completed 8 files (40.0%) in 0.005 min\n", + "00:23:10 INFO - Completed 9 files (45.0%) in 0.005 min\n", + "00:23:10 INFO - Completed 10 files (50.0%) in 0.005 min\n", + "00:23:10 INFO - Completed 11 files (55.0%) in 0.006 min\n", + "00:23:10 INFO - Completed 12 files (60.0%) in 0.006 min\n", + "00:23:10 INFO - Completed 13 files (65.0%) in 0.006 min\n", + "00:23:10 INFO - Completed 14 files (70.0%) in 0.006 min\n", + "00:23:10 INFO - Completed 15 files (75.0%) in 0.006 min\n", + "00:23:10 INFO - Completed 16 files (80.0%) in 0.006 min\n", + "00:23:10 INFO - Completed 17 files (85.0%) in 0.007 min\n", + "00:23:10 INFO - Completed 18 files (90.0%) in 0.007 min\n", + "00:23:10 INFO - Completed 19 files (95.0%) in 0.007 min\n", + "00:23:10 INFO - Completed 20 files (100.0%) in 0.007 min\n", + "00:23:10 INFO - Done processing 20 files, waiting for flush() completion.\n", + "00:23:10 INFO - done flushing in 0.0 sec\n", + "00:23:10 INFO - Completed execution in 0.007 min, execution result 0\n" ] }, { @@ -113,8 +121,8 @@ "output_type": "stream", "text": [ "✅ Job completed successfully\n", - "CPU times: user 1.15 s, sys: 1.01 s, total: 2.17 s\n", - "Wall time: 1.28 s\n" + "CPU times: user 1.3 s, sys: 1.06 s, total: 2.36 s\n", + "Wall time: 1.42 s\n" ] } ], @@ -170,7 +178,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Output dimensions (rows x columns)= (12, 6)\n" + "Output dimensions (rows x columns)= (20, 6)\n" ] }, { @@ -210,7 +218,7 @@ " On August 8th, The AI Alliance, in collaborati...\n", " 7802bb7e50653e6b21f571b28843fd9a4bcf5023eaab3a...\n", " 3151\n", - " 2024-11-21T23:40:23.824364\n", + " 2024-11-26T00:23:10.251906\n", " \n", " \n", " 1\n", @@ -219,7 +227,7 @@ " The AI Alliance, a community of technology cre...\n", " bbfed07faf040c9f276df43207437f0501cf9da14ec956...\n", " 7184\n", - " 2024-11-21T23:40:23.877930\n", + " 2024-11-26T00:23:10.303543\n", " \n", " \n", " 2\n", @@ -228,25 +236,25 @@ " # Open Foundation Models and Datasets\\n\\n### E...\n", " cace8c007c2c65b7a92d9f152b7e012502b1614205e7c9...\n", " 4499\n", - " 2024-11-21T23:40:23.906316\n", + " 2024-11-26T00:23:10.341442\n", " \n", " \n", " 3\n", - " thealliance_ai_focus-areas-hardware-enablement...\n", - " thealliance_ai_focus-areas-hardware-enablement...\n", - " # Hardware Enablement\\n\\n### Fostering a vibra...\n", - " ddd2ecd7f5cc71f661f41f80d95525b1a6478d0fbff231...\n", - " 582\n", - " 2024-11-21T23:40:23.918782\n", + " thealliance_ai_focus-areas-skills-education_te...\n", + " thealliance_ai_focus-areas-skills-education_te...\n", + " # Skills & Education\\n\\n### Supporting global ...\n", + " d98ef830df5e293bb7903e021b60194e8b4e529ef4824b...\n", + " 334\n", + " 2024-11-26T00:23:10.362269\n", " \n", " \n", " 4\n", - " thealliance_ai_community_text.html\n", - " thealliance_ai_community_text.html\n", - " # Join the Community\\n\\nJoin leading AI innova...\n", - " eaa326274fea96c05c270dd7527faeb070c6ca704e63ae...\n", - " 2957\n", - " 2024-11-21T23:40:23.838481\n", + " thealliance_ai_focus-areas-applications-and-to...\n", + " thealliance_ai_focus-areas-applications-and-to...\n", + " ![abstract gradient](https://images.prismic.io...\n", + " 37752caba69be871c683c399ca2d5ab2afbec4d2623563...\n", + " 568\n", + " 2024-11-26T00:23:10.333829\n", " \n", " \n", "\n", @@ -257,36 +265,36 @@ "0 thealliance_ai_blog-open-source-ai-demo-night-... \n", "1 thealliance_ai_core-projects-sb1047_text.html \n", "2 thealliance_ai_focus-areas-foundation-models-d... \n", - "3 thealliance_ai_focus-areas-hardware-enablement... \n", - "4 thealliance_ai_community_text.html \n", + "3 thealliance_ai_focus-areas-skills-education_te... \n", + "4 thealliance_ai_focus-areas-applications-and-to... \n", "\n", " document \\\n", "0 thealliance_ai_blog-open-source-ai-demo-night-... \n", "1 thealliance_ai_core-projects-sb1047_text.html \n", "2 thealliance_ai_focus-areas-foundation-models-d... \n", - "3 thealliance_ai_focus-areas-hardware-enablement... \n", - "4 thealliance_ai_community_text.html \n", + "3 thealliance_ai_focus-areas-skills-education_te... \n", + "4 thealliance_ai_focus-areas-applications-and-to... \n", "\n", " contents \\\n", "0 On August 8th, The AI Alliance, in collaborati... \n", "1 The AI Alliance, a community of technology cre... \n", "2 # Open Foundation Models and Datasets\\n\\n### E... \n", - "3 # Hardware Enablement\\n\\n### Fostering a vibra... \n", - "4 # Join the Community\\n\\nJoin leading AI innova... \n", + "3 # Skills & Education\\n\\n### Supporting global ... \n", + "4 ![abstract gradient](https://images.prismic.io... \n", "\n", " document_id size \\\n", "0 7802bb7e50653e6b21f571b28843fd9a4bcf5023eaab3a... 3151 \n", "1 bbfed07faf040c9f276df43207437f0501cf9da14ec956... 7184 \n", "2 cace8c007c2c65b7a92d9f152b7e012502b1614205e7c9... 4499 \n", - "3 ddd2ecd7f5cc71f661f41f80d95525b1a6478d0fbff231... 582 \n", - "4 eaa326274fea96c05c270dd7527faeb070c6ca704e63ae... 2957 \n", + "3 d98ef830df5e293bb7903e021b60194e8b4e529ef4824b... 334 \n", + "4 37752caba69be871c683c399ca2d5ab2afbec4d2623563... 568 \n", "\n", " date_acquired \n", - "0 2024-11-21T23:40:23.824364 \n", - "1 2024-11-21T23:40:23.877930 \n", - "2 2024-11-21T23:40:23.906316 \n", - "3 2024-11-21T23:40:23.918782 \n", - "4 2024-11-21T23:40:23.838481 " + "0 2024-11-26T00:23:10.251906 \n", + "1 2024-11-26T00:23:10.303543 \n", + "2 2024-11-26T00:23:10.341442 \n", + "3 2024-11-26T00:23:10.362269 \n", + "4 2024-11-26T00:23:10.333829 " ] }, "execution_count": 4, @@ -356,6 +364,8 @@ "name": "stdout", "output_type": "stream", "text": [ + "content length: 3151 \n", + "\n", "On August 8th, The AI Alliance, in collaboration with Cerebral Valley and Ollama, hosted Open Source AI Demo Night in San Francisco, bringing together more than 200+ developers and innovators to showcase and celebrate the latest advances in open-source AI. There were 7 demo teams and a panel discussion on [why open technologies and communities are essential to driving innovation in California](https://youtu.be/tOXzyHJvOKw).\n", "\n", "The demo teams included:\n", @@ -368,10 +378,8 @@ } ], "source": [ - "import pprint \n", - "\n", - "# pprint.pprint (output_df.iloc[0,]['contents'])\n", - "\n", + "## Display markdown text\n", + "print ('content length:', len(output_df.iloc[0,]['contents']), '\\n')\n", "print (output_df.iloc[0,]['contents'])\n" ] }, @@ -401,6 +409,7 @@ } ], "source": [ + "## display markdown in pretty format\n", "from IPython.display import Markdown\n", "display(Markdown(output_df.iloc[0,]['contents']))\n" ] @@ -421,66 +430,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Index: 0\n", - "html_file': thealliance_ai_blog-open-source-ai-demo-night-sf-2024_text.html\n", - "base_name': thealliance_ai_blog-open-source-ai-demo-night-sf-2024_text\n", - "md_output_file': output/2-markdown/thealliance_ai_blog-open-source-ai-demo-night-sf-2024_text.md\n", - "\n", - "Index: 1\n", - "html_file': thealliance_ai_core-projects-sb1047_text.html\n", - "base_name': thealliance_ai_core-projects-sb1047_text\n", - "md_output_file': output/2-markdown/thealliance_ai_core-projects-sb1047_text.md\n", - "\n", - "Index: 2\n", - "html_file': thealliance_ai_focus-areas-foundation-models-datasets_text.html\n", - "base_name': thealliance_ai_focus-areas-foundation-models-datasets_text\n", - "md_output_file': output/2-markdown/thealliance_ai_focus-areas-foundation-models-datasets_text.md\n", - "\n", - "Index: 3\n", - "html_file': thealliance_ai_focus-areas-hardware-enablement_text.html\n", - "base_name': thealliance_ai_focus-areas-hardware-enablement_text\n", - "md_output_file': output/2-markdown/thealliance_ai_focus-areas-hardware-enablement_text.md\n", - "\n", - "Index: 4\n", - "html_file': thealliance_ai_community_text.html\n", - "base_name': thealliance_ai_community_text\n", - "md_output_file': output/2-markdown/thealliance_ai_community_text.md\n", - "\n", - "Index: 5\n", - "html_file': thealliance_ai_become-a-collaborator_text.html\n", - "base_name': thealliance_ai_become-a-collaborator_text\n", - "md_output_file': output/2-markdown/thealliance_ai_become-a-collaborator_text.md\n", - "\n", - "Index: 6\n", - "html_file': thealliance_ai_core-projects-ntia_request_text.html\n", - "base_name': thealliance_ai_core-projects-ntia_request_text\n", - "md_output_file': output/2-markdown/thealliance_ai_core-projects-ntia_request_text.md\n", - "\n", - "Index: 7\n", - "html_file': thealliance_ai_focus-areas-advocacy_text.html\n", - "base_name': thealliance_ai_focus-areas-advocacy_text\n", - "md_output_file': output/2-markdown/thealliance_ai_focus-areas-advocacy_text.md\n", - "\n", - "Index: 8\n", - "html_file': thealliance_ai_contact_text.html\n", - "base_name': thealliance_ai_contact_text\n", - "md_output_file': output/2-markdown/thealliance_ai_contact_text.md\n", - "\n", - "Index: 9\n", - "html_file': thealliance_ai_core-projects-trusted-evals_text.html\n", - "base_name': thealliance_ai_core-projects-trusted-evals_text\n", - "md_output_file': output/2-markdown/thealliance_ai_core-projects-trusted-evals_text.md\n", - "\n", - "Index: 10\n", - "html_file': thealliance_ai__text.html\n", - "base_name': thealliance_ai__text\n", - "md_output_file': output/2-markdown/thealliance_ai__text.md\n", - "\n", - "Index: 11\n", - "html_file': thealliance_ai_contribute_text.html\n", - "base_name': thealliance_ai_contribute_text\n", - "md_output_file': output/2-markdown/thealliance_ai_contribute_text.md\n", - "\n" + "✅ Saved 20 md files into 'output/2-markdown'\n" ] } ], @@ -488,20 +438,19 @@ "import os\n", "\n", "for index, row in output_df.iterrows():\n", - " print(f\"Index: {index}\")\n", " html_file = row['document']\n", " base_name = os.path.splitext(os.path.basename(html_file))[0]\n", " md_output_file = os.path.join(MY_CONFIG.OUTPUT_DIR_MARKDOWN, base_name + '.md')\n", - " print(f\"html_file': {html_file}\")\n", - " print(f\"base_name': {base_name}\")\n", - " print(f\"md_output_file': {md_output_file}\")\n", + " # print(f\"html_file': {html_file}\")\n", + " # print(f\"base_name': {base_name}\")\n", + " # print(f\"md_output_file': {md_output_file}\")\n", + " # print()\n", " \n", " with open(md_output_file, 'w') as md_output_file_handle:\n", " md_output_file_handle.write (row['contents'])\n", - " \n", - " \n", - " print()\n", - " " + "# -- end loop --- \n", + "\n", + "print (f\"✅ Saved {index+1} md files into '{MY_CONFIG.OUTPUT_DIR_MARKDOWN}'\")" ] } ], diff --git a/examples/notebooks/html-processing/3_save_to_vector_db.ipynb b/examples/notebooks/html-processing/3_save_to_vector_db.ipynb new file mode 100644 index 000000000..9729c1956 --- /dev/null +++ b/examples/notebooks/html-processing/3_save_to_vector_db.ipynb @@ -0,0 +1,744 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Save Markdown text into Vector DB" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from my_config import MY_CONFIG" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import glob\n", + "\n", + "pattern = os.path.join(MY_CONFIG.OUTPUT_DIR_MARKDOWN, '*.md')\n", + "\n", + "md_file_count = len(glob.glob(pattern, recursive=True)) " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded 74 chunks from 20 files\n" + ] + } + ], + "source": [ + "from llama_index.core import SimpleDirectoryReader\n", + "\n", + "reader = SimpleDirectoryReader(input_dir=MY_CONFIG.OUTPUT_DIR_MARKDOWN, recursive=True )\n", + "documents = reader.load_data()\n", + "\n", + "print (f\"Loaded {len(documents)} chunks from {md_file_count} files\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Doc ID: c444d3d4-f3a9-4e86-9cbe-1f86ea82f226\n", + "Text: !\n", + "---\n", + "Doc ID: 072ecafe-306c-4138-96a2-ef1eb56a7d01\n", + "Text: Open Source AI Demo Night The AI Alliance, in collaboration\n", + "with Cerebral Valley and Ollama, hosted Open Source AI Demo Night in\n", + "San Francisco, bringing together more than 200+ developers and\n", + "innovators to showcase and celebrate the latest advances in open-\n", + "source AI.\n", + "---\n", + "Doc ID: 9d05c74f-113f-4632-a0d6-e62616b56a77\n", + "Text: AI Alliance members plan to start or enhance projects that meet\n", + "the following objectives: Deploy benchmarks, tools, and other\n", + "resources that enable the responsible development and use of AI\n", + "systems at a global scale, including the creation of a catalog of\n", + "vetted safety, security, and trust tools. Support the advocacy and\n", + "enablement of these to...\n", + "---\n", + "Doc ID: 9fc93e5e-c6c0-4cc8-b57e-b5b687b5cef9\n", + "Text: Meet the AI Alliance The AI Alliance is a collaborative network\n", + "of companies, startups, universities, research institutions,\n", + "government organizations, and non-profit foundations that are working\n", + "at the forefront of AI technology, applications, and governance. We\n", + "unite top developers, scientists, academic institutions, companies,\n", + "and policymake...\n", + "---\n", + "Doc ID: cb0afc89-b0ac-4ed0-a2ad-c01cbc788fa1\n", + "Text: Become a collaborator Want to get involved? Apply to join one\n", + "of our working groups or major initiatives. Our team will review your\n", + "request and get back to you as soon as possible (within 30 days or\n", + "fewer). Want to get involved? Apply to join one of our working groups\n", + "or major initiatives. Our team will review your request and get back\n", + "to yo...\n", + "---\n", + "Doc ID: b7b521fe-661b-47ef-b0dc-ee13064b636f\n", + "Text: **AI Alliance Expands with Seven New Members from India**\n", + "---\n", + "Doc ID: 0bf8007c-bc73-48cf-8406-8a63acb57789\n", + "Text: Strengthening the Country's Role in Shaping the Global AI\n", + "Landscape through Open, Safe, and Responsible AI Practices **INDIA,\n", + "Bengaluru, September 17, 2024** –The AI Alliance, a global community\n", + "dedicated to advancing safe and responsible AI through open\n", + "innovation, has expanded its membership to include seven leading\n", + "Indian organizations: AI4B...\n", + "---\n", + "Doc ID: d4975e3d-8f06-4b4b-a608-4487af9b9359\n", + "Text: AI Alliance Launches as an International Community of Leading\n", + "Technology Developers, Researchers, and Adopters Collaborating\n", + "Together to Advance Open, Safe, Responsible AI !abstract gradient AI\n", + "advancements are leading to new opportunities that can improve how we\n", + "work, live, learn and interact with one another. Open and transparent\n", + "innovation ...\n", + "---\n", + "Doc ID: b5768ab8-090c-43e3-8ab5-6b07f86967ca\n", + "Text: Who We Are To ensure open innovation in AI benefits everyone\n", + "and that it is built responsibly, the AI Alliance consists of a broad\n", + "range of organizations that are working across aspects of AI\n", + "education, research, development and deployment, and governance. We\n", + "are: - The creators of the tooling driving AI benchmarking, trust and\n", + "validation met...\n", + "---\n", + "Doc ID: bce66f25-7717-4fe8-b075-b994c9924374\n", + "Text: How the AI Alliance Will Work The AI Alliance will begin its\n", + "work with the formation of member-driven working groups across all\n", + "major topical areas listed above. The Alliance will also establish a\n", + "governing board and technical oversight committee dedicated to\n", + "advancing the above project areas, as well as establishing overall\n", + "project standards a...\n", + "---\n", + "Doc ID: 361dff47-895b-46ac-8844-e34e6e2cdfa4\n", + "Text: On August 8th, The AI Alliance, in collaboration with Cerebral\n", + "Valley and Ollama, hosted Open Source AI Demo Night in San Francisco,\n", + "bringing together more than 200+ developers and innovators to showcase\n", + "and celebrate the latest advances in open-source AI. There were 7 demo\n", + "teams and a panel discussion on why open technologies and communities\n", + "ar...\n", + "---\n", + "Doc ID: 33f0d501-5004-4587-8299-947879f19ed1\n", + "Text: Join the Community Join leading AI innovators, researchers,\n", + "developers, engineers, and early enterprise adopters in our goal to\n", + "advance open, safe, and responsible AI. Follow the links on this page\n", + "to subscribe to our community newsletter, contribute to a working\n", + "group, join The Alliance, follow us on social media, or attend one of\n", + "our upcoming...\n", + "---\n", + "Doc ID: 65d9d165-0267-432e-a367-9f807379da2e\n", + "Text: Subscribe to our community newsletter Get updates about\n", + "everything that’s happening in the community and be the first to know\n", + "about new projects, new resources, upcoming events, opportunities to\n", + "collaborate and contribute, etc. To subscribe, please fill out the\n", + "newsletter sign-up form in the footer below.\n", + "---\n", + "Doc ID: 96c15f94-7ce6-41f7-a421-5f8d5fa1143c\n", + "Text: Contribute code We strive to provide a clear and simple path\n", + "for anyone to join our community and contribute to AI Alliance\n", + "projects, whether they are part of an existing member organization, or\n", + "not. Learn more...\n", + "---\n", + "Doc ID: 59003bc2-9599-42ab-81c1-28e8159df43c\n", + "Text: Become a collaborator in one of our working groups The\n", + "heartbeat of the AI Alliance is our working groups and major\n", + "initiatives! Since launch, they have rolled out core projects and\n", + "affiliated projects – with more to come: - Skills, Education, and\n", + "Exploratory Research WG - AI Trust and Safety WG - Application and\n", + "Tools WG - Diverse Hardware En...\n", + "---\n", + "Doc ID: 9c998bb3-0511-407e-a483-21cc3b7ad9a3\n", + "Text: Join The Alliance If your organization would like to join the\n", + "AI Alliance as an official member, please use the contact form, and\n", + "someone on our membership team will get back to you. Thanks!\n", + "---\n", + "Doc ID: c2a891e8-cc9a-4a5a-9227-43ca39203084\n", + "Text: Attend an AI Alliance event - Oct 21–24: OpenModelCamp at\n", + "TechXChange Community Dayin Las Vegas, NV - Oct 26: TechEquity AI\n", + "Summitin Mountain View, CA - Oct 30: AI for Drug Discovery Open\n", + "Innovation Forumin Boston, MA - Nov 6–8: PyData NYCin New York, NY -\n", + "Nov 7: Data, Cloud and AI in Singapore Meetup: “Leading AI:\n", + "Collaboration Meets Ethical I...\n", + "---\n", + "Doc ID: e111dae0-7bc4-4965-9f67-c271b1d953c9\n", + "Text: Follow us on social media - Linkedin:\n", + "https://linkedin.com/company/the-aialliance/ - X/Twitter:\n", + "https://x.com/thealliance_ai - Instagram:\n", + "https://www.instagram.com/aialliance/ - Threads:\n", + "https://www.threads.net/@aialliance - YouTube:\n", + "https://www.youtube.com/@thealliance_ai\n", + "---\n", + "Doc ID: d6f2d050-d746-40b5-aebe-4ae2623764b2\n", + "Text: Contact us We’re here to help and answer any questions you may\n", + "have regarding the AI Alliance. We look forward to hearing from you!\n", + "Please ensure to include enough detail so we can route your inquiry to\n", + "the right team member. Thanks! We’re here to help and answer any\n", + "questions you may have regarding the AI Alliance. We look forward to\n", + "hearing...\n", + "---\n", + "Doc ID: a63b9dab-9b6a-4e85-aedb-a163eeddf3a9\n", + "Text: Join us in building the open future of AI We strive to provide\n", + "a clear and simple path for anyone to join our community and\n", + "contribute to AI Alliance projects, whether they are part of an\n", + "existing member organization, or not.\n", + "---\n", + "Doc ID: 6972c48f-b8f4-4b62-b063-bdeb24106511\n", + "Text: We are excited to welcome contributions to AI Alliance Core and\n", + "Affiliated projects. **Core Projects** Core projects address\n", + "substantial cross-community challenges and are an opportunity for\n", + "individual contributors and members to collaborate, build, and make an\n", + "impact on the future of AI. Core Projects are managed directly by the\n", + "AI Alliance a...\n", + "---\n", + "Doc ID: 89b59840-4a4a-4d2e-b17f-f1defcdf81e0\n", + "Text: Making a contribution The AI Alliance utilizes the Linux\n", + "Foundation’s Developer Certificate of Origin (DCO) on a per-commit\n", + "basis to enable contributions to Core Projects. The DCO does not\n", + "require the committer to give away ownership of the contribution -\n", + "your IP remains yours and others' IP remains theirs. **Core Project\n", + "Licenses** AI Allian...\n", + "---\n", + "Doc ID: 848f3a37-9c57-49f0-ae98-9c5d28f497f0\n", + "Text: Responding to the U.S. NTIA request for comment on Dual Use\n", + "Foundation Artificial Intelligence Models with Widely Available Model\n", + "Weights Project !abstract gradient The AI Alliance recently drafted\n", + "a response to the NTIA’s request for comment on “Dual Use Foundation\n", + "Artificial Intelligence Models with Widely Available Model Weights.”\n", + "The requ...\n", + "---\n", + "Doc ID: 80dd54c2-36b5-43ec-b606-02ae6e7634c2\n", + "Text: The AI Alliance, a community of technology creators, developers,\n", + "and adopters collaborating to advance safe, responsible AI rooted in\n", + "open innovation, respectfully opposes SB 1047, the proposed *Safe and\n", + "Secure Innovation for Frontier Artificial Intelligence Models Act*. It\n", + "would slow innovation, thwart advancements in safety and security, and\n", + "u...\n", + "---\n", + "Doc ID: a68d64ee-1a21-45ef-a467-5560e20b54a2\n", + "Text: Context Model evaluation, or ‘evals’, in the generative AI era\n", + "is simultaneously one of the most important areas of investment but\n", + "also one of the highest areas of entropy. Our goals for the Trusted\n", + "Evals working group are to: for community efforts around trust and\n", + "safety – including, but not limited to, work happening globally in\n", + "various lan...\n", + "---\n", + "Doc ID: 26733281-e36f-4d28-b23e-75b8735184aa\n", + "Text: Request for proposal We are seeking new perspectives in the AI\n", + "evaluation domain. To foster further innovation in this area, we are\n", + "pleased to invite the community to participate in the AI Alliance’s\n", + "Trusted Evals working group by submitting a response to our Request\n", + "for Proposal to be included in the working group’s efforts to:\n", + "**Raise awaren...\n", + "---\n", + "Doc ID: 39765acd-c476-413d-adf6-8b4ff81c6044\n", + "Text: Areas of interest - Cybersecurity threats - Sensitive data\n", + "detection including areas such as toxic content (e.g. hate speech),\n", + "personally identifiable information (PII), bias, etc. - Model\n", + "performance including helpfulness, quality, alignment, robustness,\n", + "etc., (as opposed to operational concerns like throughput, latency,\n", + "scalability, etc.) - K...\n", + "---\n", + "Doc ID: e60b0a5d-54f0-42f4-9e57-36e909a87447\n", + "Text: Requirements Proposals should include a link to a summary of\n", + "your project, in English, explaining the area of focus, dataset\n", + "description and any relevant prior work. This includes: - Name of\n", + "project and abstract - Core team member bios & affiliations - Link to\n", + "website (if applicable) - Link to GitHub repository - License - Links\n", + "to whitepapers...\n", + "---\n", + "Doc ID: 4504925f-505d-4669-8fff-4716501b64ee\n", + "Text: !\n", + "---\n", + "Doc ID: b6327a28-d0d2-4e6d-af26-dff93944eeec\n", + "Text: Statement from AI Alliance Regarding Commerce's Final Report on\n", + "Open-Weights Models Commerce Department has taken a crucial step\n", + "towards ensuring that the U.S. remains at the forefront of AI\n", + "development by concluding that \"the government should not restrict the\n", + "wide availability of model weights for dual-use foundation models.\"\n", + "---\n", + "Doc ID: 9b6971ae-ba9b-4cce-92c4-1b7b17bc461f\n", + "Text: !abstract gradient\n", + "---\n", + "Doc ID: 5a1f19ef-a866-4b0b-b334-7bf2609bd361\n", + "Text: Advancing Domain-Specific Q&A: The AI Alliance's Guide to Best\n", + "Practices The AI Alliance application and tools working group has\n", + "conducted a comprehensive study on best practices for advancing\n", + "domain-specific Q&A using retrieval-augmented generation (RAG)\n", + "techniques. The findings of this research, provide insights and\n", + "recommendations for maximi...\n", + "---\n", + "Doc ID: 182c4a8e-6d2b-4ddd-a7c8-e952d8bfd340\n", + "Text: Open Foundation Models and Datasets\n", + "---\n", + "Doc ID: 1b25bcf0-5c97-4802-a7c0-51b7ed77e259\n", + "Text: Enabling an ecosystem of open foundation models, including those\n", + "with multilingual and multi-modal capabilities, and open datasets. We\n", + "are responsibly enhancing the ecosystem of open foundation models and\n", + "datasets. We are embracing multilingual and multimodal models, as well\n", + "as science models tackling broad societal issues like climate change\n", + "a...\n", + "---\n", + "Doc ID: 1dca16b0-68b4-43b1-bb32-40037254a37b\n", + "Text: Working groups\n", + "---\n", + "Doc ID: f2993603-7b32-476c-89e2-088707af5ecc\n", + "Text: AI for Drug Discovery This working group aims to create a\n", + "world-class research community that harnesses the potential of AI\n", + "foundation models, transforms the field of drug discovery, and\n", + "accelerates scientific progress by driving interdisciplinary\n", + "collaboration on AI-powered drug discovery projects in the open.\n", + "!abstract gradient\n", + "---\n", + "Doc ID: cf9ee4db-3c01-4a9f-ae00-03a1d902221e\n", + "Text: Foundation Models !abstract gradient\n", + "---\n", + "Doc ID: 39126747-9240-48c2-bd8e-b4a0eb5b7952\n", + "Text: Materials and Chemistry We aim to curate datasets, tasks and\n", + "benchmarks for materials science, build out foundation models in\n", + "chemistry for prediction of properties, experimental outcomes or\n", + "generation of new candidates and create a framework to foster\n", + "collaboration between human experts and AI agents that will ultimately\n", + "help solve global urge...\n", + "---\n", + "Doc ID: e51a162b-9e3c-46e1-b9aa-89f65b8a2436\n", + "Text: Current or recent projects !Home page for the Open Trusted Data\n", + "Initiative !Home page for the Open Trusted Data Initiative\n", + "---\n", + "Doc ID: 86ba2b6c-7631-4f0d-bfa3-6eac283d95c1\n", + "Text: Open Trusted Data Initiative Foundation Models and Datasets\n", + "Cataloging and managing trustworthy datasets. !abstract gradient\n", + "!abstract gradient\n", + "---\n", + "Doc ID: ae8c8fe3-187e-44ed-9303-42d819e672b6\n", + "Text: Time Series Data and Model Initiative Foundation Models and\n", + "Datasets Time-series applications are an important target for AI. In\n", + "addition to gathering high-quality and fully-governed time series\n", + "datasets as part of the Open Trusted Data Initiative, Alliance members\n", + "are collaborating on new and improved time series models (as part of\n", + "the Indust...\n", + "---\n", + "Doc ID: 87ef3822-1d4f-44bf-9f28-e69fa4aa627f\n", + "Text: Industry Open FMs Initiative Foundation Models and Datasets We\n", + "have seen rapid progress in building and releasing highly-capable and\n", + "open foundation models for general language, coding, scientific\n", + "discovery, and multi-modal scenarios. A key development in model\n", + "strategies is a focus on building smaller, more specialized models.\n", + "More details ...\n", + "---\n", + "Doc ID: 529128a6-d3c4-4999-96b9-d66ac957bc38\n", + "Text: Hardware Enablement\n", + "---\n", + "Doc ID: d796f7f0-7207-4c04-bca5-75420977a45d\n", + "Text: Fostering a vibrant AI hardware accelerator ecosystem through\n", + "enabling software technology. We collaborate on the benchmarking,\n", + "optimization, and adaptation of AI workloads to accelerate innovation\n", + "in a diverse set of hardware. Our work focuses on scalability,\n", + "platform adaptation, and advanced power, energy, and carbon modeling.\n", + "Benchmarks and ...\n", + "---\n", + "Doc ID: 1094484d-f0e6-4d23-8dd2-37410e432d54\n", + "Text: Our projects Projects we are currently working on:\n", + "---\n", + "Doc ID: 4131f4b7-8c0d-4173-87fa-56a25e597eda\n", + "Text: Skills & Education\n", + "---\n", + "Doc ID: a6d026ae-524f-4704-9428-c1e657eb771b\n", + "Text: Supporting global AI skill-building, education, and exploratory\n", + "research. Our members partner with academics, industry leaders,\n", + "researchers, and students, empowering them to develop and leverage\n", + "resources to build an AI-ready workforce and engage with vital\n", + "research in AI models, algorithms, and platforms.\n", + "---\n", + "Doc ID: 55b60157-348d-4246-a8b3-214d918d48a6\n", + "Text: Trust and Safety\n", + "---\n", + "Doc ID: 4e0fed30-fc07-4271-97fc-acfb40ebbc3f\n", + "Text: Ensuring safe and trusted generative AI with benchmarks, tools,\n", + "and methodologies Our members work together to understand the\n", + "landscape of AI trust and safety risks, as well as other uses of AI\n", + "system evaluation. We identify use cases specific to various important\n", + "domains (finance, healthcare, education, and others). We work together\n", + "and with e...\n", + "---\n", + "Doc ID: 21047c6c-cf5b-4f99-950e-036d26f70f32\n", + "Text: Current or recent projects ! !\n", + "---\n", + "Doc ID: 00a45255-126b-4905-ab65-85eafed747e1\n", + "Text: Understanding AI Trust and Safety: A Living Guide A major\n", + "challenge for the successful use of AI is the importance of\n", + "understanding potential trust and safety issues, along with their\n", + "mitigation strategies. Failure to consider these issues could impact\n", + "an organization's operations and the experience of its users. Concerns\n", + "about safety are also ...\n", + "---\n", + "Doc ID: a41c8f30-470c-4598-9bc5-fa59cc27ba69\n", + "Text: Trust and Safety Evaluations Trust & Safety Much like other\n", + "software, generative AI (“GenAI”) *models* and the *AI systems* that\n", + "use them need to be trusted and useful to their users. The *Trust and\n", + "Safety Evaluations* project fills gaps in the current landscape of the\n", + "taxonomy of different kinds of evaluation, the tools for creating and\n", + "runni...\n", + "---\n", + "Doc ID: 0f230e93-6cfa-4b43-aca0-ca5edb8614da\n", + "Text: Ranking AI Safety Priorities by Domain Trust & Safety A\n", + "challenge for software development teams adopting generative AI is\n", + "making sense of the safety issues that their applications must\n", + "address. The AI safety ecosystem is broad and growing quickly, making\n", + "it difficult for these development teams to know where they should\n", + "focus their efforts. W...\n", + "---\n", + "Doc ID: 31076aa2-fa81-4ac2-b734-a6fb27cd28f4\n", + "Text: Trusted Evals request for proposals Trust & Safety The AI\n", + "Alliance Trusted Evals request for proposals is aimed at seeking new\n", + "perspectives in the AI evaluation domain. We are excited to work with\n", + "those in academia, industry, startups and anyone excited to\n", + "collaborate in the open and build an ecosystem around their work.\n", + "---\n", + "Doc ID: 97f0de37-3ab7-45d6-803f-0f298e9c56ec\n", + "Text: Accelerating responsible innovation in AI Open and transparent\n", + "innovation is essential for equipping AI researchers, developers, and\n", + "users with the knowledge and tools to leverage AI advancements safely\n", + "and inclusively, prioritizing diversity and economic opportunity for\n", + "all. Through member-driven working groups, we bring together builders\n", + "and...\n", + "---\n", + "Doc ID: 7c507e93-5f49-4086-b3d9-db761b2de087\n", + "Text: Core projects View all Core projects address substantial\n", + "cross-community challenges and are an opportunity for individual\n", + "contributors and members to collaborate, build, and make an impact on\n", + "the future of AI. Core Projects are managed directly by the AI\n", + "Alliance and governed as described below. !abstract gradient\n", + "!abstract gradient\n", + "---\n", + "Doc ID: cab794c4-78b3-438a-861c-f61a8d41ff4d\n", + "Text: A statement in opposition to California SB 1047 Advocacy Our\n", + "perspectives and recommendations in opposition to California SB 1047,\n", + "the proposed *Safe and Secure Innovation for Frontier Artificial\n", + "Intelligence Models Act*. !Trusted Evals RFP !Trusted Evals RFP\n", + "---\n", + "Doc ID: 72214679-919b-4c96-b21d-08881ef4082b\n", + "Text: Trusted Evals request for proposals Trust & Safety The AI\n", + "Alliance Trusted Evals request for proposals is aimed at seeking new\n", + "perspectives in the AI evaluation domain. We are excited to work with\n", + "those in academia, industry, startups and anyone excited to\n", + "collaborate in the open and build an ecosystem around their work.\n", + "!abstract gradient ...\n", + "---\n", + "Doc ID: c12a6b5c-2961-416d-82c6-eee525b36f59\n", + "Text: Responding to the U.S. NTIA request for comment on Dual Use\n", + "Foundation Artificial Intelligence Models with Widely Available Model\n", + "Weights Advocacy The request seeks public input on the potential\n", + "risks, benefits, and policy approaches for AI foundation models whose\n", + "weights are broadly accessible. ! !\n", + "---\n", + "Doc ID: 16c0c4ac-40b2-4be8-a6e8-eb1105274f5d\n", + "Text: Ranking AI Safety Priorities by Domain Trust & Safety A\n", + "challenge for software development teams adopting generative AI is\n", + "making sense of the safety issues that their applications must\n", + "address. The AI safety ecosystem is broad and growing quickly, making\n", + "it difficult for these development teams to know where they should\n", + "focus their efforts. W...\n", + "---\n", + "Doc ID: b6987811-2807-4e95-b4a7-387af6c37a59\n", + "Text: Open Trusted Data Initiative Foundation Models and Datasets\n", + "Cataloging and managing trustworthy datasets. !abstract gradient\n", + "!abstract gradient\n", + "---\n", + "Doc ID: 31123c95-6abe-4bd9-9e7c-b9cc5b707185\n", + "Text: Time Series Data and Model Initiative Foundation Models and\n", + "Datasets Time-series applications are an important target for AI. In\n", + "addition to gathering high-quality and fully-governed time series\n", + "datasets as part of the Open Trusted Data Initiative, Alliance members\n", + "are collaborating on new and improved time series models (as part of\n", + "the Indust...\n", + "---\n", + "Doc ID: 44b6430b-8dd4-4ecb-87f2-eef9c4dbf215\n", + "Text: AI Accelerator Software Ecosystem Guide Hardware Enablement\n", + "The **AI Accelerator Software Ecosystem Guide** provides an\n", + "introduction to the broad topic of software tools that support\n", + "seamless execution of AI workloads on different hardware accelerators.\n", + "Industry experts provide guidance on the state of the art and how to\n", + "ensure success. !abst...\n", + "---\n", + "Doc ID: c349331b-abdb-4782-b126-9fffdb3e5b81\n", + "Text: Industry Open FMs Initiative Foundation Models and Datasets We\n", + "have seen rapid progress in building and releasing highly-capable and\n", + "open foundation models for general language, coding, scientific\n", + "discovery, and multi-modal scenarios. A key development in model\n", + "strategies is a focus on building smaller, more specialized models.\n", + "More details ...\n", + "---\n", + "Doc ID: 13267e45-c786-4806-8c86-9f4f40a53fef\n", + "Text: Understanding AI Trust and Safety: A Living Guide A major\n", + "challenge for the successful use of AI is the importance of\n", + "understanding potential trust and safety issues, along with their\n", + "mitigation strategies. Failure to consider these issues could impact\n", + "an organization's operations and the experience of its users. Concerns\n", + "about safety are also ...\n", + "---\n", + "Doc ID: 83d2727e-cbe1-4ae1-96e1-62dac2d53f31\n", + "Text: The Living Guide to Applying AI Applications and Tools A\n", + "\"living guide\" for building AI-enabled applications, this guide\n", + "provides an introduction to several established design patterns for\n", + "building AI systems and products, with contributions from different\n", + "experts in the AI Alliance. Several common patterns, like RAG\n", + "(retrieval-augmented gener...\n", + "---\n", + "Doc ID: b3123acf-4a56-42de-ac6c-5b22b160c68e\n", + "Text: Trust and Safety Evaluations Trust & Safety Much like other\n", + "software, generative AI (“GenAI”) *models* and the *AI systems* that\n", + "use them need to be trusted and useful to their users. The *Trust and\n", + "Safety Evaluations* project fills gaps in the current landscape of the\n", + "taxonomy of different kinds of evaluation, the tools for creating and\n", + "runni...\n", + "---\n", + "Doc ID: 8c96d27f-1ef3-4063-835b-248fab3d48af\n", + "Text: Affiliated projects View all Affiliated projects are led and\n", + "managed by members and identified as being aligned to the AI Alliance\n", + "mission. Although Affiliated Projects are not managed by the AI\n", + "Alliance, we highlight them as contributing to open, safe, and\n", + "responsible AI. !IBM logo\n", + "---\n", + "Doc ID: fa277a88-e744-4a85-a6df-e68b1d2ddb3c\n", + "Text: IBM Granite Code Models A series of code models trained by IBM\n", + "licensed under Apache 2.0 license. Includes base pretrained and\n", + "instruct models. !\n", + "---\n", + "Doc ID: e924df60-a844-4a0e-859e-9a96a84f82a7\n", + "Text: AIConfig & Editor ! AIConfig is a config-based framework for\n", + "building generative AI applications. !\n", + "---\n", + "Doc ID: a2c51552-6e48-4614-ac74-5930ccfe49c4\n", + "Text: GenAI in Education: Usage Guidance ! A report evaluating the\n", + "feasibility, benefits, and limitations of using generative AI\n", + "technologies in an educational setting and its impact on learning\n", + "outcomes. !\n", + "---\n", + "Doc ID: ebaf6236-f3c9-454c-8807-9dc3264ad4a6\n", + "Text: LLM 360: Amber ! The Amber project includes a 7B English\n", + "language model with the LLaMA architecture, an instruction-following\n", + "model finetuned from LLM360-Amber, and a safety-finetuned instruction\n", + "model using LLM360-AmberChat as the base. !Red Hat logo\n", + "---\n", + "Doc ID: d1b13520-3056-46aa-a22a-56f2cacdddd8\n", + "Text: TrustyAI !Red Hat logo TrustyAI is an open-source toolkit\n", + "designed for responsible AI. !\n", + "---\n", + "Doc ID: f5f283f0-c274-4641-90a8-7485b91427ef\n", + "Text: Ragna ! Ragna is an open-source RAG-based (Retrieval-Augmented\n", + "Generation) AI orchestration framework designed to bridge the gap\n", + "between AI research and production deployment.\n", + "---\n" + ] + } + ], + "source": [ + "for doc in documents:\n", + " print (doc)\n", + " print ('---')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step-3: Setup Embedding Model" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# If connection to https://huggingface.co/ failed, uncomment the following path\n", + "import os\n", + "os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n", + "from llama_index.core import Settings\n", + "\n", + "Settings.embed_model = HuggingFaceEmbedding(\n", + " model_name = MY_CONFIG.EMBEDDING_MODEL\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step-4: Connect to Milvus" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ Connected Llama-index to Milvus instance: ./rag_html.db\n" + ] + } + ], + "source": [ + "# connect to vector db\n", + "from llama_index.core import VectorStoreIndex, StorageContext\n", + "from llama_index.vector_stores.milvus import MilvusVectorStore\n", + "\n", + "vector_store = MilvusVectorStore(\n", + " uri = MY_CONFIG.DB_URI ,\n", + " dim = MY_CONFIG.EMBEDDING_LENGTH , \n", + " collection_name = MY_CONFIG.COLLECTION_NAME,\n", + " overwrite=True\n", + ")\n", + "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n", + "\n", + "print (\"✅ Connected Llama-index to Milvus instance: \", MY_CONFIG.DB_URI )" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ Connected to Milvus instance: ./rag_html.db\n", + "✅ Cleared collection : docs\n" + ] + } + ], + "source": [ + "from pymilvus import MilvusClient\n", + "\n", + "milvus_client = MilvusClient(MY_CONFIG.DB_URI)\n", + "print (\"✅ Connected to Milvus instance: \", MY_CONFIG.DB_URI )\n", + "\n", + "# if we already have a collection, clear it first\n", + "if milvus_client.has_collection(collection_name = MY_CONFIG.COLLECTION_NAME):\n", + " milvus_client.drop_collection(collection_name = MY_CONFIG.COLLECTION_NAME)\n", + " print ('✅ Cleared collection :', MY_CONFIG.COLLECTION_NAME)\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ Connected Llama-index to Milvus instance: ./rag_html.db\n" + ] + } + ], + "source": [ + "# connect to vector db\n", + "from llama_index.core import VectorStoreIndex, StorageContext\n", + "from llama_index.vector_stores.milvus import MilvusVectorStore\n", + "\n", + "vector_store = MilvusVectorStore(\n", + " uri = MY_CONFIG.DB_URI ,\n", + " dim = MY_CONFIG.EMBEDDING_LENGTH , \n", + " collection_name = MY_CONFIG.COLLECTION_NAME,\n", + " overwrite=True\n", + ")\n", + "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n", + "\n", + "print (\"✅ Connected Llama-index to Milvus instance: \", MY_CONFIG.DB_URI )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step-5: Create Index and Save to DB" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ Created index: \n", + "✅ Saved index to db ./rag_html.db\n", + "CPU times: user 532 ms, sys: 129 ms, total: 661 ms\n", + "Wall time: 657 ms\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "# create an index\n", + "\n", + "from llama_index.core import VectorStoreIndex\n", + "\n", + "index = VectorStoreIndex.from_documents(\n", + " documents, storage_context=storage_context\n", + ")\n", + "print (\"✅ Created index:\", index )\n", + "print (\"✅ Saved index to db \", MY_CONFIG.DB_URI )" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dpk-html-processing-2-022dev3-py312", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/notebooks/html-processing/4_query.ipynb b/examples/notebooks/html-processing/4_query.ipynb new file mode 100644 index 000000000..53bd9d8be --- /dev/null +++ b/examples/notebooks/html-processing/4_query.ipynb @@ -0,0 +1,246 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# RAG on HTML documents\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step-1: Configuration" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from my_config import MY_CONFIG" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step-2: Setup Embeddings" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# If connection to https://huggingface.co/ failed, uncomment the following path\n", + "import os\n", + "os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n", + "from llama_index.core import Settings\n", + "\n", + "Settings.embed_model = HuggingFaceEmbedding(\n", + " model_name = MY_CONFIG.EMBEDDING_MODEL\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step-3: Connect to Milvus" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ Connected to Milvus instance: ./rag_html.db\n" + ] + } + ], + "source": [ + "# connect to vector db\n", + "from llama_index.core import VectorStoreIndex, StorageContext\n", + "from llama_index.vector_stores.milvus import MilvusVectorStore\n", + "\n", + "vector_store = MilvusVectorStore(\n", + " uri = MY_CONFIG.DB_URI ,\n", + " dim = MY_CONFIG.EMBEDDING_LENGTH , \n", + " collection_name = MY_CONFIG.COLLECTION_NAME,\n", + " overwrite=False # so we load the index from db\n", + ")\n", + "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n", + "\n", + "print (\"✅ Connected to Milvus instance: \", MY_CONFIG.DB_URI )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step-4: Load Document Index from DB" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "✅ Loaded index from vector db: ./rag_html.db\n", + "CPU times: user 113 ms, sys: 23.6 ms, total: 137 ms\n", + "Wall time: 135 ms\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "from llama_index.core import VectorStoreIndex\n", + "\n", + "index = VectorStoreIndex.from_vector_store(\n", + " vector_store=vector_store, storage_context=storage_context)\n", + "\n", + "print (\"✅ Loaded index from vector db:\", MY_CONFIG.DB_URI )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step-5: Setup LLM" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "from llama_index.llms.replicate import Replicate\n", + "from llama_index.core import Settings\n", + "\n", + "llm = Replicate(\n", + " model= MY_CONFIG.LLM_MODEL,\n", + " temperature=0.1\n", + ")\n", + "\n", + "Settings.llm = llm" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step-6: Query" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The AI Alliance is an international community of leading technology developers, researchers, and adopters collaborating together to advance open, safe, and responsible AI. It is a group of organizations across industry, startups, academia, research, and government that are committed to open innovation and open science in AI. The AI Alliance aims to foster an open community and enable developers and researchers to accelerate responsible innovation in AI while ensuring scientific rigor, trust, safety, security, diversity, and economic competitiveness. It plans to start or enhance projects that meet various objectives, including developing and deploying benchmarks and evaluation standards, advancing the ecosystem of open foundation models, fostering a vibrant AI hardware accelerator ecosystem, supporting global AI skills building and exploratory research, developing educational content and resources, and launching initiatives that encourage open development of AI in safe and beneficial ways.\n" + ] + } + ], + "source": [ + "query_engine = index.as_query_engine()\n", + "res = query_engine.query(\"What is AI Alliance?\")\n", + "# res = query_engine.query(\"What is Apache license?\")\n", + "print(res)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The demo night was held in San Francisco.\n" + ] + } + ], + "source": [ + "query_engine = index.as_query_engine()\n", + "res = query_engine.query(\"Where was the demo night held?\")\n", + "# res = query_engine.query(\"What are some Apache software?\")\n", + "print(res)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The context information provided does not include any details about the moon landing. Therefore, I cannot provide an answer to the query.\n" + ] + } + ], + "source": [ + "query_engine = index.as_query_engine()\n", + "res = query_engine.query(\"When was the moon landing?\")\n", + "print(res)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dpk-html-processing-2-022dev3-py312", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.7" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/notebooks/html-processing/README.md b/examples/notebooks/html-processing/README.md index c3189f8cd..238d84dd0 100644 --- a/examples/notebooks/html-processing/README.md +++ b/examples/notebooks/html-processing/README.md @@ -40,4 +40,16 @@ or use `wget` We will process downloaded HTML files and extract the text as markdown. The output will be saved in `output/2-markdown` directory -[2_process_html_python.ipynb](2_process_html_python.ipynb) \ No newline at end of file +[2_extract_text_from_html.ipynb](2_extract_text_from_html.ipynb) + +## Step-4: Save data into DB + +We will save the extracted text (markdown) into a vector database (Milvus) + +[3_save_to_vector_db.ipynb](3_save_to_vector_db.ipynb) + +## Step-5: Query documents + +Query documents using LLM + +[4_query.ipynb](4_query.ipynb) \ No newline at end of file diff --git a/examples/notebooks/html-processing/env.sample.txt b/examples/notebooks/html-processing/env.sample.txt new file mode 100644 index 000000000..f8466f488 --- /dev/null +++ b/examples/notebooks/html-processing/env.sample.txt @@ -0,0 +1 @@ +REPLICATE_API_TOKEN=xyz \ No newline at end of file diff --git a/examples/notebooks/html-processing/my_config.py b/examples/notebooks/html-processing/my_config.py index 18c27aad6..949c8ade3 100644 --- a/examples/notebooks/html-processing/my_config.py +++ b/examples/notebooks/html-processing/my_config.py @@ -8,8 +8,8 @@ class MyConfig: ## Crawl settings MY_CONFIG.CRAWL_URL_BASE = 'https://thealliance.ai/' -# MY_CONFIG.CRAWL_URL_BASE = 'https://thealliance.ai/our-work' -MY_CONFIG.CRAWL_MAX_DOWNLOADS = 10 +# MY_CONFIG.CRAWL_URL_BASE = 'https://apache.org/' +MY_CONFIG.CRAWL_MAX_DOWNLOADS = 20 MY_CONFIG.CRAWL_MAX_DEPTH = 2 MY_CONFIG.CRAWL_MIME_TYPE = 'text/html' @@ -19,5 +19,21 @@ class MyConfig: MY_CONFIG.OUTPUT_DIR = "output" MY_CONFIG.OUTPUT_DIR_HTML = os.path.join(MY_CONFIG.OUTPUT_DIR , "1-html2parquet") MY_CONFIG.OUTPUT_DIR_MARKDOWN = os.path.join(MY_CONFIG.OUTPUT_DIR , "2-markdown") -MY_CONFIG.OUTPUT_DIR_FINAL = os.path.join(MY_CONFIG.OUTPUT_DIR , "output_final") ### ------------------------------- + + +# MY_CONFIG.EMBEDDING_MODEL = 'sentence-transformers/all-MiniLM-L6-v2' +MY_CONFIG.EMBEDDING_MODEL = 'BAAI/bge-small-en-v1.5' +MY_CONFIG.EMBEDDING_LENGTH = 384 + + +### Milvus config +MY_CONFIG.DB_URI = './rag_html.db' # For embedded instance +MY_CONFIG.COLLECTION_NAME = 'docs' + + +## LLM Model +# MY_CONFIG.LLM_MODEL = "meta/meta-llama-3-8b-instruct" +# MY_CONFIG.LLM_MODEL = "meta/meta-llama-3-70b-instruct" +# MY_CONFIG.LLM_MODEL = "ibm-granite/granite-3.0-2b-instruct" +MY_CONFIG.LLM_MODEL = "ibm-granite/granite-3.0-8b-instruct" diff --git a/examples/notebooks/html-processing/requirements.txt b/examples/notebooks/html-processing/requirements.txt index 7fb2be3bc..3143df746 100644 --- a/examples/notebooks/html-processing/requirements.txt +++ b/examples/notebooks/html-processing/requirements.txt @@ -7,6 +7,19 @@ data-prep-toolkit-transforms[web2parquet]==0.2.2.dev3 ## HTML processing trafilatura +# Milvus +pymilvus +pymilvus[model] + +## --- Replicate +replicate + +## --- llama-index +llama-index +llama-index-embeddings-huggingface +llama-index-llms-replicate +llama-index-vector-stores-milvus + ## utils humanfriendly pandas @@ -16,4 +29,4 @@ nest-asyncio ## jupyter and friends jupyterlab ipykernel -ipywidgets \ No newline at end of file +ipywidgets From 9275f1059eed4b67b949d4e84bc62a7aa432a84b Mon Sep 17 00:00:00 2001 From: Sujee Maniyam Date: Tue, 26 Nov 2024 12:08:44 -0800 Subject: [PATCH 07/31] updated readme instructions Signed-off-by: Sujee Maniyam --- examples/notebooks/html-processing/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/examples/notebooks/html-processing/README.md b/examples/notebooks/html-processing/README.md index 238d84dd0..3711f51e6 100644 --- a/examples/notebooks/html-processing/README.md +++ b/examples/notebooks/html-processing/README.md @@ -50,6 +50,23 @@ We will save the extracted text (markdown) into a vector database (Milvus) ## Step-5: Query documents +### 5.1 - Setup `.env` file with API Token + +For this step, we will be using Replicate API service. We need a Replicate API token for this step. + +Follow these steps: + +- Get a **free** account at [replicate](https://replicate.com/home) +- Use this [invite](https://replicate.com/invites/a8717bfe-2f3d-4a52-88ed-1356231cdf03) to add some credit 💰 to your Replicate account! +- Create an API token on Replicate dashboard + +Once you have an API token, add it to the project like this: + +- Copy the file `env.sample.txt` into `.env` (note the dot in the beginning of the filename) +- Add your token to `REPLICATE_API_TOKEN` in the .env file. + +### 5.2 - Query + Query documents using LLM [4_query.ipynb](4_query.ipynb) \ No newline at end of file From 9ce07059f99ef9b517064c46b87b065e9bd52d4d Mon Sep 17 00:00:00 2001 From: Sujee Maniyam Date: Wed, 27 Nov 2024 11:53:01 -0800 Subject: [PATCH 08/31] HTML RAG demo working - code cleanup - updated documentation Signed-off-by: Sujee Maniyam --- .../html-processing/1_crawl_site.ipynb | 14 - .../2_extract_text_from_html.ipynb | 10 +- .../html-processing/3_save_to_vector_db.ipynb | 524 +----------------- examples/notebooks/html-processing/README.md | 37 +- .../notebooks/html-processing/my_config.py | 2 +- 5 files changed, 52 insertions(+), 535 deletions(-) diff --git a/examples/notebooks/html-processing/1_crawl_site.ipynb b/examples/notebooks/html-processing/1_crawl_site.ipynb index f60824eff..8d8d6d298 100644 --- a/examples/notebooks/html-processing/1_crawl_site.ipynb +++ b/examples/notebooks/html-processing/1_crawl_site.ipynb @@ -59,8 +59,6 @@ "shutil.rmtree(MY_CONFIG.INPUT_DIR, ignore_errors=True)\n", "shutil.os.makedirs(MY_CONFIG.INPUT_DIR, exist_ok=True)\n", "\n", - "\n", - "\n", "print (\"✅ Cleared download directory\")" ] }, @@ -112,18 +110,6 @@ "execution_count": 4, "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Exception in callback BaseAsyncIOLoop._handle_events(8, 1)\n", - "handle: \n", - "Traceback (most recent call last):\n", - " File \"/home/sujee/apps/anaconda3/envs/dpk-html-processing-2-022dev3-py312/lib/python3.12/asyncio/events.py\", line 88, in _run\n", - " self._context.run(self._callback, *self._args)\n", - "RuntimeError: cannot enter context: <_contextvars.Context object at 0x70577a041080> is already entered\n" - ] - }, { "name": "stdout", "output_type": "stream", diff --git a/examples/notebooks/html-processing/2_extract_text_from_html.ipynb b/examples/notebooks/html-processing/2_extract_text_from_html.ipynb index 6dea17a74..c5bd69012 100644 --- a/examples/notebooks/html-processing/2_extract_text_from_html.ipynb +++ b/examples/notebooks/html-processing/2_extract_text_from_html.ipynb @@ -20,7 +20,7 @@ "\n", "We will process data that is downloaded using [1_crawl_site.ipynb](1_crawl_site.ipynb).\n", "\n", - "We have a couple of crawled HTML files in `input*` directory. " + "We have a couple of crawled HTML files in `input` directory. " ] }, { @@ -418,12 +418,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Step-5: Save the markdown in a dir" + "## Step-5: Save the markdown" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -441,10 +441,6 @@ " html_file = row['document']\n", " base_name = os.path.splitext(os.path.basename(html_file))[0]\n", " md_output_file = os.path.join(MY_CONFIG.OUTPUT_DIR_MARKDOWN, base_name + '.md')\n", - " # print(f\"html_file': {html_file}\")\n", - " # print(f\"base_name': {base_name}\")\n", - " # print(f\"md_output_file': {md_output_file}\")\n", - " # print()\n", " \n", " with open(md_output_file, 'w') as md_output_file_handle:\n", " md_output_file_handle.write (row['contents'])\n", diff --git a/examples/notebooks/html-processing/3_save_to_vector_db.ipynb b/examples/notebooks/html-processing/3_save_to_vector_db.ipynb index 9729c1956..608ffa2d8 100644 --- a/examples/notebooks/html-processing/3_save_to_vector_db.ipynb +++ b/examples/notebooks/html-processing/3_save_to_vector_db.ipynb @@ -7,6 +7,13 @@ "# Save Markdown text into Vector DB" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step-1: Config" + ] + }, { "cell_type": "code", "execution_count": 1, @@ -16,6 +23,13 @@ "from my_config import MY_CONFIG" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step-2: Read Markdown" + ] + }, { "cell_type": "code", "execution_count": 2, @@ -26,7 +40,6 @@ "import glob\n", "\n", "pattern = os.path.join(MY_CONFIG.OUTPUT_DIR_MARKDOWN, '*.md')\n", - "\n", "md_file_count = len(glob.glob(pattern, recursive=True)) " ] }, @@ -61,501 +74,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "Doc ID: c444d3d4-f3a9-4e86-9cbe-1f86ea82f226\n", - "Text: !\n", - "---\n", - "Doc ID: 072ecafe-306c-4138-96a2-ef1eb56a7d01\n", - "Text: Open Source AI Demo Night The AI Alliance, in collaboration\n", - "with Cerebral Valley and Ollama, hosted Open Source AI Demo Night in\n", - "San Francisco, bringing together more than 200+ developers and\n", - "innovators to showcase and celebrate the latest advances in open-\n", - "source AI.\n", - "---\n", - "Doc ID: 9d05c74f-113f-4632-a0d6-e62616b56a77\n", - "Text: AI Alliance members plan to start or enhance projects that meet\n", - "the following objectives: Deploy benchmarks, tools, and other\n", - "resources that enable the responsible development and use of AI\n", - "systems at a global scale, including the creation of a catalog of\n", - "vetted safety, security, and trust tools. Support the advocacy and\n", - "enablement of these to...\n", - "---\n", - "Doc ID: 9fc93e5e-c6c0-4cc8-b57e-b5b687b5cef9\n", - "Text: Meet the AI Alliance The AI Alliance is a collaborative network\n", - "of companies, startups, universities, research institutions,\n", - "government organizations, and non-profit foundations that are working\n", - "at the forefront of AI technology, applications, and governance. We\n", - "unite top developers, scientists, academic institutions, companies,\n", - "and policymake...\n", - "---\n", - "Doc ID: cb0afc89-b0ac-4ed0-a2ad-c01cbc788fa1\n", - "Text: Become a collaborator Want to get involved? Apply to join one\n", - "of our working groups or major initiatives. Our team will review your\n", - "request and get back to you as soon as possible (within 30 days or\n", - "fewer). Want to get involved? Apply to join one of our working groups\n", - "or major initiatives. Our team will review your request and get back\n", - "to yo...\n", - "---\n", - "Doc ID: b7b521fe-661b-47ef-b0dc-ee13064b636f\n", - "Text: **AI Alliance Expands with Seven New Members from India**\n", - "---\n", - "Doc ID: 0bf8007c-bc73-48cf-8406-8a63acb57789\n", - "Text: Strengthening the Country's Role in Shaping the Global AI\n", - "Landscape through Open, Safe, and Responsible AI Practices **INDIA,\n", - "Bengaluru, September 17, 2024** –The AI Alliance, a global community\n", - "dedicated to advancing safe and responsible AI through open\n", - "innovation, has expanded its membership to include seven leading\n", - "Indian organizations: AI4B...\n", - "---\n", - "Doc ID: d4975e3d-8f06-4b4b-a608-4487af9b9359\n", - "Text: AI Alliance Launches as an International Community of Leading\n", - "Technology Developers, Researchers, and Adopters Collaborating\n", - "Together to Advance Open, Safe, Responsible AI !abstract gradient AI\n", - "advancements are leading to new opportunities that can improve how we\n", - "work, live, learn and interact with one another. Open and transparent\n", - "innovation ...\n", - "---\n", - "Doc ID: b5768ab8-090c-43e3-8ab5-6b07f86967ca\n", - "Text: Who We Are To ensure open innovation in AI benefits everyone\n", - "and that it is built responsibly, the AI Alliance consists of a broad\n", - "range of organizations that are working across aspects of AI\n", - "education, research, development and deployment, and governance. We\n", - "are: - The creators of the tooling driving AI benchmarking, trust and\n", - "validation met...\n", - "---\n", - "Doc ID: bce66f25-7717-4fe8-b075-b994c9924374\n", - "Text: How the AI Alliance Will Work The AI Alliance will begin its\n", - "work with the formation of member-driven working groups across all\n", - "major topical areas listed above. The Alliance will also establish a\n", - "governing board and technical oversight committee dedicated to\n", - "advancing the above project areas, as well as establishing overall\n", - "project standards a...\n", - "---\n", - "Doc ID: 361dff47-895b-46ac-8844-e34e6e2cdfa4\n", - "Text: On August 8th, The AI Alliance, in collaboration with Cerebral\n", - "Valley and Ollama, hosted Open Source AI Demo Night in San Francisco,\n", - "bringing together more than 200+ developers and innovators to showcase\n", - "and celebrate the latest advances in open-source AI. There were 7 demo\n", - "teams and a panel discussion on why open technologies and communities\n", - "ar...\n", - "---\n", - "Doc ID: 33f0d501-5004-4587-8299-947879f19ed1\n", - "Text: Join the Community Join leading AI innovators, researchers,\n", - "developers, engineers, and early enterprise adopters in our goal to\n", - "advance open, safe, and responsible AI. Follow the links on this page\n", - "to subscribe to our community newsletter, contribute to a working\n", - "group, join The Alliance, follow us on social media, or attend one of\n", - "our upcoming...\n", - "---\n", - "Doc ID: 65d9d165-0267-432e-a367-9f807379da2e\n", - "Text: Subscribe to our community newsletter Get updates about\n", - "everything that’s happening in the community and be the first to know\n", - "about new projects, new resources, upcoming events, opportunities to\n", - "collaborate and contribute, etc. To subscribe, please fill out the\n", - "newsletter sign-up form in the footer below.\n", - "---\n", - "Doc ID: 96c15f94-7ce6-41f7-a421-5f8d5fa1143c\n", - "Text: Contribute code We strive to provide a clear and simple path\n", - "for anyone to join our community and contribute to AI Alliance\n", - "projects, whether they are part of an existing member organization, or\n", - "not. Learn more...\n", - "---\n", - "Doc ID: 59003bc2-9599-42ab-81c1-28e8159df43c\n", - "Text: Become a collaborator in one of our working groups The\n", - "heartbeat of the AI Alliance is our working groups and major\n", - "initiatives! Since launch, they have rolled out core projects and\n", - "affiliated projects – with more to come: - Skills, Education, and\n", - "Exploratory Research WG - AI Trust and Safety WG - Application and\n", - "Tools WG - Diverse Hardware En...\n", - "---\n", - "Doc ID: 9c998bb3-0511-407e-a483-21cc3b7ad9a3\n", - "Text: Join The Alliance If your organization would like to join the\n", - "AI Alliance as an official member, please use the contact form, and\n", - "someone on our membership team will get back to you. Thanks!\n", - "---\n", - "Doc ID: c2a891e8-cc9a-4a5a-9227-43ca39203084\n", - "Text: Attend an AI Alliance event - Oct 21–24: OpenModelCamp at\n", - "TechXChange Community Dayin Las Vegas, NV - Oct 26: TechEquity AI\n", - "Summitin Mountain View, CA - Oct 30: AI for Drug Discovery Open\n", - "Innovation Forumin Boston, MA - Nov 6–8: PyData NYCin New York, NY -\n", - "Nov 7: Data, Cloud and AI in Singapore Meetup: “Leading AI:\n", - "Collaboration Meets Ethical I...\n", - "---\n", - "Doc ID: e111dae0-7bc4-4965-9f67-c271b1d953c9\n", - "Text: Follow us on social media - Linkedin:\n", - "https://linkedin.com/company/the-aialliance/ - X/Twitter:\n", - "https://x.com/thealliance_ai - Instagram:\n", - "https://www.instagram.com/aialliance/ - Threads:\n", - "https://www.threads.net/@aialliance - YouTube:\n", - "https://www.youtube.com/@thealliance_ai\n", - "---\n", - "Doc ID: d6f2d050-d746-40b5-aebe-4ae2623764b2\n", - "Text: Contact us We’re here to help and answer any questions you may\n", - "have regarding the AI Alliance. We look forward to hearing from you!\n", - "Please ensure to include enough detail so we can route your inquiry to\n", - "the right team member. Thanks! We’re here to help and answer any\n", - "questions you may have regarding the AI Alliance. We look forward to\n", - "hearing...\n", - "---\n", - "Doc ID: a63b9dab-9b6a-4e85-aedb-a163eeddf3a9\n", - "Text: Join us in building the open future of AI We strive to provide\n", - "a clear and simple path for anyone to join our community and\n", - "contribute to AI Alliance projects, whether they are part of an\n", - "existing member organization, or not.\n", - "---\n", - "Doc ID: 6972c48f-b8f4-4b62-b063-bdeb24106511\n", - "Text: We are excited to welcome contributions to AI Alliance Core and\n", - "Affiliated projects. **Core Projects** Core projects address\n", - "substantial cross-community challenges and are an opportunity for\n", - "individual contributors and members to collaborate, build, and make an\n", - "impact on the future of AI. Core Projects are managed directly by the\n", - "AI Alliance a...\n", - "---\n", - "Doc ID: 89b59840-4a4a-4d2e-b17f-f1defcdf81e0\n", - "Text: Making a contribution The AI Alliance utilizes the Linux\n", - "Foundation’s Developer Certificate of Origin (DCO) on a per-commit\n", - "basis to enable contributions to Core Projects. The DCO does not\n", - "require the committer to give away ownership of the contribution -\n", - "your IP remains yours and others' IP remains theirs. **Core Project\n", - "Licenses** AI Allian...\n", - "---\n", - "Doc ID: 848f3a37-9c57-49f0-ae98-9c5d28f497f0\n", - "Text: Responding to the U.S. NTIA request for comment on Dual Use\n", - "Foundation Artificial Intelligence Models with Widely Available Model\n", - "Weights Project !abstract gradient The AI Alliance recently drafted\n", - "a response to the NTIA’s request for comment on “Dual Use Foundation\n", - "Artificial Intelligence Models with Widely Available Model Weights.”\n", - "The requ...\n", - "---\n", - "Doc ID: 80dd54c2-36b5-43ec-b606-02ae6e7634c2\n", - "Text: The AI Alliance, a community of technology creators, developers,\n", - "and adopters collaborating to advance safe, responsible AI rooted in\n", - "open innovation, respectfully opposes SB 1047, the proposed *Safe and\n", - "Secure Innovation for Frontier Artificial Intelligence Models Act*. It\n", - "would slow innovation, thwart advancements in safety and security, and\n", - "u...\n", - "---\n", - "Doc ID: a68d64ee-1a21-45ef-a467-5560e20b54a2\n", - "Text: Context Model evaluation, or ‘evals’, in the generative AI era\n", - "is simultaneously one of the most important areas of investment but\n", - "also one of the highest areas of entropy. Our goals for the Trusted\n", - "Evals working group are to: for community efforts around trust and\n", - "safety – including, but not limited to, work happening globally in\n", - "various lan...\n", - "---\n", - "Doc ID: 26733281-e36f-4d28-b23e-75b8735184aa\n", - "Text: Request for proposal We are seeking new perspectives in the AI\n", - "evaluation domain. To foster further innovation in this area, we are\n", - "pleased to invite the community to participate in the AI Alliance’s\n", - "Trusted Evals working group by submitting a response to our Request\n", - "for Proposal to be included in the working group’s efforts to:\n", - "**Raise awaren...\n", - "---\n", - "Doc ID: 39765acd-c476-413d-adf6-8b4ff81c6044\n", - "Text: Areas of interest - Cybersecurity threats - Sensitive data\n", - "detection including areas such as toxic content (e.g. hate speech),\n", - "personally identifiable information (PII), bias, etc. - Model\n", - "performance including helpfulness, quality, alignment, robustness,\n", - "etc., (as opposed to operational concerns like throughput, latency,\n", - "scalability, etc.) - K...\n", - "---\n", - "Doc ID: e60b0a5d-54f0-42f4-9e57-36e909a87447\n", - "Text: Requirements Proposals should include a link to a summary of\n", - "your project, in English, explaining the area of focus, dataset\n", - "description and any relevant prior work. This includes: - Name of\n", - "project and abstract - Core team member bios & affiliations - Link to\n", - "website (if applicable) - Link to GitHub repository - License - Links\n", - "to whitepapers...\n", - "---\n", - "Doc ID: 4504925f-505d-4669-8fff-4716501b64ee\n", - "Text: !\n", - "---\n", - "Doc ID: b6327a28-d0d2-4e6d-af26-dff93944eeec\n", - "Text: Statement from AI Alliance Regarding Commerce's Final Report on\n", - "Open-Weights Models Commerce Department has taken a crucial step\n", - "towards ensuring that the U.S. remains at the forefront of AI\n", - "development by concluding that \"the government should not restrict the\n", - "wide availability of model weights for dual-use foundation models.\"\n", - "---\n", - "Doc ID: 9b6971ae-ba9b-4cce-92c4-1b7b17bc461f\n", - "Text: !abstract gradient\n", - "---\n", - "Doc ID: 5a1f19ef-a866-4b0b-b334-7bf2609bd361\n", - "Text: Advancing Domain-Specific Q&A: The AI Alliance's Guide to Best\n", - "Practices The AI Alliance application and tools working group has\n", - "conducted a comprehensive study on best practices for advancing\n", - "domain-specific Q&A using retrieval-augmented generation (RAG)\n", - "techniques. The findings of this research, provide insights and\n", - "recommendations for maximi...\n", - "---\n", - "Doc ID: 182c4a8e-6d2b-4ddd-a7c8-e952d8bfd340\n", - "Text: Open Foundation Models and Datasets\n", - "---\n", - "Doc ID: 1b25bcf0-5c97-4802-a7c0-51b7ed77e259\n", - "Text: Enabling an ecosystem of open foundation models, including those\n", - "with multilingual and multi-modal capabilities, and open datasets. We\n", - "are responsibly enhancing the ecosystem of open foundation models and\n", - "datasets. We are embracing multilingual and multimodal models, as well\n", - "as science models tackling broad societal issues like climate change\n", - "a...\n", - "---\n", - "Doc ID: 1dca16b0-68b4-43b1-bb32-40037254a37b\n", - "Text: Working groups\n", - "---\n", - "Doc ID: f2993603-7b32-476c-89e2-088707af5ecc\n", - "Text: AI for Drug Discovery This working group aims to create a\n", - "world-class research community that harnesses the potential of AI\n", - "foundation models, transforms the field of drug discovery, and\n", - "accelerates scientific progress by driving interdisciplinary\n", - "collaboration on AI-powered drug discovery projects in the open.\n", - "!abstract gradient\n", - "---\n", - "Doc ID: cf9ee4db-3c01-4a9f-ae00-03a1d902221e\n", - "Text: Foundation Models !abstract gradient\n", - "---\n", - "Doc ID: 39126747-9240-48c2-bd8e-b4a0eb5b7952\n", - "Text: Materials and Chemistry We aim to curate datasets, tasks and\n", - "benchmarks for materials science, build out foundation models in\n", - "chemistry for prediction of properties, experimental outcomes or\n", - "generation of new candidates and create a framework to foster\n", - "collaboration between human experts and AI agents that will ultimately\n", - "help solve global urge...\n", - "---\n", - "Doc ID: e51a162b-9e3c-46e1-b9aa-89f65b8a2436\n", - "Text: Current or recent projects !Home page for the Open Trusted Data\n", - "Initiative !Home page for the Open Trusted Data Initiative\n", - "---\n", - "Doc ID: 86ba2b6c-7631-4f0d-bfa3-6eac283d95c1\n", - "Text: Open Trusted Data Initiative Foundation Models and Datasets\n", - "Cataloging and managing trustworthy datasets. !abstract gradient\n", - "!abstract gradient\n", - "---\n", - "Doc ID: ae8c8fe3-187e-44ed-9303-42d819e672b6\n", - "Text: Time Series Data and Model Initiative Foundation Models and\n", - "Datasets Time-series applications are an important target for AI. In\n", - "addition to gathering high-quality and fully-governed time series\n", - "datasets as part of the Open Trusted Data Initiative, Alliance members\n", - "are collaborating on new and improved time series models (as part of\n", - "the Indust...\n", - "---\n", - "Doc ID: 87ef3822-1d4f-44bf-9f28-e69fa4aa627f\n", - "Text: Industry Open FMs Initiative Foundation Models and Datasets We\n", - "have seen rapid progress in building and releasing highly-capable and\n", - "open foundation models for general language, coding, scientific\n", - "discovery, and multi-modal scenarios. A key development in model\n", - "strategies is a focus on building smaller, more specialized models.\n", - "More details ...\n", - "---\n", - "Doc ID: 529128a6-d3c4-4999-96b9-d66ac957bc38\n", - "Text: Hardware Enablement\n", - "---\n", - "Doc ID: d796f7f0-7207-4c04-bca5-75420977a45d\n", - "Text: Fostering a vibrant AI hardware accelerator ecosystem through\n", - "enabling software technology. We collaborate on the benchmarking,\n", - "optimization, and adaptation of AI workloads to accelerate innovation\n", - "in a diverse set of hardware. Our work focuses on scalability,\n", - "platform adaptation, and advanced power, energy, and carbon modeling.\n", - "Benchmarks and ...\n", - "---\n", - "Doc ID: 1094484d-f0e6-4d23-8dd2-37410e432d54\n", - "Text: Our projects Projects we are currently working on:\n", - "---\n", - "Doc ID: 4131f4b7-8c0d-4173-87fa-56a25e597eda\n", - "Text: Skills & Education\n", - "---\n", - "Doc ID: a6d026ae-524f-4704-9428-c1e657eb771b\n", - "Text: Supporting global AI skill-building, education, and exploratory\n", - "research. Our members partner with academics, industry leaders,\n", - "researchers, and students, empowering them to develop and leverage\n", - "resources to build an AI-ready workforce and engage with vital\n", - "research in AI models, algorithms, and platforms.\n", - "---\n", - "Doc ID: 55b60157-348d-4246-a8b3-214d918d48a6\n", - "Text: Trust and Safety\n", - "---\n", - "Doc ID: 4e0fed30-fc07-4271-97fc-acfb40ebbc3f\n", - "Text: Ensuring safe and trusted generative AI with benchmarks, tools,\n", - "and methodologies Our members work together to understand the\n", - "landscape of AI trust and safety risks, as well as other uses of AI\n", - "system evaluation. We identify use cases specific to various important\n", - "domains (finance, healthcare, education, and others). We work together\n", - "and with e...\n", - "---\n", - "Doc ID: 21047c6c-cf5b-4f99-950e-036d26f70f32\n", - "Text: Current or recent projects ! !\n", - "---\n", - "Doc ID: 00a45255-126b-4905-ab65-85eafed747e1\n", - "Text: Understanding AI Trust and Safety: A Living Guide A major\n", - "challenge for the successful use of AI is the importance of\n", - "understanding potential trust and safety issues, along with their\n", - "mitigation strategies. Failure to consider these issues could impact\n", - "an organization's operations and the experience of its users. Concerns\n", - "about safety are also ...\n", - "---\n", - "Doc ID: a41c8f30-470c-4598-9bc5-fa59cc27ba69\n", - "Text: Trust and Safety Evaluations Trust & Safety Much like other\n", - "software, generative AI (“GenAI”) *models* and the *AI systems* that\n", - "use them need to be trusted and useful to their users. The *Trust and\n", - "Safety Evaluations* project fills gaps in the current landscape of the\n", - "taxonomy of different kinds of evaluation, the tools for creating and\n", - "runni...\n", - "---\n", - "Doc ID: 0f230e93-6cfa-4b43-aca0-ca5edb8614da\n", - "Text: Ranking AI Safety Priorities by Domain Trust & Safety A\n", - "challenge for software development teams adopting generative AI is\n", - "making sense of the safety issues that their applications must\n", - "address. The AI safety ecosystem is broad and growing quickly, making\n", - "it difficult for these development teams to know where they should\n", - "focus their efforts. W...\n", - "---\n", - "Doc ID: 31076aa2-fa81-4ac2-b734-a6fb27cd28f4\n", - "Text: Trusted Evals request for proposals Trust & Safety The AI\n", - "Alliance Trusted Evals request for proposals is aimed at seeking new\n", - "perspectives in the AI evaluation domain. We are excited to work with\n", - "those in academia, industry, startups and anyone excited to\n", - "collaborate in the open and build an ecosystem around their work.\n", - "---\n", - "Doc ID: 97f0de37-3ab7-45d6-803f-0f298e9c56ec\n", - "Text: Accelerating responsible innovation in AI Open and transparent\n", - "innovation is essential for equipping AI researchers, developers, and\n", - "users with the knowledge and tools to leverage AI advancements safely\n", - "and inclusively, prioritizing diversity and economic opportunity for\n", - "all. Through member-driven working groups, we bring together builders\n", - "and...\n", - "---\n", - "Doc ID: 7c507e93-5f49-4086-b3d9-db761b2de087\n", - "Text: Core projects View all Core projects address substantial\n", - "cross-community challenges and are an opportunity for individual\n", - "contributors and members to collaborate, build, and make an impact on\n", - "the future of AI. Core Projects are managed directly by the AI\n", - "Alliance and governed as described below. !abstract gradient\n", - "!abstract gradient\n", - "---\n", - "Doc ID: cab794c4-78b3-438a-861c-f61a8d41ff4d\n", - "Text: A statement in opposition to California SB 1047 Advocacy Our\n", - "perspectives and recommendations in opposition to California SB 1047,\n", - "the proposed *Safe and Secure Innovation for Frontier Artificial\n", - "Intelligence Models Act*. !Trusted Evals RFP !Trusted Evals RFP\n", - "---\n", - "Doc ID: 72214679-919b-4c96-b21d-08881ef4082b\n", - "Text: Trusted Evals request for proposals Trust & Safety The AI\n", - "Alliance Trusted Evals request for proposals is aimed at seeking new\n", - "perspectives in the AI evaluation domain. We are excited to work with\n", - "those in academia, industry, startups and anyone excited to\n", - "collaborate in the open and build an ecosystem around their work.\n", - "!abstract gradient ...\n", - "---\n", - "Doc ID: c12a6b5c-2961-416d-82c6-eee525b36f59\n", - "Text: Responding to the U.S. NTIA request for comment on Dual Use\n", - "Foundation Artificial Intelligence Models with Widely Available Model\n", - "Weights Advocacy The request seeks public input on the potential\n", - "risks, benefits, and policy approaches for AI foundation models whose\n", - "weights are broadly accessible. ! !\n", - "---\n", - "Doc ID: 16c0c4ac-40b2-4be8-a6e8-eb1105274f5d\n", - "Text: Ranking AI Safety Priorities by Domain Trust & Safety A\n", - "challenge for software development teams adopting generative AI is\n", - "making sense of the safety issues that their applications must\n", - "address. The AI safety ecosystem is broad and growing quickly, making\n", - "it difficult for these development teams to know where they should\n", - "focus their efforts. W...\n", - "---\n", - "Doc ID: b6987811-2807-4e95-b4a7-387af6c37a59\n", - "Text: Open Trusted Data Initiative Foundation Models and Datasets\n", - "Cataloging and managing trustworthy datasets. !abstract gradient\n", - "!abstract gradient\n", - "---\n", - "Doc ID: 31123c95-6abe-4bd9-9e7c-b9cc5b707185\n", - "Text: Time Series Data and Model Initiative Foundation Models and\n", - "Datasets Time-series applications are an important target for AI. In\n", - "addition to gathering high-quality and fully-governed time series\n", - "datasets as part of the Open Trusted Data Initiative, Alliance members\n", - "are collaborating on new and improved time series models (as part of\n", - "the Indust...\n", - "---\n", - "Doc ID: 44b6430b-8dd4-4ecb-87f2-eef9c4dbf215\n", - "Text: AI Accelerator Software Ecosystem Guide Hardware Enablement\n", - "The **AI Accelerator Software Ecosystem Guide** provides an\n", - "introduction to the broad topic of software tools that support\n", - "seamless execution of AI workloads on different hardware accelerators.\n", - "Industry experts provide guidance on the state of the art and how to\n", - "ensure success. !abst...\n", - "---\n", - "Doc ID: c349331b-abdb-4782-b126-9fffdb3e5b81\n", - "Text: Industry Open FMs Initiative Foundation Models and Datasets We\n", - "have seen rapid progress in building and releasing highly-capable and\n", - "open foundation models for general language, coding, scientific\n", - "discovery, and multi-modal scenarios. A key development in model\n", - "strategies is a focus on building smaller, more specialized models.\n", - "More details ...\n", - "---\n", - "Doc ID: 13267e45-c786-4806-8c86-9f4f40a53fef\n", - "Text: Understanding AI Trust and Safety: A Living Guide A major\n", - "challenge for the successful use of AI is the importance of\n", - "understanding potential trust and safety issues, along with their\n", - "mitigation strategies. Failure to consider these issues could impact\n", - "an organization's operations and the experience of its users. Concerns\n", - "about safety are also ...\n", - "---\n", - "Doc ID: 83d2727e-cbe1-4ae1-96e1-62dac2d53f31\n", - "Text: The Living Guide to Applying AI Applications and Tools A\n", - "\"living guide\" for building AI-enabled applications, this guide\n", - "provides an introduction to several established design patterns for\n", - "building AI systems and products, with contributions from different\n", - "experts in the AI Alliance. Several common patterns, like RAG\n", - "(retrieval-augmented gener...\n", - "---\n", - "Doc ID: b3123acf-4a56-42de-ac6c-5b22b160c68e\n", - "Text: Trust and Safety Evaluations Trust & Safety Much like other\n", - "software, generative AI (“GenAI”) *models* and the *AI systems* that\n", - "use them need to be trusted and useful to their users. The *Trust and\n", - "Safety Evaluations* project fills gaps in the current landscape of the\n", - "taxonomy of different kinds of evaluation, the tools for creating and\n", - "runni...\n", - "---\n", - "Doc ID: 8c96d27f-1ef3-4063-835b-248fab3d48af\n", - "Text: Affiliated projects View all Affiliated projects are led and\n", - "managed by members and identified as being aligned to the AI Alliance\n", - "mission. Although Affiliated Projects are not managed by the AI\n", - "Alliance, we highlight them as contributing to open, safe, and\n", - "responsible AI. !IBM logo\n", - "---\n", - "Doc ID: fa277a88-e744-4a85-a6df-e68b1d2ddb3c\n", - "Text: IBM Granite Code Models A series of code models trained by IBM\n", - "licensed under Apache 2.0 license. Includes base pretrained and\n", - "instruct models. !\n", - "---\n", - "Doc ID: e924df60-a844-4a0e-859e-9a96a84f82a7\n", - "Text: AIConfig & Editor ! AIConfig is a config-based framework for\n", - "building generative AI applications. !\n", - "---\n", - "Doc ID: a2c51552-6e48-4614-ac74-5930ccfe49c4\n", - "Text: GenAI in Education: Usage Guidance ! A report evaluating the\n", - "feasibility, benefits, and limitations of using generative AI\n", - "technologies in an educational setting and its impact on learning\n", - "outcomes. !\n", - "---\n", - "Doc ID: ebaf6236-f3c9-454c-8807-9dc3264ad4a6\n", - "Text: LLM 360: Amber ! The Amber project includes a 7B English\n", - "language model with the LLaMA architecture, an instruction-following\n", - "model finetuned from LLM360-Amber, and a safety-finetuned instruction\n", - "model using LLM360-AmberChat as the base. !Red Hat logo\n", - "---\n", - "Doc ID: d1b13520-3056-46aa-a22a-56f2cacdddd8\n", - "Text: TrustyAI !Red Hat logo TrustyAI is an open-source toolkit\n", - "designed for responsible AI. !\n", - "---\n", - "Doc ID: f5f283f0-c274-4641-90a8-7485b91427ef\n", - "Text: Ragna ! Ragna is an open-source RAG-based (Retrieval-Augmented\n", - "Generation) AI orchestration framework designed to bridge the gap\n", - "between AI research and production deployment.\n", - "---\n" + "Doc ID: a773fc8a-fa26-4e65-a276-1578cb1f85f9\n", + "Text: !\n" ] } ], "source": [ - "for doc in documents:\n", - " print (doc)\n", - " print ('---')" + "## Inspect a sample doc\n", + "print (documents[0])" ] }, { @@ -641,6 +167,8 @@ } ], "source": [ + "## Clear up any old data\n", + "\n", "from pymilvus import MilvusClient\n", "\n", "milvus_client = MilvusClient(MY_CONFIG.DB_URI)\n", @@ -686,7 +214,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Step-5: Create Index and Save to DB" + "## Step-5: Save to DB" ] }, { @@ -698,10 +226,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "✅ Created index: \n", - "✅ Saved index to db ./rag_html.db\n", - "CPU times: user 532 ms, sys: 129 ms, total: 661 ms\n", - "Wall time: 657 ms\n" + "✅ Saved documents to db ./rag_html.db\n", + "CPU times: user 502 ms, sys: 132 ms, total: 634 ms\n", + "Wall time: 605 ms\n" ] } ], @@ -715,8 +242,7 @@ "index = VectorStoreIndex.from_documents(\n", " documents, storage_context=storage_context\n", ")\n", - "print (\"✅ Created index:\", index )\n", - "print (\"✅ Saved index to db \", MY_CONFIG.DB_URI )" + "print (\"✅ Saved documents to db \", MY_CONFIG.DB_URI )" ] } ], diff --git a/examples/notebooks/html-processing/README.md b/examples/notebooks/html-processing/README.md index 3711f51e6..bc05beaea 100644 --- a/examples/notebooks/html-processing/README.md +++ b/examples/notebooks/html-processing/README.md @@ -1,10 +1,12 @@ # Using Data Prep Kit to Process HTML files -## Step-1: Setup Python Env +This example will show how to crawl a website, process HTML files and query them using RAG. + +Here is the process: -See [project README](../../../README.md#create-a-virtual-environment). +`website ---(crawler) --> HTML files --- (html2pq)--> markdown content ---(llama-index)--> save to vector db (Milvus) ---(query)---> LLM` -Here is a quick setup guide +## Step-1: Setup Python Env ```bash conda create -n dpk-html-processing-py311 python=3.11 @@ -25,32 +27,39 @@ Install modules pip install -r requirements.txt ``` +## Step-2: Configuration -## Step-2: Crawl a website +Inspect configuration here: [my_config.py](my_config.py) -This step will crawl a site and download HTML files in `output/1-html2parquet` directory +Here you can set: -[1_crawl_site.ipynb](1_crawl_site.ipynb) +- site to crawl +- how many files to download and crawl depth +- embedding model +- LLM to use -or use `wget` +## Step-3: Crawl a website + +This step will crawl a site and download HTML files in `input` directory + +[1_crawl_site.ipynb](1_crawl_site.ipynb) -`wget --recursive --level=1 --adjust-extension https://thealliance.ai/` -## Step-3: Process HTML files +## Step-4: Process HTML files -We will process downloaded HTML files and extract the text as markdown. The output will be saved in `output/2-markdown` directory +We will process downloaded HTML files and extract the text as markdown. The output will be saved in the`output/2-markdown` directory in markdown format [2_extract_text_from_html.ipynb](2_extract_text_from_html.ipynb) -## Step-4: Save data into DB +## Step-5: Save data into DB We will save the extracted text (markdown) into a vector database (Milvus) [3_save_to_vector_db.ipynb](3_save_to_vector_db.ipynb) -## Step-5: Query documents +## Step-6: Query documents -### 5.1 - Setup `.env` file with API Token +### 6.1 - Setup `.env` file with API Token For this step, we will be using Replicate API service. We need a Replicate API token for this step. @@ -65,7 +74,7 @@ Once you have an API token, add it to the project like this: - Copy the file `env.sample.txt` into `.env` (note the dot in the beginning of the filename) - Add your token to `REPLICATE_API_TOKEN` in the .env file. -### 5.2 - Query +### 6.2 - Query Query documents using LLM diff --git a/examples/notebooks/html-processing/my_config.py b/examples/notebooks/html-processing/my_config.py index 949c8ade3..7848370d1 100644 --- a/examples/notebooks/html-processing/my_config.py +++ b/examples/notebooks/html-processing/my_config.py @@ -13,7 +13,7 @@ class MyConfig: MY_CONFIG.CRAWL_MAX_DEPTH = 2 MY_CONFIG.CRAWL_MIME_TYPE = 'text/html' -## Input Data - configure this to the folder we want to process +## Directories MY_CONFIG.INPUT_DIR = "input" # MY_CONFIG.INPUT_DIR = "input2/thealliance.ai/" MY_CONFIG.OUTPUT_DIR = "output" From 601087881730b327db4203e815efbc726976ad9a Mon Sep 17 00:00:00 2001 From: Constantin M Adam Date: Thu, 28 Nov 2024 01:05:39 -0500 Subject: [PATCH 09/31] Simplified fix for issue 803 Signed-off-by: Constantin M Adam --- .../src/data_processing_ray/runtime/ray/ray_utils.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/data-processing-lib/ray/src/data_processing_ray/runtime/ray/ray_utils.py b/data-processing-lib/ray/src/data_processing_ray/runtime/ray/ray_utils.py index 5225508fb..d5e6754ac 100644 --- a/data-processing-lib/ray/src/data_processing_ray/runtime/ray/ray_utils.py +++ b/data-processing-lib/ray/src/data_processing_ray/runtime/ray/ray_utils.py @@ -15,13 +15,16 @@ from typing import Any import ray -from ray.experimental.state.api import list_actors from data_processing.utils import GB, UnrecoverableException from ray.actor import ActorHandle from ray.exceptions import RayError +from ray.experimental.state.api import list_actors from ray.util.actor_pool import ActorPool +RAY_MAX_ACTOR_LIMIT = 10000 + + class RayUtils: """ Class implementing support methods for Ray execution @@ -109,11 +112,13 @@ def operator() -> ActorHandle: time.sleep(creation_delay) return clazz.options(**actor_options).remote(params) - cls_name = clazz.__class__.__name__.replace('ActorClass(', '').replace(')','') + cls_name = clazz.__class__.__name__.replace("ActorClass(", "").replace(")", "") actors = [operator() for _ in range(n_actors)] for i in range(120): time.sleep(1) - alive = list_actors(filters=[("class_name", "=", cls_name), ("state", "=", "ALIVE")]) + alive = list_actors( + filters=[("class_name", "=", cls_name), ("state", "=", "ALIVE")], limit=RAY_MAX_ACTOR_LIMIT + ) if len(actors) == len(alive): return actors # failed - raise an exception From 8cb170d6cb1a90e0f3fd00420d6ffa2a82b28352 Mon Sep 17 00:00:00 2001 From: Aisha Mohammed Farooq Darga Date: Fri, 29 Nov 2024 14:16:23 +0530 Subject: [PATCH 10/31] Hap Example Notebook Signed-off-by: Aisha Mohammed Farooq Darga --- .../hap/generate_hap_score_csv.ipynb | 490 ++++++++++++++++++ .../hap/input/customer_feedback_file.csv | 66 +++ examples/notebooks/hap/input/data.parquet | Bin 0 -> 47309 bytes examples/notebooks/hap/output/data.parquet | Bin 0 -> 40282 bytes .../hap/output/hap_complete_output.csv | 66 +++ .../hap/output/hap_filtered_output.csv | 66 +++ examples/notebooks/hap/output/metadata.json | 56 ++ .../universal/hap/python/src/hap_config.py | 23 + 8 files changed, 767 insertions(+) create mode 100644 examples/notebooks/hap/generate_hap_score_csv.ipynb create mode 100644 examples/notebooks/hap/input/customer_feedback_file.csv create mode 100644 examples/notebooks/hap/input/data.parquet create mode 100644 examples/notebooks/hap/output/data.parquet create mode 100644 examples/notebooks/hap/output/hap_complete_output.csv create mode 100644 examples/notebooks/hap/output/hap_filtered_output.csv create mode 100644 examples/notebooks/hap/output/metadata.json create mode 100644 transforms/universal/hap/python/src/hap_config.py diff --git a/examples/notebooks/hap/generate_hap_score_csv.ipynb b/examples/notebooks/hap/generate_hap_score_csv.ipynb new file mode 100644 index 000000000..497218c2e --- /dev/null +++ b/examples/notebooks/hap/generate_hap_score_csv.ipynb @@ -0,0 +1,490 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "HAP Transform Example Notebook\n", + "=====================================\n", + "This notebook picks a CSV file from the `input` folder, converts it to Parquet format,\n", + "runs the `hap_local_python.py` transform, and displays the results." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Overview\n", + "This notebook demonstrates the use of the HAP transformation to annotate documents with a `hap_score`, \n", + "indicating the likelihood of Hate, Abuse, or Profanity in the text.\n", + "\n", + "### Workflow\n", + "The HAP process consists of:\n", + "1. **Sentence Splitting**: Documents are split into sentences using NLTK.\n", + "2. **HAP Annotation**: Each sentence is scored between 0 and 1 (1 = high HAP, 0 = no HAP).\n", + "3. **Aggregation**: The document's final HAP score is the maximum score among all sentences.\n", + "\n", + "\n", + "### Configuration\n", + "- **Model Name**: IBM Granite Guardian (`ibm-granite/granite-guardian-hap-38m` by default).\n", + "- **Document Text Column** (`--doc_text_column`): Specify the input column containing document text to generate the hap_score against. Defaults to `contents`.\n", + "- **Annotation Column** (`--annotation_column`): Specify the output column for HAP scores. Defaults to `hap_score`.\n", + "\n", + "\n", + "### Steps in This Notebook\n", + "1. Define paths and import libraries.\n", + "2. Convert CSV input to Parquet.\n", + "3. Run the HAP transformation script.\n", + "4. View and analyze the results.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 491, + "metadata": {}, + "outputs": [], + "source": [ + "# Import necessary libraries\n", + "import os\n", + "import pandas as pd\n", + "import subprocess\n", + "import sys" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 1: Define Paths\n", + "---------------------\n", + "Define the paths for the script, input folder, and output folder." + ] + }, + { + "cell_type": "code", + "execution_count": 492, + "metadata": {}, + "outputs": [], + "source": [ + "hap_script_path = \"./transforms/universal/hap/python/src/hap_local_python.py\"\n", + "input_folder = \"./input\"\n", + "output_folder = \"./output\"" + ] + }, + { + "cell_type": "code", + "execution_count": 493, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Script Path: ./transforms/universal/hap/python/src/hap_local_python.py\n", + "Input Folder: ./input\n", + "Output Folder: ./output\n" + ] + } + ], + "source": [ + "# Ensure the necessary folders exist.\n", + "os.makedirs(input_folder, exist_ok=True)\n", + "os.makedirs(output_folder, exist_ok=True)\n", + "\n", + "print(f\"Script Path: {hap_script_path}\")\n", + "print(f\"Input Folder: {input_folder}\")\n", + "print(f\"Output Folder: {output_folder}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 2: Check for CSV Files in Input Folder\n", + "\n", + "- Place your CSV file in the `input_folder`.\n", + "- Ensure the column containing the text matches the `doc_text_column` parameter.\n", + "- If your text column has a different name, update the `doc_text_column` parameter in later cells.\n", + "- This cell sets up the file paths for the input file.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 494, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found CSV file(s): ['customer_feedback_file.csv']\n", + "Using CSV file: ./input/customer_feedback_file.csv\n" + ] + } + ], + "source": [ + "csv_files = [f for f in os.listdir(input_folder) if f.endswith(\".csv\")]\n", + "\n", + "if not csv_files:\n", + " print(f\"No CSV files found in the input folder: {input_folder}\")\n", + " print(\"Please place a CSV file in the input folder and rerun this notebook.\")\n", + "else:\n", + " print(f\"Found CSV file(s): {csv_files}\")\n", + "\n", + "# Pick the first CSV file in the folder\n", + "csv_file_path = os.path.join(input_folder, csv_files[0])\n", + "print(f\"Using CSV file: {csv_file_path}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 3: Convert CSV to Parquet\n", + "Convert the selected CSV file to Parquet format." + ] + }, + { + "cell_type": "code", + "execution_count": 495, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CSV file converted to Parquet format at: ./input/data.parquet\n" + ] + } + ], + "source": [ + "parquet_file_path = os.path.join(input_folder, \"data.parquet\")\n", + "df = pd.read_csv(csv_file_path)\n", + "df.to_parquet(parquet_file_path, index=False)\n", + "print(f\"CSV file converted to Parquet format at: {parquet_file_path}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 4: Simulate Command-Line Arguments for HAP Parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 496, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Command-line arguments: ['hap_local_python.py', '--input_folder', './input', '--output_folder', './output', '--model_name_or_path', 'ibm-granite/granite-guardian-hap-38m', '--annotation_column', 'hap_score', '--doc_text_column', 'Customer Feedback', '--inference_engine', 'CPU', '--max_length', '512', '--batch_size', '128']\n" + ] + } + ], + "source": [ + "\n", + "# Clear sys.argv first to avoid conflicts (reset arguments list)\n", + "sys.argv = [\n", + " \"hap_local_python.py\", # Script name\n", + " \"--input_folder\", input_folder, # Correct input folder\n", + " \"--output_folder\", output_folder, # Correct output folder\n", + " \"--model_name_or_path\", hap_params[\"model_name_or_path\"],\n", + " \"--annotation_column\", hap_params[\"annotation_column\"],\n", + " \"--doc_text_column\", hap_params[\"doc_text_column\"],\n", + " \"--inference_engine\", hap_params[\"inference_engine\"],\n", + " \"--max_length\", str(hap_params[\"max_length\"]),\n", + " \"--batch_size\", str(hap_params[\"batch_size\"]),\n", + "]\n", + "\n", + "print(f\"Command-line arguments: {sys.argv}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 5: Run the Transform with Simulated Arguments\n", + "\n", + "This cell executes the HAP transformation script:\n", + "- `--input_file`: Path to your input CSV/Parquet file.\n", + "- `--output_file`: Path where the output file with HAP scores will be saved.\n", + "- `--doc_text_column`: The column containing the text for analysis (default: `Customer Feedback`).\n", + "- `--annotation_column`: The column where HAP scores will be saved (default: `hap_score`).\n", + "\n", + "**Customization**: \n", + "- If your text column has a different name, update the value of `--doc_text_column` accordingly.\n", + "- You can adjust other parameters like `--batch_size` and `--max_length` if needed." + ] + }, + { + "cell_type": "code", + "execution_count": 497, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error occurred during transform execution.\n", + "python: can't open file '/Users/aisha/Documents/GitHub/Personal/DPK/examples/notebooks/hap/./transforms/universal/hap/python/src/hap_local_python.py': [Errno 2] No such file or directory\n", + "\n" + ] + } + ], + "source": [ + "# Copy the current environment variables\n", + "env = os.environ.copy()\n", + "\n", + "# Set Environment Variables for HAP Parameters\n", + "os.environ[\"MODEL_NAME_OR_PATH\"] = \"ibm-granite/granite-guardian-hap-38m\"\n", + "os.environ[\"ANNOTATION_COLUMN\"] = \"hap_score\"\n", + "os.environ[\"DOC_TEXT_COLUMN\"] = \"Customer Feedback\"\n", + "os.environ[\"INFERENCE_ENGINE\"] = \"CPU\"\n", + "os.environ[\"MAX_LENGTH\"] = \"512\"\n", + "os.environ[\"BATCH_SIZE\"] = \"128\"\n", + "os.environ[\"INPUT_FOLDER\"] = input_folder\n", + "os.environ[\"OUTPUT_FOLDER\"] = output_folder\n", + "try:\n", + " result = subprocess.run(\n", + " [\"python\", hap_script_path],\n", + " check=True,\n", + " text=True,\n", + " capture_output=True\n", + " )\n", + "\n", + " # If successful, print the result of the transform\n", + " print(\"Transform completed successfully.\")\n", + " print(result.stdout)\n", + "\n", + "except subprocess.CalledProcessError as e:\n", + " # If there was an error, print the error message\n", + " print(\"Error occurred during transform execution.\")\n", + " print(e.stderr)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Step 6: Generate the Output CSV\n", + "\n", + "This step checks for any existing CSV files in the output folder and removes them before generating new ones. The following actions are performed:\n", + "\n", + "1. **Listing Output Files**: The script lists all files in the output folder.\n", + "2. **Check for Parquet Files**: It identifies `.parquet` files in the output folder.\n", + "3. **Remove Old CSV Files**: If any previous output files (`hap_complete_output.csv` or `hap_filtered_output.csv`) exist, they are deleted.\n", + "4. **Read Parquet File**: The Parquet file is read into a DataFrame.\n", + "5. **Filter Data**: The relevant columns, `doc_text_column` (from the environment variable) and `hap_score_column`, are selected from the DataFrame.\n", + "6. **Save New CSV Files**: The filtered data is saved into two new CSV files:\n", + " - `hap_complete_output.csv` (containing the full output)\n", + " - `hap_filtered_output.csv` (containing only the filtered relevant columns).\n", + "\n", + "This ensures that only the latest output is retained, and no old files remain in the output folder." + ] + }, + { + "cell_type": "code", + "execution_count": 498, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Complete Output Parquet File Path: ./output/data.parquet\n", + "Old complete CSV file removed: ./output/hap_complete_output.csv\n", + "Old filtered CSV file removed: ./output/hap_filtered_output.csv\n", + "Filtered Output (only HAP score and document text):\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Customer Feedbackhap_score
0Rating: 4 Comments: \"Service was prompt, but ...0.000195
1Rating: 5 Comments: \"Great help from Peter! H...0.000153
2Rating: 3 Comments: \"The service was quick, b...0.000169
3Rating: 5 Comments: \"Excellent service and ad...0.000158
4Rating: 2 Comments: \"I’m really frustrated. T...0.000875
.........
60Rating: 3 Comments: \"This is not what I expec...0.000150
61Rating: 1 Comments: \"This is insane. I’ve onl...0.579251
62Rating: 4 Comments: \"I need this fixed. I can’...0.000384
63Rating: 2 Comments: \"I’m so done with this ma...0.000285
64Rating: 3 Comments: \"I’ve leveled the machine...0.052517
\n", + "

65 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " Customer Feedback hap_score\n", + "0 Rating: 4 Comments: \"Service was prompt, but ... 0.000195\n", + "1 Rating: 5 Comments: \"Great help from Peter! H... 0.000153\n", + "2 Rating: 3 Comments: \"The service was quick, b... 0.000169\n", + "3 Rating: 5 Comments: \"Excellent service and ad... 0.000158\n", + "4 Rating: 2 Comments: \"I’m really frustrated. T... 0.000875\n", + ".. ... ...\n", + "60 Rating: 3 Comments: \"This is not what I expec... 0.000150\n", + "61 Rating: 1 Comments: \"This is insane. I’ve onl... 0.579251\n", + "62 Rating: 4 Comments: \"I need this fixed. I can’... 0.000384\n", + "63 Rating: 2 Comments: \"I’m so done with this ma... 0.000285\n", + "64 Rating: 3 Comments: \"I’ve leveled the machine... 0.052517\n", + "\n", + "[65 rows x 2 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Complete output saved to: ./output/hap_complete_output.csv\n", + "Filtered output saved to: ./output/hap_filtered_output.csv\n" + ] + } + ], + "source": [ + "import os\n", + "import pandas as pd\n", + "\n", + "# List all files in the output folder\n", + "output_files = os.listdir(output_folder)\n", + "\n", + "if output_files:\n", + " for file in output_files:\n", + " if file.endswith(\".parquet\"): # Check for Parquet output files\n", + " output_file_path = os.path.join(output_folder, file)\n", + " output_df = pd.read_parquet(output_file_path) # Read the Parquet file\n", + " print(f\"Complete Output Parquet File Path: {output_file_path}\")\n", + "\n", + " # Define the output CSV file paths\n", + " complete_output_csv = os.path.join(output_folder, \"hap_complete_output.csv\")\n", + " filtered_output_csv = os.path.join(output_folder, \"hap_filtered_output.csv\")\n", + "\n", + " # Remove old CSV files if they exist\n", + " if os.path.exists(complete_output_csv):\n", + " os.remove(complete_output_csv)\n", + " print(f\"Old complete CSV file removed: {complete_output_csv}\")\n", + "\n", + " if os.path.exists(filtered_output_csv):\n", + " os.remove(filtered_output_csv)\n", + " print(f\"Old filtered CSV file removed: {filtered_output_csv}\")\n", + "\n", + " # Filter the output DataFrame to only include the relevant columns\n", + " hap_score_column = hap_params[\"annotation_column\"]\n", + " doc_text_column = os.getenv('DOC_TEXT_COLUMN') # Read from environment variable\n", + " filtered_df = output_df[[doc_text_column, hap_score_column]]\n", + "\n", + " # Print the filtered DataFrame (only showing the HAP score and document text)\n", + " print(f\"Filtered Output (only HAP score and document text):\")\n", + " display(filtered_df)\n", + "\n", + " # Save the complete output as a CSV file\n", + " output_df.to_csv(complete_output_csv, index=False) # Convert the Parquet to CSV\n", + " print(f\"Complete output saved to: {complete_output_csv}\")\n", + "\n", + " # Save the filtered output as a CSV file\n", + " filtered_df.to_csv(filtered_output_csv, index=False)\n", + " print(f\"Filtered output saved to: {filtered_output_csv}\")\n", + "\n", + "else:\n", + " print(\"No output files found. Please check the script or configuration.\")\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dataprepkit", + "language": "python", + "name": "data-prep-kit" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/notebooks/hap/input/customer_feedback_file.csv b/examples/notebooks/hap/input/customer_feedback_file.csv new file mode 100644 index 000000000..996effc27 --- /dev/null +++ b/examples/notebooks/hap/input/customer_feedback_file.csv @@ -0,0 +1,66 @@ +Customer Name,Date,Customer ID,Email Address,Phone Number,Shipping Address,Order Number,Issue Reported,Service Representative,Response Time,Resolution,Follow-up Needed,Customer Feedback, +Ethan Johnson,11/7/24,782011,ethan.johnson@email.com,-4109,"134 Oak Street, Portland, OR 97204",4578342890," Ethan contacted customer service to report a recurring problem with his washing machine, Model EcoWash 300. He mentioned that during the spin cycle, the machine produces a loud, irregular banging noise. Ethan initially thought the noise might be related to something inside the drum, so he checked the drum for foreign objects like coins or small clothing items. He confirmed that no objects were present. However, the noise continues to persist even with minimal loads. He is concerned that the issue could be related to the suspension system or drum assembly, potentially causing damage to the internal components if not addressed soon. Ethan requested a technician visit to prevent further damage and to get a professional diagnosis. ",Jane Cooper,15 minutes," Jane took the time to walk Ethan through a detailed troubleshooting process, including verifying the load size and checking the balance of the washing machine. She explained that unbalanced loads can sometimes cause unusual noise during the spin cycle. However, since the issue persisted even with small loads, she advised him to check the drum suspension system and ensure that it was properly secured. Jane further explained that if the drum or suspension system had become misaligned or the shock absorbers were worn out, this could cause the banging noise. Realizing the issue might require more complex intervention, Jane scheduled a technician visit to perform a more thorough inspection of the internal components, including the suspension springs and drum motor. ","Yes, technician visit scheduled for drum suspension and motor"," Rating: 4 Comments: ""Service was prompt, but I'm worried it might be a bigger issue with the drum. Hope it gets fixed soon."" Source System: CRM --- ### Customer Service Log 2 Customer Information", +Sophia Wilson,11/8/24,785673,sophia.wilson@email.com,-5331,"1025 Pine Avenue, San Francisco, CA 94107",4678452100," Sophia reported an issue with her SmartWash 500, which has been malfunctioning during its washing cycle. Specifically, the machine displays the error code ""E4"" after completing the cycle, which indicates a drainage problem. Despite attempting to troubleshoot on her own, including checking the drain filter and ensuring the drain hose was clear of obstructions, the issue persists. Sophia mentioned that there is also water left in the drum after the cycle finishes. She is concerned that it might be an issue with the pump motor or a deeper internal blockage. ",Peter Foster,30 minutes," Peter explained that the E4 error code typically indicates a drainage issue, often due to a clogged or blocked drain hose or filter. After walking Sophia through the steps to clean the filter and ensure no external blockages were present, he confirmed that the issue was still unresolved. He hypothesized that the issue could be related to the pump motor, which may not be functioning properly to expel the water. Peter emphasized that if the motor had failed, the water would not be drained ebiciently, leading to standing water in the drum. He arranged for a technician to inspect the internal pump motor and check for any blockages or malfunctioning parts that might be abecting the drainage. Peter assured Sophia that the technician would be able to further diagnose the problem and ober a solution. ","Yes, technician visit scheduled to inspect the pump motor and"," Rating: 5 Comments: ""Great help from Peter! Hope the technician resolves the issue."" Source System: CRM --- ### Customer Service Log 3 Customer Information", +Liam Smith,11/6/24,781924,liam.smith@email.com,-6553,"432 Birch Road, Los Angeles, CA 90001",4589231843," Liam’s QuickWash 700 washing machine has been experiencing an issue where it intermittently stops mid-cycle and does not complete the wash. He reported that the cycle halts after about 15 minutes, and he is forced to restart the machine manually, only for it to happen again. Liam checked the load balance and even tried running the machine with smaller loads, but the problem persists. He suspects that the issue might be related to the control board or an internal sensor malfunction that is causing the cycle to terminate prematurely. ",Emily Davis,20 minutes," Emily took the time to explain to Liam that one possible cause for a washing machine to stop mid-cycle is a faulty control board or a malfunctioning sensor that fails to communicate the proper cycle status. She walked Liam through a series of checks to verify whether the machine was properly loaded or if any specific wash settings might be triggering the premature shutdown. After ruling out the simpler causes, Emily acknowledged that the problem might lie within the machine’s electronic control system. She advised Liam to keep track of the error codes, if any, that appeared on the display. Emily escalated the issue for a more in-depth inspection of the control board and its connections, as well as a sensor diagnostic. She arranged for a technician to visit and diagnose the problem further. ","Yes, technician visit scheduled to inspect the control board and"," Rating: 3 Comments: ""The service was quick, but I’m still unsure if the issue will be fixed. It keeps happening."" Source System: CRM --- ### Customer Service Log 4 Customer Information", +Isabella Martinez,11/9/24,783210,isabella.martinez@email.com,-7775,"785 Oak Drive, Chicago, IL 60601",4599872301," Isabella reached out regarding her SmartWash 200 washing machine, which is failing to spin correctly. After completing the washing cycle, her clothes are still wet and heavy. Isabella mentioned that the drum doesn’t seem to be spinning at all, even when she tries to set the machine to a spin cycle manually. She noted that the drainage seems fine, and there are no unusual noises during the cycle, which led her to believe the problem could be with the spin motor or drive belt. ",Peter Foster,40 minutes," Peter first confirmed that Isabella had checked for any obvious external blockages in the drain and the drum, and all appeared fine. He then walked her through testing the spin cycle to see if the machine would respond when set to high spin speed. After determining that the drum was indeed not turning, Peter suspected that the issue might lie with either the spin motor or the drive belt. He explained that if the motor was malfunctioning, it would prevent the drum from spinning at all, even though the rest of the machine seemed to be operating normally. Given the nature of the issue, Peter scheduled a technician visit to further investigate the motor and drive belt system. ","Yes, technician visit scheduled to inspect the spin motor and drive"," Rating: 5 Comments: ""Excellent service and advice. Hopefully the technician can fix it soon."" Source System: CRM --- ### Customer Service Log 5 Customer Information", +Noah White,11/10/24,780512,noah.white@email.com,-10219,"912 Willow Lane, Miami, FL 33101",4602741920," Noah’s SuperClean 800 washing machine has been experiencing intermittent water leakage around the door seal during the wash cycle. He observed that after each wash, water begins to pool around the base of the washing machine. Despite checking the door seal for any visible cracks or damage, he has not been able to identify the source of the leak. He also confirmed that the water inlet valve is functioning correctly. Noah expressed concern that the issue might be due to a defective seal or an issue with the drum assembly causing the leakage . ",Laura Simmons,10 minutes," Laura advised Noah to carefully inspect the door seal and ensure that no detergent or fabric softener had built up in the rubber, as that could cause poor sealing and leakage. After confirming that the seal appeared clean and in good condition, Laura suspected that the issue could be related to improper drum alignment or even a small crack in the seal that was dibicult to detect. She scheduled a technician visit to replace the seal and inspect the drum alignment to prevent future leaks. ","Yes, technician visit scheduled for door seal replacement and drum"," Rating: 2 Comments: ""I’m really frustrated. This issue has been ongoing for weeks and still isn’t resolved."" Source System: CRM --- These logs now include more detailed issues and resolutions, which are useful for analysis and decision-making. Sure! Here are 5 more customer service logs with detailed issues and resolutions, including a variety of feedback: --- ### Customer Service Log 6 Customer Information", +Mia Green,11/10/24,787654,mia.green@email.com,-2664,"212 Cedar Way, Austin, TX 78701",4698765432," Mia reported that her EcoWash 1000 washing machine has started to vibrate excessively during the spin cycle. She mentioned that it was noticeable even with smaller loads of laundry, and the vibration causes the machine to move slightly across the floor. Mia checked that the machine was level, and the floor seemed solid, so she suspected that the issue could be related to the shock absorbers or suspension system, which might not be absorbing the vibrations properly. ",Kevin Mitchell,25 minutes," Kevin walked Mia through some troubleshooting steps to check if the machine was perfectly level and if the laundry was balanced. After confirming that the machine was indeed level and the load was balanced, Kevin suspected that the issue could be with the shock absorbers or the suspension springs, which could have worn out over time and were no longer absorbing the vibrations. He scheduled a technician visit to inspect and replace the faulty components. Kevin also recommended that Mia avoid overloading the machine until the issue is resolved, as that could further stress the system. ","Yes, technician visit scheduled to inspect and replace shock"," Rating: 4 Comments: ""The service was helpful, but I’m hoping the issue will be fully resolved with the new parts. It’s been a bit of a nuisance."" Source System: CRM --- ### Customer Service Log 7 Customer Information", +Daniel Thomas,11/11/24,788901,daniel.thomas@email.com,-9219,"520 Maple Road, Denver, CO 80205",4692345678," Daniel's QuickWash 800 washing machine has started to stop halfway through its wash cycle. He noted that it happens frequently, regardless of the type of load or wash cycle selected. The washing machine will start the cycle normally, but then it freezes and the timer stops progressing. Daniel confirmed that the power supply is stable and the machine is correctly plugged in. He suspected that the control board or timer could be malfunctioning, causing the machine to halt unexpectedly. ",Sarah Lewis,15 minutes," Sarah explained that the issue could indeed be related to a malfunctioning control board or timer, as these are the primary components that regulate the wash cycle. Sarah walked Daniel through a few basic steps, such as turning the machine ob for a few minutes and resetting it, but the problem persisted. She recommended that he check for any error codes or flashing lights on the display that might provide more insight into the issue. Sarah escalated the issue for further inspection and arranged for a technician to diagnose the problem with the control board and timer, as they would need to be thoroughly tested and potentially replaced. ","Yes, technician visit scheduled to inspect the control board and"," Rating: 3 Comments: ""The support was helpful, but I’m frustrated the machine keeps stopping. I’ve been waiting for a technician for a while now."" Source System: CRM --- ### Customer Service Log 8 Customer Information", +Charlotte Davis,11/12/24,786502,charlotte.davis@email.com,-11330,"1500 Elm Street, New York, NY 10001",4701293847," Charlotte reported that her SmartWash 200 washing machine is experiencing issues with water leakage during the rinse cycle. She noticed water was dripping from the bottom of the door seal while the cycle was running. Charlotte has inspected the door seal and found no visible tears or cracks, but still, the machine continues to leak water. She is concerned that the issue may lie in the water inlet valve or a possible internal gasket problem. ",James Walker,20 minutes," James acknowledged that water leakage during the rinse cycle is often due to a broken door seal or an issue with the water inlet valve. He walked Charlotte through checking for any detergent residue or buildup that might cause improper sealing around the door, but Charlotte confirmed that the seal appeared clean. James explained that sometimes the gasket or internal seals can wear out or become misaligned, causing water to leak out. He recommended a technician visit to inspect both the water inlet valve and the internal seals for potential issues. James also advised Charlotte to check that the door was properly closed before starting the cycle to avoid further leaks. ","Yes, technician visit scheduled to inspect the water inlet valve and"," Rating: 5 Comments: ""The service was very quick and the representative explained everything clearly. I hope the technician can fix the leak soon."" Source System: CRM --- ### Customer Service Log 9 Customer Information", +Benjamin Harris,11/10/24,780893,benjamin.harris@email.com,-5220,"934 Maple Ave, Miami, FL 33132",4691273856," Benjamin’s EcoWash 500 washing machine has been displaying an ""E3"" error code, indicating a sensor issue. The error typically appears mid-cycle, and the washing machine stops working. Benjamin attempted a reset of the machine, but the issue continued. He mentioned that the machine would run the cycle for a while, but then it would halt and display the E3 error code without completing the wash. Benjamin believes the problem may be related to a faulty water level sensor or another internal sensor. ",Olivia Taylor,10 minutes," Olivia explained that the E3 error code often indicates a sensor malfunction, especially related to the water level sensor or the pressure switch, which measures the water levels in the drum during a cycle. Olivia walked Benjamin through a series of troubleshooting steps, such as checking the water supply and ensuring the water pressure was subicient. However, since the error persisted, Olivia arranged for a technician visit to thoroughly inspect and test the sensors and wiring to identify the cause of the issue. She also noted that if a faulty sensor was found, it would need to be replaced to restore proper function. ","Yes, technician visit scheduled to inspect and replace faulty"," Rating: 5 Comments: ""The support was excellent. I’m confident the technician will sort out the issue once and for all."" Source System: CRM --- ### Customer Service Log 10 Customer Information", +Grace Clark,11/11/24,789234,grace.clark@email.com,-3998,"600 Oakwood Blvd, San Diego, CA 92101",4697682012," Grace’s QuickWash 900 washing machine has been giving her problems with uneven washes. She reported that no matter what cycle she uses, the clothes come out either soaking wet or still have detergent residue on them. Grace tried adjusting the load size and used the proper detergent, but the problem persisted. She suspects that the issue might be related to an imbalance in the drum or a problem with the dispensing system that isn’t releasing the detergent properly. ",Emma Harris,30 minutes," Emma explained that an uneven wash result could be caused by several factors, including a misaligned drum, water supply issues, or even a clogged detergent dispenser. Emma walked Grace through checking the detergent compartment to make sure it was not clogged or blocked by detergent buildup. She also suggested that Grace check if the load was unbalanced, as this can sometimes cause uneven washing results. Since the issue was more complex, Emma scheduled a technician visit to inspect the drum alignment and the detergent dispensing system for blockages or malfunctions. ","Yes, technician visit scheduled to inspect drum alignment and"," Rating: 2 Comments: ""I’m frustrated with this issue. I’ve tried everything, and the machine still doesn’t wash properly. I hope the technician can figure it out."" Source System: CRM --- ### Customer Service Log 11 Customer Information", +Aiden Walker,11/13/24,790321,aiden.walker@email.com,-7664,"2324 Ashwood Drive, Dallas, TX 75201",4703409821," Aiden contacted customer service about his UltraSpin 1200 washing machine, which is not heating the water properly during the wash cycle. He noticed that even after setting the machine to a hot wash, the clothes were still cold and soapy when the cycle finished. Aiden confirmed that the water inlet valve was functioning and that the temperature setting on the machine was correct. He suspects that the issue might be with the heating element or a faulty temperature sensor. ",Michelle Adams,18 minutes," Michelle explained that the issue could be caused by a broken heating element or a malfunctioning temperature sensor, both of which would prevent the water from reaching the correct temperature. She walked Aiden through verifying the settings and ensuring that the water supply was properly connected. Since the issue persisted, Michelle recommended that the heating element be inspected, as it might have failed. She scheduled a technician visit to test the element and the temperature sensor and replace any faulty parts. ","Yes, technician visit scheduled to inspect and replace the heating"," Rating: 4 Comments: ""Good support, but I'm hoping the technician can get the heating issue fixed quickly. The laundry’s been piling up!"" Source System: CRM --- ### Customer Service Log 12 Customer Information", +Olivia Thompson,11/14/24,791014,olivia.thompson@email.com,-10219,"1030 Riverbend Street, Orlando, FL 32801",4704891342," Olivia reported that her WashPro 700 washing machine is exhibiting strange odors after completing wash cycles. She mentioned that the odor is sour and unpleasant, even though she has regularly cleaned the drum and used the proper detergent. Olivia is worried that the odor might be caused by mold or mildew buildup, but she hasn’t found any visible signs of it. She is concerned that the smell is coming from the drain pump or an issue with the drum gasket, as these are areas that could trap water and promote mold growth. ",Jacob Carter,25 minutes," Jacob explained that mold and mildew odors are often caused by water remaining stagnant in hidden areas, such as the drain pump, drain hose, or drum gasket, which can trap moisture and detergent residue. He recommended that Olivia run a maintenance wash with hot water and vinegar to help eliminate any buildup inside the drum. Jacob also advised checking the drain filter for any debris that might be obstructing proper drainage. Since the issue persisted, Jacob scheduled a technician visit to inspect the drain pump, gasket, and drain hose for any blockages or signs of mildew buildup. ","Yes, technician visit scheduled to inspect the drain pump, gasket,"," Rating: 3 Comments: ""The advice was helpful, but the odor is still there. Hoping the technician can sort this out quickly."" Source System: CRM --- ### Customer Service Log 13 Customer Information", +Henry Lewis,11/15/24,792785,henry.lewis@email.com,-7775,"875 Pine Street, Philadelphia, PA 19101",4706765432," Henry’s QuickWash 1000 washing machine has started to not respond to any button presses on the control panel. He reported that the machine powers on normally, but when he tries to start a wash cycle, none of the buttons work. The display screen also appears frozen, showing a blank screen after the initial power-up. Henry suspects that the control board or touchpad might be malfunctioning, and that this is preventing the machine from operating properly. ",Emily Scott,35 minutes," Emily explained that when a washing machine’s control panel or touchpad becomes unresponsive, it’s often a sign of a control board issue or a problem with the wiring connections. Emily recommended that Henry unplug the machine for a few minutes to reset the system, which might help if the issue was due to a temporary electronic glitch. Since this didn’t resolve the problem, Emily scheduled a technician visit to inspect the control board, touchpad, and the wiring connections for any issues. She noted that if the control board was faulty, it would need to be replaced. ","Yes, technician visit scheduled to inspect the control board and"," Rating: 4 Comments: ""Service was ebicient, but I really need this issue fixed soon as the washing machine is practically unusable right now."" Source System: CRM --- ### Customer Service Log 14 Customer Information", +Ava Robinson,11/16/24,793214,ava.robinson@email.com,-5331,"520 Pinehurst Road, San Jose, CA 95128",4710923845," Ava reported that her SuperClean 800 washing machine is having trouble with the spin cycle, as it is not reaching full speed. Despite running a heavy-duty cycle, the clothes remain wet after the spin cycle finishes. Ava suspects that the issue could be with the drive motor or the spin belt, which she believes may be slipping or failing to engage properly during the spin cycle. She confirmed that the drainage is working fine and that no blockages are present. ",Sarah Morgan,30 minutes," Sarah explained that a washing machine that doesn’t reach full spin speed is often due to a malfunctioning motor or a broken spin belt, both of which can prevent the drum from reaching the correct RPM. Sarah advised Ava to check the spin cycle settings and make sure the machine was not overloaded, as that could also cause the spin cycle to run inebiciently. Since this didn’t resolve the issue, Sarah scheduled a technician visit to inspect the motor, spin belt, and drum assembly. She also advised Ava to avoid using large, heavy loads until the problem was fixed to avoid damaging the system further. ","Yes, technician visit scheduled to inspect the drive motor and spin"," Rating: 5 Comments: ""Great support. I’m confident the technician will figure out the issue and fix it."" Source System: CRM --- ### Customer Service Log 15 Customer Information", +Lucas Wright,11/17/24,790539,lucas.wright@email.com,-8997,"874 Birch Street, Miami, FL 33139",4709281365," Lucas reported that his WashPro 900 washing machine is consistently overfilling with water during the wash cycle, even when he sets it to a low water level. He mentioned that the machine will fill up beyond the normal water line and cause water to overflow from the door. Lucas suspects that the issue might be due to a faulty water inlet valve or pressure switch that is not regulating the water flow properly. ",Rachel Bennett,28 minutes," Rachel explained that overfilling during the wash cycle is typically caused by a malfunctioning pressure switch or water inlet valve, both of which control the amount of water entering the drum. She recommended that Lucas check for any blockages in the water inlet valve and ensure that the hose was not kinked or obstructed. Since the issue persisted, Rachel arranged for a technician visit to inspect the pressure switch, water inlet valve, and the drainage system to identify the root cause of the overfilling issue. ","Yes, technician visit scheduled to inspect the water inlet valve and"," Rating: 3 Comments: ""Support was helpful, but I’ve been dealing with this overfilling issue for weeks now. I hope the technician can fix it soon."" Source System: CRM --- ### Customer Service Log 16 Customer Information", +Lily Adams,11/18/24,795023,lily.adams@email.com,-11108,"1070 Spruce Lane, Houston, TX 77001",4713485932," Lily’s SmartWash 900 washing machine is displaying a ""water flow error"" during the initial filling cycle. She reported that the water does not fill up to the appropriate level, and the machine stops after a few minutes with the error code displayed. Lily confirmed that the water supply is working in her home, and no issues with the pipes were detected. She suspects a malfunction with the water inlet valve or flow sensor. ",Ethan Clarke,15 minutes," Ethan explained that the water flow error is typically caused by issues with the water inlet valve or a faulty flow sensor, both of which control the water supply to the drum. He walked Lily through checking the inlet hose for kinks or obstructions, which could restrict water flow. Since the issue persisted, Ethan escalated the case to the Field Service Management team for an on-site technician visit. The technician would inspect the water inlet valve and flow sensor to identify any malfunction. ","Yes, technician visit scheduled to inspect and replace water inlet"," Rating: 4 Comments: ""The troubleshooting steps were useful, but I hope the technician can fix the issue soon. I need the machine working properly."" Source System: Field Service Management --- ### Customer Service Log 17 Customer Information", +George Marshall,11/19/24,796521,george.marshall@email.com,-6331,"453 Birchwood Ave, Chicago, IL 60602",4712457890," George reported that his WashMax 3000 washing machine has been making a loud, grinding noise during the spin cycle. The noise has become progressively louder, and George is concerned that it could be a serious mechanical issue. He suspects the noise may be coming from the motor, drum bearings, or the drive belt. He noted that the machine still spins, but the noise is very disruptive. ",Natalie Rogers,25 minutes," Natalie explained that a loud grinding noise during the spin cycle could be caused by worn drum bearings, a loose drive belt, or a faulty motor coupling. She walked George through checking the drum alignment and ensuring that the machine was not overloaded, which could exacerbate the noise. Since the issue was mechanical and likely internal, Natalie scheduled a technician visit through SAP to inspect the motor, drum bearings, and drive belt for any signs of wear or damage. ","Yes, technician visit scheduled to inspect motor, drum bearings,"," Rating: 3 Comments: ""The support was helpful, but I’m concerned about the cost of fixing the motor or bearings. It’s been quite a frustrating experience."" Source System: SAP --- ### Customer Service Log 18 Customer Information", +Zoe Carter,11/20/24,797631,zoe.carter@email.com,-5331,"901 Oak Ridge Blvd, Los Angeles, CA 90001",4714579023," Zoe’s QuickSpin 1000 washing machine has an issue with excessive vibrations during the spin cycle, even with small loads. She mentioned that the machine moves across the floor during the spin, and she has already checked the leveling of the unit, which seems fine. Zoe suspects that the problem could be with the shock absorbers or suspension springs. ",Michael Harris,30 minutes," Michael explained that excessive vibrations during the spin cycle are often due to issues with the shock absorbers or suspension springs, which are responsible for stabilizing the drum during high-speed spinning. He walked Zoe through verifying that the machine was level and balanced, which she confirmed. Michael recommended that Zoe avoid overloading the machine and advised that a technician inspect the shock absorbers and suspension springs for wear or damage. He scheduled the technician visit through Field Service Management to assess and replace the faulty parts. ","Yes, technician visit scheduled to inspect and replace shock"," Rating: 4 Comments: ""The support was helpful, but I hope the technician can resolve the vibration issue soon. It's been annoying trying to balance the machine."" Source System: Field Service Management --- ### Customer Service Log 19 Customer Information", +Jack Davis,11/21/24,798234,jack.davis@email.com,-9219,"3500 Maplewood Lane, San Francisco, CA 94105",4720293847," Jack reported that his EcoSpin 600 washing machine is not draining water properly at the end of the wash cycle. The machine completes the cycle, but there is still standing water in the drum. Jack has checked the drain filter and confirmed that it’s clean, but the issue persists. He suspects the drain pump or drain hose may be clogged. ",Rebecca Foster,22 minutes," Rebecca explained that improper drainage is often caused by issues with the drain pump or a clogged drain hose, which could be obstructing the flow of water. She guided Jack through inspecting the drain hose for any visible blockages. Since the hose was clear, Rebecca suggested that the drain pump might have a fault. She escalated the issue through SAP for a technician visit to inspect and possibly replace the drain pump and verify the entire drainage system. ","Yes, technician visit scheduled to inspect and replace drain pump."," Rating: 5 Comments: ""The troubleshooting steps were helpful, and the technician visit is scheduled quickly. I appreciate the fast response."" Source System: SAP --- ### Customer Service Log 20 Customer Information", +Sarah Mitchell,11/22/24,799012,sarah.mitchell@email.com,-4331,"2180 Oakwood Ave, Phoenix, AZ 85001",4723128475," Sarah reported that her TurboWash 5000 washing machine is experiencing intermittent power issues. The machine powers ob unexpectedly mid-cycle and sometimes won’t turn on again without multiple attempts. Sarah noted that the power cord and outlet are fine, and there are no visible signs of a power surge. She suspects a problem with the power supply board or internal wiring. ",Thomas Hill,40 minutes," Thomas explained that intermittent power issues are typically caused by a faulty power supply board or wiring issues that cause the machine to lose connection. He advised Sarah to check if there were any error codes displayed when the power failure occurred, but Sarah confirmed there were none. Since the issue seemed related to the internal components, Thomas arranged a technician visit through Field Service Management to inspect the power supply board and internal wiring for faults and perform any necessary replacements. ","Yes, technician visit scheduled to inspect and replace the power"," Rating: 3 Comments: ""The support was good, but I’m concerned about the internal wiring issue. I’ve had problems with this machine since I got it."" Source System: Field Service Management --- ### Customer Service Log 21 Customer Information", +Ben Thompson,11/23/24,799748,ben.thompson@email.com,-2321,"1155 Redwood Lane, Austin, TX 73301",4724657892," Ben reported that his TurboClean 900 washing machine is making a strange beeping noise after every wash cycle. The machine operates normally during the cycle, but the beeping happens when the cycle finishes, and it continues until the door is opened. Ben confirmed that there is no error code displayed, and the machine is functioning properly otherwise. He suspects it could be a settings issue or something related to software malfunctions. ",Olivia White,20 minutes," Olivia explained that the beeping could be a result of an alert setting indicating that the cycle has completed. She recommended checking the machine’s alarm settings and confirmed that no errors were logged. After troubleshooting, Olivia suggested that the issue might be due to a minor software glitch and escalated the case to SAP for further analysis. A technician was scheduled to check if a firmware update is needed. ","Yes, technician visit scheduled to check for software issue or"," Rating: 4 Comments: ""The beeping is annoying, but I appreciate the fast response. Hopefully, it’s just a simple fix."" Source System: SAP --- ### Customer Service Log 22 Customer Information", +Rachel Clark,11/24/24,800214,rachel.clark@email.com,-6886,"2300 Oak Drive, Boston, MA 02115",4730567884," Rachel reported that her WashPro 600 washing machine has a leaking door seal. She observed that after every wash cycle, there is water around the machine’s door, and it pools on the floor. Rachel confirmed that the door is properly closed during each cycle, but water still leaks out. She suspects that the door gasket is damaged or worn out. ",John Smith,18 minutes," John explained that a leaking door seal is usually caused by a worn gasket or improper sealing. He advised Rachel to check for any visible cracks or damage on the door seal. Since Rachel confirmed that the gasket seemed worn, John scheduled a Field Service Management technician to inspect the seal and replace the damaged gasket. ","Yes, technician visit scheduled to replace door seal."," Rating: 3 Comments: ""The support was helpful, but I’ve had this problem for a while. I really hope the technician fixes it this time."" Source System: Field Service Management --- ### Customer Service Log 23 Customer Information", +Lisa Roberts,11/25/24,801658,lisa.roberts@email.com,-5220,"540 Maplewood Ave, Denver, CO 80202",4731245793," Lisa’s SuperWash 1000 washing machine is not spinning correctly during the final rinse. She noted that although the machine agitates during the wash cycle, it fails to spin properly and her clothes are coming out soaking wet. She suspects a malfunction with the spin motor or clutch assembly. ",Charles Miller,22 minutes," Charles explained that the issue is likely related to the spin motor, clutch assembly, or even a broken spin belt. He guided Lisa through checking for any obstructions in the drum or any blockages in the drain hose, but since these appeared clear, he scheduled a technician visit through SAP to inspect the spin motor, clutch, and drain system. ","Yes, technician visit scheduled to inspect spin motor and clutch"," Rating: 4 Comments: ""Good support, but I’m getting frustrated that I can’t use the machine properly. Hope it gets fixed soon."" Source System: SAP --- ### Customer Service Log 24 Customer Information", +Amy Wilson,11/26/24,802341,amy.wilson@email.com,-7997,"1805 Pine Ridge Dr, Seattle, WA 98001",4732018439," Amy’s UltraSpin 4000 washing machine is displaying a ""drain error"" after completing a wash cycle. The machine stops draining water and remains stuck in the rinse cycle. Amy mentioned that she has tried checking the filter and confirmed it’s clean, but the issue persists. She suspects a problem with the drain pump or clogged hose. ",Stephanie Evans,40 minutes," Stephanie advised Amy that a drain error is often caused by a clogged drain hose or a faulty drain pump. She suggested checking if the hose was kinked, but since it wasn’t, she escalated the issue to Field Service Management for a technician visit. The technician would inspect the pump and the drain hose for potential clogs or malfunctions. ","Yes, technician visit scheduled to inspect drain pump and hose."," Rating: 5 Comments: ""Great service, and I’m relieved that the issue is being handled. Looking forward to the technician's visit."" Source System: Field Service Management --- ### Customer Service Log 25 Customer Information", +Mark Harris,11/27/24,803547,mark.harris@email.com,-665,"784 Cedar Street, New York, NY 10001",4733349576," Mark reported that his QuickWash 1200 washing machine’s drum is wobbling during spin cycles. The wobbling is so severe that the machine shakes and vibrates against the wall. Mark confirmed that the machine is level and that it’s not overloaded, but the problem persists. He suspects it could be an issue with the suspension system or balance rings. ",Paul Jacobs,15 minutes," Paul explained that a wobbling drum during the spin cycle is often due to issues with the suspension system, shock absorbers, or balance rings that keep the drum steady during high-speed spins. He advised Mark to try redistributing the load during a wash cycle, but since the issue continued, he arranged for a technician visit via SAP to inspect and repair the suspension system and balance rings. ","Yes, technician visit scheduled to inspect suspension system and"," Rating: 4 Comments: ""I’m glad the issue is being taken seriously. Hopefully, the technician can resolve the wobbling soon."" Source System: SAP --- ### Customer Service Log 26 Customer Information", +Samuel Green,11/28/24,804412,samuel.green@email.com,-4553,"678 Elmwood Drive, Phoenix, AZ 85001",4734571035," Samuel’s EcoWash 500 washing machine is having issues with water not draining fully during the spin cycle. He reported that the machine completes the wash cycle but leaves a significant amount of water in the drum. He confirmed that the filter is clear, and there are no visible blockages in the drain hose. Samuel suspects the drain pump may be malfunctioning. ",Karen Williams,28 minutes," Karen explained that the issue could likely be caused by a faulty drain pump, which would prevent water from being pumped out during the spin cycle. She advised Samuel to check the drain filter again and make sure there were no blockages. Since the issue persisted, Karen arranged a Field Service Management technician visit to inspect the drain pump and test its functionality. ","Yes, technician visit scheduled to inspect and replace drain pump."," Rating: 3 Comments: ""The machine isn’t draining properly, and I’m getting frustrated. I hope the technician can fix this soon."" Source System: Field Service Management --- ### Customer Service Log 27 Customer Information", +Helen Brown,11/29/24,805273,helen.brown@email.com,-8442,"302 Birch Street, Atlanta, GA 30301",4735621948," Helen reported that her SpeedWash 300 washing machine’s detergent dispenser is not working correctly. The detergent doesn’t fully dispense, and she noticed that the machine is leaving detergent residue in the compartment. Helen suspects an issue with the water inlet valve or the dispenser system. ",Benjamin Cook,25 minutes," Benjamin advised Helen that the problem is often due to a clogged detergent dispenser or a malfunctioning water inlet valve. He walked Helen through cleaning the dispenser tray and ensuring the water supply was working properly. After confirming the issue persisted, he scheduled a technician visit through SAP to inspect the water inlet valve and ensure the dispenser was functioning correctly. ","Yes, technician visit scheduled to inspect and replace water inlet"," Rating: 5 Comments: ""Thanks for your help. I’m glad this is being addressed quickly. I’m hopeful the technician can resolve the issue."" Source System: SAP --- ### Customer Service Log 28 Customer Information", +Henry Smith,11/30/24,806485,henry.smith@email.com,-9886,"442 Pinehill Road, San Jose, CA 95101",4736742017," Henry’s MaxSpin 1000 washing machine is stopping mid-cycle. The machine powers ob unexpectedly during the wash cycle, and sometimes, it won’t start at all. Henry tried resetting the machine and unplugging it, but the issue persists. He suspects a problem with the power board or electrical connections. ",Monica Taylor,35 minutes," Monica explained that sudden power losses mid-cycle are often caused by a faulty power board or a short circuit in the machine’s wiring. She recommended checking the error codes on the display, but since there were none, she scheduled a technician visit through Field Service Management to inspect the power board and electrical components. ","Yes, technician visit scheduled to inspect and replace power board."," Rating: 2 Comments: ""I’m really disappointed with the frequent breakdowns of this machine. I’ve had to call for service several times."" Source System: Field Service Management --- ### Customer Service Log 29 Customer Information", +Diana Evans,12/1/24,807220,diana.evans@email.com,-7553,"2001 Aspen Court, Dallas, TX 75001",4737853099," Diana’s QuickClean 700 washing machine is displaying a ""water level sensor"" error. The machine seems to be overfilling during cycles, and Diana confirmed that the water level sensor may need to be replaced. She tried recalibrating the machine, but the issue persists. ",Jason Lee,18 minutes," Jason confirmed that a faulty water level sensor could be causing the issue and guided Diana through resetting the sensor. Since resetting didn’t resolve the error, he scheduled a technician visit through SAP to replace the water level sensor and calibrate it. ","Yes, technician visit scheduled to replace water level sensor."," Rating: 4 Comments: ""Good support, but I’m hoping the sensor replacement fixes the problem for good."" Source System: SAP --- ### Customer Service Log 30 Customer Information", +Julia Green,12/2/24,808451,julia.green@email.com,-2220,"1250 Hilltop Blvd, Miami, FL 33101",4741102358," Julia’s EcoSpin 2000 washing machine has a persistent odor after each wash cycle, even after running an empty cycle with vinegar. She mentioned that the smell seems to come from the drum, and she suspects there may be mold or mildew buildup, despite cleaning the machine regularly. ",Liam Harris,30 minutes," Liam explained that odors are commonly caused by mold or mildew buildup in the door seal or drum. He advised Julia to check the drain pump filter and door seal for residue and moisture. After confirming the issue persisted, he scheduled a technician visit via SAP to perform a thorough cleaning and inspection of the drum, pump, and seals to eliminate the odor. ","Yes, technician visit scheduled to clean and inspect drum and"," Rating: 3 Comments: ""I’m disappointed that the smell keeps coming back. I hope the technician can fix it for good."" Source System: SAP --- ### Customer Service Log 31 Customer Information", +Darren King,12/3/24,809276,darren.king@email.com,-3998,"9457 Oakwood Drive, Orlando, FL 32801",4742239081," Darren’s SpinMaster 500 washing machine is experiencing dibiculty completing the wash cycle. The machine often gets stuck on the wash phase, and Darren needs to restart the cycle to complete it. He suspects an issue with the motor or timer mechanism. ",Rachel Scott,25 minutes," Rachel explained that a machine stuck on the wash phase is often due to a faulty timer or motor not completing the cycle. She walked Darren through resetting the machine and checking for any error codes. Since the issue persisted, Rachel escalated the issue to Field Service Management for an on-site technician visit. The technician would inspect the motor, timer, and control board to identify the root cause. ","Yes, technician visit scheduled to inspect motor and timer."," Rating: 4 Comments: ""I’m glad the support team is investigating. It’s frustrating to restart the cycle every time."" Source System: Field Service Management --- ### Customer Service Log 32 Customer Information", +Martin Edwards,12/4/24,810112,martin.edwards@email.com,-998,"2030 Birchwood St, San Diego, CA 92101",4743247621," Martin’s QuickClean 1500 washing machine is not agitating during the wash cycle. The drum remains still, and the clothes are not being washed properly. Martin checked the agitator and ensured the drum was not overloaded, but the issue persists. ",Sarah White,18 minutes," Sarah explained that a lack of agitation can be caused by a faulty agitator motor, coupling, or belt. She guided Martin through verifying the belt and the agitator for any visible damage. Since the issue continued, Sarah escalated the case to SAP for a technician to inspect the agitator motor and coupling for potential failure. ","Yes, technician visit scheduled to inspect agitator motor and"," Rating: 3 Comments: ""The support was decent, but I really need the agitator fixed. It’s making washing clothes a hassle."" Source System: SAP --- ### Customer Service Log 33 Customer Information", +Emily White,12/5/24,811439,emily.white@email.com,-10441,"1287 Sunset Blvd, Los Angeles, CA 90001",4744365920," Emily’s WashSmart 800 washing machine has a noisy spin cycle. The machine makes a loud, high-pitched noise when it reaches full spin speed. Emily confirmed there are no clothes caught in the drum, and the machine seems level. She suspects a problem with the drum bearings or drive belt. ",Greg Stone,27 minutes," Greg explained that the noise is likely caused by worn drum bearings or a loose drive belt. He walked Emily through checking for any visible issues with the belt and drum but confirmed that these appeared fine. Since the noise persisted, he scheduled a Field Service Management technician visit to inspect and possibly replace the drum bearings or drive belt. ","Yes, technician visit scheduled to inspect and replace drum"," Rating: 4 Comments: ""The noise is getting really annoying. Hopefully, the technician can fix it soon."" Source System: Field Service Management --- ### Customer Service Log 34 Customer Information", +Patricia Moore,12/6/24,812876,patricia.moore@email.com,-7442,"3564 Oakhill Drive, Atlanta, GA 30301",4745523477," Patricia reported that her SuperWash 1200 washing machine stopped mid-cycle and won’t restart. She mentioned that the machine was working fine until it suddenly turned ob during the rinse cycle. She has tried pressing the reset button and unplugging the machine, but it still won't power back on. ",Jason Brooks,40 minutes," Jason explained that a machine turning ob mid-cycle could be caused by a power board failure, faulty wiring, or overheating protection. He guided Patricia through checking the plug and outlet for issues but confirmed there were none. He then escalated the case to SAP for a technician visit to inspect and possibly replace the power board and test the internal wiring. ","Yes, technician visit scheduled to inspect and replace power board."," Rating: 2 Comments: ""Very frustrating that the machine just turned ob. I’ve had enough of these power issues."" Source System: SAP --- ### Customer Service Log 35 Customer Information", +Vanessa James,12/7/24,813492,vanessa.james@email.com,-2442,"2795 Pine View Rd, Chicago, IL 60602",4746601093," Vanessa’s WashPro 2000 washing machine is draining very slowly during the rinse cycle, and the drum is often left with standing water. She confirmed that the filter is clean, and the drain hose appears clear. She suspects a malfunction in the drain pump or a blockage in the internal pump hose. ",Lisa Clark,33 minutes," Lisa advised Vanessa that slow draining could indicate a faulty drain pump or a blockage in the pump hose. She recommended checking the pump filter for any residual debris. Since the issue persisted, Lisa scheduled a technician visit via Field Service Management to inspect and test the drain pump and pump hose for clogs or faults. ","Yes, technician visit scheduled to inspect and replace drain pump."," Rating: 5 Comments: ""I’m glad the technician is coming soon. I need the machine to drain properly before I can use it again."" Source System: Field Service Management --- ### Customer Service Log 36 Customer Information", +Sophia Evans,2024-12,814256,sophia.evans@email.com,-8664,"1985 Crestwood Lane, Houston, TX 77001",4747703817, Sophia reported that her TurboClean 500 washing machine is having an issue with water temperature. The machine is stuck on cold wash even when she selects a hot water setting. She suspects the heating element or temperature sensor is malfunctioning. ,Robert Turner,22 minutes," Robert explained that a washing machine stuck on cold wash is typically due to a faulty heating element or temperature sensor. He guided Sophia through checking the water inlet valve to ensure it's working properly. Since the problem persisted, Robert arranged for a technician visit via SAP to replace the heating element and test the temperature sensor for functionality. ","Yes, technician visit scheduled to inspect and replace heating"," Rating: 4 Comments: ""I appreciate the support. I just need the temperature issue fixed quickly so I can use the machine again."" Source System: SAP --- ### Customer Service Log 37 Customer Information", +Carla Peterson,12/9/24,815774,carla.peterson@email.com,-10997,"1368 Grand Avenue, San Francisco, CA 94110",4748865105," Carla’s EcoWash 1200 washing machine is not filling with water properly during the wash cycle. The machine starts, but no water enters the drum, leaving the clothes dry. Carla checked the water supply and the inlet valve and confirmed they are both working correctly. ",Matthew Lee,30 minutes," Matthew explained that no water filling the drum could be caused by a faulty water inlet valve or a clogged filter. He walked Carla through checking the valve filter and the supply hoses for blockages. Since the issue persisted, Matthew arranged for a technician visit via Field Service Management to inspect and replace the water inlet valve. ","Yes, technician visit scheduled to inspect and replace water inlet"," Rating: 3 Comments: ""I’m frustrated that the machine isn’t working. I’m hoping the technician can fix the issue soon."" Source System: Field Service Management --- ### Customer Service Log 38 Customer Information", +Edward Ross,12/10/24,816042,edward.ross@email.com,-6442,"4507 Pine Hill Road, Portland, OR 97201",4749981340," Edward reported that his SpinWash 1800 washing machine’s door won’t lock properly. The machine starts a wash cycle but stops after a few minutes, showing a door lock error. Edward confirmed that there is no obstruction in the door mechanism, and the door appears aligned. ",Karen Fisher,24 minutes," Karen explained that a door lock error can be caused by a faulty door latch or door lock mechanism. She advised Edward to check for any visible damage to the door components. Since the issue continued, Karen scheduled a technician visit via SAP to inspect and replace the door lock mechanism. ","Yes, technician visit scheduled to inspect and replace door lock."," Rating: 4 Comments: ""The issue seems to be with the door. I hope the technician can get it fixed without much hassle."" Source System: SAP --- ### Customer Service Log 39 Customer Information", +Marcus Johnson,12/11/24,817592,marcus.johnson@email.com,-3220,"1584 Redwood Ave, Sacramento, CA 95814",4751112378," Marcus reported that his TurboClean 500 washing machine was not draining at the end of the cycle. He mentioned that there was standing water in the drum after each wash, even though the drain hose was clear. Marcus suspected an issue with the drain pump or filter. ",Anna Gray,28 minutes," Anna advised Marcus to clean the drain filter and inspect the hose for any visible clogs. Since the issue persisted, she scheduled a technician visit via Field Service Management to inspect and replace the drain pump if necessary. ","Yes, technician visit scheduled to inspect and replace drain pump."," Rating: 4 Comments: ""I’m frustrated with this issue, but the support was decent. Hopefully the technician can finally fix it."" Source System: Field Service Management --- ### Customer Service Log 40 Customer Information", +Linda Anderson,12/12/24,818130,linda.anderson@email.com,-7108,"1983 Pinecrest Way, Denver, CO 80202",4752203899," Linda’s WashMaster 3000 washing machine is stopping mid-cycle without completing the wash or spin phases. She has tried resetting the machine multiple times, but it keeps shutting ob randomly. She suspects a problem with the timer or control board. ",Oliver Harris,22 minutes," Oliver walked Linda through checking the error codes and suggested resetting the machine multiple times. After confirming the issue persisted, he escalated the case to SAP for a technician visit to inspect the control board and timer. ","Yes, technician visit scheduled to inspect and replace control"," Rating: 5 Comments: ""Thank you for helping. I hope this issue is resolved. The machine has been a nightmare for the past week."" Source System: SAP --- ### Customer Service Log 41 Customer Information", +Frank Lewis,12/13/24,819444,frank.lewis@email.com,-10108,"2156 Horizon St, Chicago, IL 60611",4753401755," Frank’s MaxSpin 2000 washing machine is making a loud banging noise during the spin cycle. Frank mentioned that he made sure the machine was properly leveled, but the noise is still very loud and disruptive. He suspects an issue with the drum bearings or suspension springs. ",Kelly Morgan,18 minutes," Kelly explained that the noise could be due to worn drum bearings or suspension springs. She advised Frank to check the suspension springs for any visible damage. Since the issue persisted, Kelly arranged a technician visit via Field Service Management to inspect and replace the drum bearings. ","Yes, technician visit scheduled to inspect and replace drum"," Rating: 3 Comments: ""The noise is unbearable. I can’t stand it anymore. Please get this fixed ASAP!"" Source System: Field Service Management --- ### Customer Service Log 42 Customer Information", +Nathan Parker,12/14/24,820289,nathan.parker@email.com,-6886,"3327 Maple St, Phoenix, AZ 85001",4754536791," Nathan reported that his QuickWash 1000 washing machine won’t start after selecting a wash cycle. The control panel lights up, but the machine does not initiate any wash actions. Nathan suspected a malfunction with the control board or start switch. ",Jasmine Lee,35 minutes, Jasmine walked Nathan through checking the control panel and ensuring the door was properly closed. She confirmed the issue persisted and scheduled a technician visit via SAP to inspect and replace the control board or start switch. ,"Yes, technician visit scheduled to inspect and replace control"," Rating: 2 Comments: ""I am so sick of this machine. This is the third time I’ve had problems! I just want a replacement!"" Source System: SAP --- ### Customer Service Log 43 Customer Information", +Amanda Green,12/15/24,821377,amanda.green@email.com,-9219,"9084 Valley Rd, Portland, OR 97223",4755641032," Amanda’s EcoSpin 2000 washing machine is not spinning properly during the final cycle. The clothes are coming out soaking wet, and the machine appears to be underperforming on the spin cycle. She suspects a problem with the drum motor or drive belt. ",Thomas White,20 minutes," Thomas advised Amanda that a malfunctioning drum motor or drive belt could cause incomplete spinning. After troubleshooting, he escalated the issue to Field Service Management for a technician visit to inspect and replace the drum motor or drive belt. ","Yes, technician visit scheduled to inspect and replace drum motor."," Rating: 5 Comments: ""It’s been frustrating, but the support has been helpful. Hopefully, the technician can get this fixed."" Source System: Field Service Management --- ### Customer Service Log 44 Customer Information", +Kevin Taylor,12/16/24,822651,kevin.taylor@email.com,-5220,"1223 Oakridge Drive, Las Vegas, NV 89101",4756724389," Kevin’s WashPro 1200 washing machine is not dispensing detergent during the wash cycle. Despite adding detergent to the dispenser, it remains in the compartment, and the clothes don’t seem to be cleaned properly. Kevin suspects a blockage in the detergent dispenser system. ",Olivia Moore,30 minutes," Olivia explained that detergent not dispensing could be caused by a clogged dispenser or a malfunctioning water inlet valve. After guiding Kevin through cleaning the dispenser, she confirmed the issue persisted and escalated the issue to SAP for a technician visit to inspect the dispenser system and water valve. ","Yes, technician visit scheduled to inspect and clean dispenser"," Rating: 4 Comments: ""I’m annoyed that the detergent isn’t dispensing correctly. I expect a better product."" Source System: SAP --- ### Customer Service Log 45 Customer Information", +Alicia Harris,12/17/24,823493,alicia.harris@email.com,-7553,"3121 Elm St, New York, NY 10001",4757834098," Alicia’s SpinMaster 1000 washing machine is producing a burning smell during the spin cycle. She said the smell is noticeable even when no heavy loads are being washed. Alicia is concerned it might be a sign of a more serious issue, possibly with the motor or wiring. ",Daniel Clark,25 minutes, Daniel recommended that Alicia unplug the machine immediately as a safety precaution and not use it until it’s inspected. He scheduled a technician visit via Field Service Management to inspect and potentially replace the motor or wiring that could be causing the burning smell. ,"Yes, technician visit scheduled to inspect and replace motor."," Rating: 5 Comments: ""I’m scared to use it after that smell. Thank you for the fast response. Please fix this!"" Source System: Field Service Management --- ### Customer Service Log 46 Customer Information", +Julie Roberts,12/18/24,824006,julie.roberts@email.com,-9886,"2217 Sun Valley Dr, Dallas, TX 75201",4758957320," Julie’s EcoWash 1500 washing machine is leaking water from underneath the machine. The leak is noticeable when the machine starts the wash cycle, and it leaves puddles around the machine. Julie suspects a problem with the water pump or drain hose. ",Nathan Collins,30 minutes," Nathan guided Julie to check the water hoses and pump filter for any visible leaks or damage. Since the leak continued, he scheduled a technician visit via SAP to inspect and replace the water pump or hose. ","Yes, technician visit scheduled to inspect and replace pump."," Rating: 1 Comments: ""This is ridiculous! The machine has been leaking for days and no one is fixing it. I’m done with this brand!"" Source System: SAP --- ### Customer Service Log 47 Customer Information", +Brian Mitchell,12/19/24,825137,brian.mitchell@email.com,-1442,"2571 Briarwood Dr, Austin, TX 73301",4760018825," Brian’s SmartWash 700 washing machine won’t drain properly at the end of the cycle. Water remains in the drum after completing the cycle, and the clothes are still soaked. Brian tried checking the drain hose and filter but the problem persists. ",Helen Scott,25 minutes," Helen explained that the issue could be caused by a faulty drain pump or blockage in the drain hose. After troubleshooting, she scheduled a technician visit via Field Service Management to inspect and replace the drain pump. ","Yes, technician visit scheduled to inspect and replace pump."," Rating: 3 Comments: ""I’ve had enough of these problems. If the technician doesn’t fix it soon, I’ll demand a refund."" Source System: Field Service Management --- ### Customer Service Log 48 Customer Information", +Thomas Williams,12/20/24,826255,thomas.williams@email.com,-10886,"1152 Ashwood Dr, Minneapolis, MN 55123",4761134120," Thomas reported that his UltraWash 3000 washing machine is vibrating excessively during the spin cycle, causing the whole unit to shake and move across the floor. He mentioned that he had leveled the machine but the issue continues. ",Mark Johnson,22 minutes," Mark explained that an unbalanced load or worn suspension springs could be causing the vibration. He guided Thomas through checking the load distribution and inspecting the suspension springs. Since the issue persisted, Mark scheduled a technician visit via SAP to inspect and replace the suspension springs. ","Yes, technician visit scheduled to inspect and replace suspension"," Rating: 4 Comments: ""I’m glad someone’s coming to fix it, but I’ve had enough of this. I just want the machine to work properly!"" Source System: SAP --- ### Customer Service Log 49 Customer Information", +Jessica Bryant,12/21/24,827404,jessica.bryant@email.com,-2442,"4632 Birchwood Rd, San Diego, CA 92103",4762231882," Jessica’s SmartWash 500 washing machine is not agitating properly during the wash cycle. The drum spins slowly, and clothes are not being cleaned as expected. Jessica believes the agitator or motor is malfunctioning. ",Nancy Hall,30 minutes," Nancy walked Jessica through checking the agitator for blockages and confirmed it was moving improperly. Since the issue wasn’t resolved, she escalated the case via Field Service Management for a technician visit to inspect and replace the agitator or motor. ","Yes, technician visit scheduled to inspect and replace agitator."," Rating: 5 Comments: ""The support team was helpful, but I’m really annoyed this happened right after the warranty expired."" Source System: Field Service Management --- ### Customer Service Log 50 Customer Information", +Natalie Cooper,12/22/24,828552,natalie.cooper@email.com,-6331,"1056 Kings Rd, Orlando, FL 32801",4763347501," Natalie’s TurboSpin 1200 washing machine is leaking water from the bottom panel during the wash cycle. The water pools on the floor around the machine, and Natalie suspects a problem with the water pump or seal. ",Leo Davis,26 minutes," Leo explained that a water leak could be caused by a damaged water seal or faulty water pump. He advised Natalie to inspect the water hose and drain filter for any visible damage. Since the leak persisted, Leo scheduled a technician visit via SAP to inspect and replace the water pump or seal. ","Yes, technician visit scheduled to inspect and replace water pump."," Rating: 2 Comments: ""This is the second time my machine is leaking. I’m getting tired of this brand. FIX IT."" Source System: SAP --- ### Customer Service Log 51 Customer Information", +Adam Rodriguez,12/23/24,829701,adam.rodriguez@email.com,-4331,"2987 Oak Valley Dr, Miami, FL 33101",4764450613," Adam’s QuickWash 1500 washing machine is not spinning at all during the final spin cycle. The clothes are coming out wet and heavy, and the drum does not rotate. Adam has checked the door lock and load balance, but the problem persists. ",Carla Moore,24 minutes," Carla explained that the issue could be due to a broken drive belt or faulty motor. She advised Adam to inspect the drive belt for any visible damage. Since the issue wasn’t resolved, Carla scheduled a technician visit via Field Service Management to inspect and replace the drive belt or motor. ","Yes, technician visit scheduled to inspect and replace drive belt."," Rating: 5 Comments: ""I’m hoping this gets fixed soon. I’ve already had too many issues with this machine."" Source System: Field Service Management --- ### Customer Service Log 52 Customer Information", +Barbara Grant,12/24/24,830014,barbara.grant@email.com,-5664,"5743 Riverside Ave, Austin, TX 73301",4765564392," Barbara’s EcoSpin 1000 washing machine is overheating during the wash cycle. The temperature sensor seems to be malfunctioning as the water gets hotter than expected, potentially damaging clothes. Barbara has confirmed that the thermostat and water inlet valve are functioning. ",Daniel Scott,33 minutes, Daniel explained that overheating can be caused by a faulty temperature sensor or malfunctioning control board. He escalated the issue via SAP and scheduled a technician visit to inspect and replace the temperature sensor or control board. ,"Yes, technician visit scheduled to inspect and replace temperature"," Rating: 3 Comments: ""The machine shouldn’t be overheating like this. It’s frustrating, especially since it’s brand new!"" Source System: SAP --- ### Customer Service Log 53 Customer Information", +Eric Foster,12/25/24,831179,eric.foster@email.com,-2553,"7455 Hilltop Ln, Los Angeles, CA 90001",4766678245," Eric’s SpinClean 800 washing machine is not draining water at the end of the cycle. The water remains in the drum, and the clothes are still soaking wet. Eric has already cleaned the drain filter and checked the drain hose but the issue persists. ",Helen Taylor,20 minutes, Helen explained that the issue could be with the drain pump or a blockage in the drain hose. She arranged a technician visit via Field Service Management to inspect the pump and hose for any obstructions. ,"Yes, technician visit scheduled to inspect and replace drain pump."," Rating: 1 Comments: ""I’ve had enough of this junk! It’s been a month of issues and no one’s fixing it! Get this thing replaced."" Source System: Field Service Management --- ### Customer Service Log 54 Customer Information", +Samuel Lewis,12/26/24,832254,samuel.lewis@email.com,-998,"1586 Willow Creek Rd, Dallas, TX 75201",4767789337," Samuel’s WashMaster 4000 washing machine is producing a strange odor during the wash cycle. He suspects the drain pump or filter might be clogged, but he cannot find any visible issues. ",Anna Walker,30 minutes," Anna advised Samuel to clean the drain pump and filter thoroughly. After confirming the issue persisted, she scheduled a technician visit via SAP to inspect the pump and filter system for any internal clogs. ","Yes, technician visit scheduled to inspect and clean pump."," Rating: 4 Comments: ""The odor is unbearable, but at least I have someone coming to fix it."" Source System: SAP --- ### Customer Service Log 55 Customer Information", +Rachel White,12/27/24,833411,rachel.white@email.com,-3998,"3185 Creekside Dr, San Francisco, CA 94105",4768890246," Rachel’s MaxSpin 2000 washing machine is not completing the wash cycle. The machine shuts ob halfway through, and she needs to restart it multiple times to finish the cycle. She suspects an issue with the timer or control board. ",Jennifer Miller,29 minutes," Jennifer advised Rachel to check for any error codes or obstructions. Since the issue persisted, she escalated the issue to Field Service Management for a technician visit to inspect and replace the timer or control board. ","Yes, technician visit scheduled to inspect and replace control"," Rating: 2 Comments: ""I don’t care about the technician visit. Just fix this machine! It’s been down too long."" Source System: Field Service Management --- ### Customer Service Log 56 Customer Information", +Peter Knight,12/28/24,834558,peter.knight@email.com,-8553,"2450 Glenwood Rd, Seattle, WA 98101",4770003942," Peter’s TurboClean 1500 washing machine is not starting when he presses the power button. The lights on the control panel come on, but it doesn’t respond when he selects a cycle. Peter suspects a problem with the start button or control board. ",Brian King,27 minutes," Brian walked Peter through checking the power connections and error codes. After confirming the issue persisted, he escalated the case via SAP and scheduled a technician visit to inspect the control board. ","Yes, technician visit scheduled to inspect and replace control"," Rating: 4 Comments: ""I’m beyond frustrated at this point. This machine is brand new and it’s already broken!"" Source System: SAP --- ### Customer Service Log 57 Customer Information", +Erica Matthews,12/29/24,835744,erica.matthews@email.com,-5331,"9725 Maplewood St, Portland, OR 97205",4771123567, Erica's EcoWash 1000 washing machine is making a loud grinding noise during the wash cycle. She suspects an issue with the drive belt or motor. ,James Carter,28 minutes, James explained that the grinding noise could be due to a worn drive belt or damaged motor bearings. He scheduled a technician visit via SAP to inspect and potentially replace the drive belt or motor. ,"Yes, technician visit scheduled to inspect and replace drive belt."," Rating: 3 Comments: ""It’s been so frustrating. I don’t know how this machine got so loud so quickly. Let’s see if the technician can fix it."" Source System: SAP --- ### Customer Service Log 58 Customer Information", +Michael Spencer,12/30/24,836011,michael.spencer@email.com,-10219,"1156 Eagle St, Denver, CO 80203",4772234568," Michael reported that his SpinMaster 2500 washing machine isn’t spinning at all. The drum remains stationary, and clothes are still soaked after the cycle. ",Emily White,24 minutes," Emily suggested that the drum motor could be faulty, or there could be an issue with the drive belt. She scheduled a technician visit via Field Service Management to inspect and replace the motor or drive belt. ","Yes, technician visit scheduled to inspect and replace motor."," Rating: 5 Comments: ""Quick and ebicient response. I hope the technician can fix it soon."" Source System: Field Service Management --- ### Customer Service Log 59 Customer Information", +Jennifer Thomas,12/31/24,836895,jennifer.thomas@email.com,-7442,"3446 Sunridge Ave, Phoenix, AZ 85001",4773345569," Jennifer's MaxSpin 3000 washing machine is not draining properly, and water is left standing in the drum after each cycle. She believes there may be a blockage in the drain hose or filter. ",Laura Phillips,21 minutes," Laura advised Jennifer to clean the drain filter and inspect the hose for obstructions. When the issue wasn’t resolved, Laura escalated the case via SAP to schedule a technician visit to check the drain pump and hose. ","Yes, technician visit scheduled to inspect and clean the drain pump."," Rating: 4 Comments: ""The technician better sort this out. I’ve cleaned everything and it’s still broken."" Source System: SAP --- ### Customer Service Log 60 Customer Information", +Steve Harvey,1/1/25,837112,steve.harvey@email.com,-6664,"7900 Northview Dr, Seattle, WA 98109",4774456670, Steve reported that his QuickClean 5000 washing machine’s display screen is flickering and showing random error codes. He’s unable to start a cycle because of this malfunction. ,Sam Collins,19 minutes, Sam explained that the issue could be related to the control board or wiring. He scheduled a technician visit via Field Service Management to inspect and replace the control board. ,"Yes, technician visit scheduled to inspect and replace control"," Rating: 2 Comments: ""How hard is it to make a washing machine that actually works? I’m sick of having to call for repairs."" Source System: Field Service Management --- ### Customer Service Log 61 Customer Information", +Karen Brooks,1/2/25,838225,karen.brooks@email.com,-2553,"6715 Valley Dr, Chicago, IL 60601",4775567781," Karen’s EcoSpin 2000 washing machine is overheating during the wash cycle. The water temperature is rising too high, potentially ruining clothes. ",Patricia Lee,32 minutes, Patricia explained that overheating could be caused by a malfunctioning thermostat or control board. She scheduled a technician visit via SAP to inspect and replace the thermostat or control board. ,"Yes, technician visit scheduled to inspect and replace thermostat."," Rating: 3 Comments: ""This is not what I expected from a high-end machine. Can you get this sorted out quickly?"" Source System: SAP --- ### Customer Service Log 62 Customer Information", +David King,1/3/25,839478,david.king@email.com,-3775,"8903 Pinehurst Dr, Houston, TX 77001",4776678892," David’s TurboWash 800 washing machine is not starting at all. The power button doesn’t respond, and there are no lights on the control panel. ",Brian Martin,25 minutes," Brian suggested checking the power supply and circuit breaker. Since the machine still wouldn’t start, he escalated the case via Field Service Management to schedule a technician visit to inspect the control board and power connections. ","Yes, technician visit scheduled to inspect and replace control"," Rating: 1 Comments: ""This is insane. I’ve only had this machine for a year, and it’s already broken. This brand is trash."" Source System: Field Service Management --- ### Customer Service Log 63 Customer Information", +Angela Hill,1/4/25,840325,angela.hill@email.com,-8997,"1108 Crestwood Rd, Los Angeles, CA 90015",4777789921, Angela’s SmartWash 7000 washing machine is not spinning clothes dry. The machine goes through the wash cycle but leaves the clothes soaking wet. ,Kevin Clark,27 minutes, Kevin suggested that the spin motor or pump could be malfunctioning. He arranged a technician visit via SAP to inspect and replace the spin motor or pump. ,"Yes, technician visit scheduled to inspect and replace motor."," Rating: 4 Comments: ""I need this fixed. I can’t abord to keep waiting for a solution."" Source System: SAP --- ### Customer Service Log 64 Customer Information", +Richard Young,1/5/25,841196,richard.young@email.com,-9553,"4822 Oakview Dr, Dallas, TX 75209",4778891104, Richard’s SpinForce 2500 washing machine is stopping midway through the wash cycle and not draining the water . ,Olivia Stone,34 minutes," Olivia suggested checking for any blockages in the drain hose and pump. Since the issue wasn’t resolved, she escalated the case via Field Service Management to schedule a technician visit to inspect the pump and drain hose. ","Yes, technician visit scheduled to inspect pump."," Rating: 2 Comments: ""I’m so done with this machine. I can’t trust it to work. Just get it fixed already."" Source System: Field Service Management --- ### Customer Service Log 65 Customer Information", +Lisa Green,1/6/25,842401,lisa.green@email.com,-5775,"5224 Ashford Ave, Miami, FL 33165",4779912335, Lisa’s TurboClean 1500 washing machine is vibrating excessively during the spin cycle and making an unusual screeching noise. ,Scott Gray,29 minutes," Scott explained that the issue could be related to unbalanced loads or a faulty suspension system. Since the issue persisted, he escalated the case via SAP and scheduled a technician visit to inspect the suspension springs and balance. ","Yes, technician visit scheduled to inspect suspension springs."," Rating: 3 Comments: ""I’ve leveled the machine, but it’s still vibrating like crazy. Something’s clearly wrong."" Source System: SAP ", \ No newline at end of file diff --git a/examples/notebooks/hap/input/data.parquet b/examples/notebooks/hap/input/data.parquet new file mode 100644 index 0000000000000000000000000000000000000000..1d8bf2e6468b5457e0651836cc448cd25a57a394 GIT binary patch literal 47309 zcmeEudw3Jo*7xjohR!6D-DH{!Y1)RF(v+sQLz6bW38g7bp_LYBxdzZ?|Wg8o>S0sd-?%I!)ZAPdcMB<2q(OVwLi!g7}|x62<> z82GChzCIN^>WCs1cX^#;h-E%hij9&g$<*(#%~%prOjym%d} zL}F1igLVh|ioNIw7M{IG_X}#yZj*y?7=tNdJsOiYRS7_6>LTG@T{`;QwgFl5>iu#^ z7Cb$2$X3lZ#RLB0<&-rKMgwj<%OC1Qj}gtbhs}b=*DFVSREg0O^g~Z z%39=@9Pq=~hr``4_ME#s4;tr#DT^yhQNUOyN806x*h9nZVvpc~?g|azXpGraym$7d zW~(iZUbRi2g|kBLO{g7B5>(BM#k!PU&p=tT-`gd_*tc~nA+Ms4*-c6)QrFlra}kMZlV==XGMI%yl_0! zsp>sU{D`WDOoVA2GF^{~)a9i&jWUhOE++~!e`u8%tE%@$*WlYzk55U3Un>ww%UHhr zg4zNzUbV7)sH@VtZk4ZY+u72MS9Ux*yrgE!x~}@dn_9MByKwiyoeS>2G0<@L!O)|( zgkJdKr)9P90QfKex%6xko9QA(=`+o`$?<3`98@Ag3uu8Ug_`Sky; zU71B-DhN{w0YVSI2m=1TRcrc$oMyTxX*ph_3ewv^n$sFFDx+3l$(iQ*#lq?bO4fc`yS{AZb#eKo zhZk=;nHZ?tFxYq7w#Od0_n*&fpYX=qEK?f^k;15tAq zh?oBk;@1a2T>6Omug5`bc^bssLmCvQ6Y^`=R}p{b;Nou^!K8_(N&=f4Am; zZqEL{XyRWg)lrM-L;NqLdQjIj8oHvWC`4K~finvpy59!&Wf@O(Y*cljE;1${{LEa9gK}P9(32PBPmj@bHPO z0u-q?2~aXEC}<2P5el`6=yqxmnjv(l07VKYy^^1xW&lEwB7jiJ=OJV%3aAi8>Ppq-g}YnTGZ^eMTS)8IjTWmMTOC0H}OU15n9%1g)UuU|-#&6tRqjaa&Zk2_eNC zO=1s8zU+sWZ&a*L@;_~V%Q&47}PlYRz0&pegdXz~oj0YOJ;qwAt60t~& z0I^iH6K3}o&p*tPlEf_4Ed;3q#fPf*ryy5J_i4-`1p%|1cS1g=2%e>a772i$k~71W zN`#f3>PHDQvjBP;kfm-Fx71yh!raWl$VGFeBbO1{4A7hY|qhlx9@(^1L75g-G@13>2Di_z>Nc$8sV^LtdQ#D} z{V{h={&0K0GRKK zzyyt0udO-!Rzv>vj{IwL)xGckwy}R{#qOs}52Q{r%}D~Hw^QhEzy6IvWbqeN8vR@6 zQLy*}he4R$265x}u(ka1j(YvcH>fo7w_9|TWczX08uTB*<#*l%am$+^JfFh8VbXPz zU;h9XEl1#^XFmkd3EM4M@fCi$Tc(lHqEDD(yXu^e*V(SW5e;mzJ$6SSd>`PZ_I@TI7-We>GC3>Ee7i#RwT>_M`8i6 zG)2PnR-vlGSyE<4`NtbdoGxJoc*BCbM+wE1BB2eOYSoSM`lH@(kucdURFyeODs1q` zvQno|=Z|>1z!{L?LDRxf@VmgxP@;^zproY4A;4V~l`iU{S@jWr4?Jyh7x+w_@SMhJ zLU~C!lXtwf%HdRf^gy^*mbl29_E1@1G0;@C1S`wyZ_z z1+OQvs7Pp;Cpgf8dklg5c4WoQNnyr zm$^!rY19w4`kJ2+>9mVoWQjv;FhcWuol5akVp_{tDf%L4QmOca?JzZ`@eK4z!yPHM zFIlIfvIH%lBx+?XD6A6lFM*+-FI2*4I;c0y4wp-4ReaPqW;W@L&s#v2mNqURou&3~ z9M6$;VdkW{*)5bf9S$l&yUI#(2B>Ite~M=U4Ua8;n)$K59%iXc0UtL2vpyI4R#`lS zuBa?){@!s7(NcF`irYt*S5&(9Q_nPeTfZRa(h{fr2Gb0}onT60FkDmILMe>2!zRB+ zmN^^CXi%g1r76(C#5}?&A1QLHI}3Q^@kp$w{7?!}!sHh@oUW3j8ypLF7k}cMpTfl1 zsxnuF>+ibVb-$#z_rW*qtg6?I=aFr()(cbRaRo5w6+&A)6jfrnC9UVCpl|6?SNUKb z^{sAMT{aRI(awsh;wQ{={Gea0zJ@nR!inz8b5vEjgvnsQW47xX&#o&3jYw@jhdnucpb+{_aW*_vdh4tr+0GbHJ?)?Yrs&tg0 z{VY7S6g0q#wr^{%*_mQHUgU6<3sb`p|I%5VL5fv(#-QBmTsg;v3AV zuBxmMdFchBdS1m;&Ikr?(sE%9t ziEVlqb7c{AM@=1aze$!l8h<6>&x^!5bYj6aDjNn{m;#Wn%Id__`dqrg>1sTiDsV5a zsQ~41xwVAD{WsPlcLo|0)i0&M%E)0$%gV|VS&j3RoVu+ksGzK(3T#RX7}2gCm~&=N z^)0Xz4^L#vD;zFq9jeYu9p>4}s*+Nrb9k-D*Ks6K;66z^N-9Np?F9Gurky0|aG^f7 ztkUUp>r9)guSV3Zq|3>#r0I@uq+fC0UGgVGSg!ise-W9nf$esU7{8>oYWpkO?lRjQ z|EHE6EW!UIruk&fADBkp*5&W+hAIB9ux8&G{H>P!pWE`EpP<5;|M?#M3yT&v(dUr zN>48@*bMUA;{?b}bKoxeA16W}m#n!0r03U{K+gVZJIDb44anO%K7uDre=a2xolU=W zavpqib7eosPp-NJp)Mv-~FrH$CE0Ur%TI)2B;5W^Hk(j4A!C?ip9WEBAqopw@b-Dt&0GEZ^n=jC8-0L7 zqVV=WI1Z@L5ApxbI7AtQcJR^ESBAp=s3Hl>%K$X}F+Xfceei^E9C91#lbErz5cDIx zxm|${1Z4D8HuNb9ao`yBxX~Yi9YCSRPxVFO!6Nt=f)WA*3cQOHpUKP$U^)@GPVsk! zgmC*p#T$!)SG@>s3Bxyqa72g(;e~=1ayQI8b4&?Fp`R$%82b92L?lQ}*F!HuVN>0# zUL~Tyuqja(0k|dH3nBJM5!FL|XuCu22W%Wg50^lsKkARh1h^IwZ2d)$|MA1~q3g1b z36lOODxlD@KtLNveri9Dmsb6=nBz3@Nn@FcgL z?torCUI{OPU`b~v41JXZm-|!m>D*f5OL};b94cn?NyS2KTiVpvRZ@ijCKd@0w*vp5 z3vfj{xZj{8fsS~{tEv?BH5refG}aGopZU{-q$Wvfg2U^FsqnflwLb-st_%TQ9qkUt zeNo%kX-XslGw20P6>>dgxq{pgQ;=RC1mqt>=ZaIQS%xdk5JLecBc^D~zKHA($(?Qo z5tdg?kc4^#lom2}06CRlH&a&|7GjZb92Q2jD-2@?jT$Zp_kw5P4|xM|AL{aafEcux zI$JM5n*!)6@q8MljQKKi4BHisD(GY_Xp#cD9qs^ii^c$gi&i4aeAn2~d?y)|9LtHI zyNU=)>rzNaYi%{lkc@=2hvMdd(h-CAfR`K(7F*CIwsdr>KID&fDQFJSMb(hru=8jN z>5ugdmw$$kC$Bpm>=uIISUBQ-CtDVLiUQg<@4}LnvXdiQ?9@34mXmN42+#Zg{1cxhO(#K;;w1}E# zFJ@sCdJ~rnyv0Uia%7ueco9N#p?KqCY&aC?L&f6s39zKQHi zs$PBr{bGAajkL+#XG7dJ76;qYCkdaxZX}$}6(`O$g0+Okh!^U?sL2uK{PJ@E={KC2 z692#$6*?edW3%hQSatgkU#|zl=Z&F7gu7)7={BMgYAaDhE!rJRjLien0xm4rq27TU zA>;!aCinD7nfEnZk-F_O_a%8Dn8KJ4Rlq3xO0*AevOQ-8y99=-$o@fgFN9$Q&=cEd zyKf+)iv1SHrz{Motb4^aZRYu;B*>4Gq42;JvtR7LmjHt-#(2!=sO6UU223Nv>&31Q)iw>cuydr*S$vk$`-vYQx74) z9tBmiU=Rxu46A0#`R@xWF z6CtlU^z_cSc$9gkpCH5Scq60qifcV)_fo2WDAJt^vEEL61#N~7b%($b2(W$yaFV=>xD)j7Z|+NN#Npv16@-k&>ES7U z-@s2q2fmgu!O=#c|GgWpM4r4in~0`bm) zl@=fUs%4(9+C`GSxbor9D7FLURQ%R<2^o&;dSa&NOk;9?rqONaxoj9|L6C%l2_w-g zEuuXiCelccmuDf3S2DjMW$^^Rrj;dL{3Z=yP0+E=Q;2B)7`e^gRQ+ly z;dlFXduZ6nZ7T<_CVJ6J&N$`Pm;-V=*9yaCoDB1;nH!>Nn41^osjV$xWymz7T+YDv-ID_?oIzP7(?>Yao*D<1W@F z&f(T;xQ(o-$^Kk?sPGjn>M8@9=v^ep?XXfHOy#+is+aa_*5~b+*!Xu{sjNp+>FdCQ=3Elns`$DI{SA@2zFt3LBw>Syf43uZ915SUPLXq zSmSw=i25(HeyQ1$y6Y=F04Z3zYy9uPT9tQ07Sbu#s0*yS8$H8zM8ZMaJYzc$On`9m zs@S=6I}2NBV9!bWJE+ET6+=il2k|Zq4G&>{s-8@Un~09x14JlsJrN!JiH<3 zHZz-}jg2X^HyprsaX5paR6Rj z)!L%PR~YCSk#Or(1_64bEa~#M`)z$CfY8)Lkr4KUBhljPHf^kW!P3g1P340ebtamF zA7=?cbkgxqxBE7gsAFN%`5>;E#o_Pt!+(Dmm*~7_Z>KnB1i__fR@ND?=W3&M~=Xa%H;V>**pv9{hrc;%_lU zQhHL3S?*?7=vR01{o+_Nm^;9T3iF=O3Hi=gmn6I*|Dd0$CT^?Wq^_E!!=GtE1)|+y z{HzA_EzR5>2O|>_`0<#C&ivwV4Gg@#8zS)m&yn)aDRgl0OWd0ps=B;V%Vis!Sq3sD zbM=|?!ulexz9GeTD|~%JwZ7!3GsNayKEn zIx?d>0Aoqi!3ATe+|BHcF?@u}O>ihi&1C^5#rAHI|B+k4Sk7R`Xuy)r;u|dabw*YR zb;`VG=5}Fe=O}8fP4s`)5;rIZk$d%OZZpk-Hxc4z(YEiw_T(0iCZgP?40#Ioq=gs| z-_j@9e$~~f?s409wp`BeMiw8kaL2G-fW1C~uQGF^O++t#O221sp+hIlNIlZCcqt%3 z++*P;S@4M5zWy?9YBtVEH=ITIi_??7C0>w@ zw{m1Gtb3SV5$@svE4RE`UBg}%6}QpzY(pyN}fc^Lax+CaK~99 zSGpFhkK5{y^SofgSrju3FX5?OluPWzPg`J#C%(;&gu5ZI98>JO@+avW_wf}v|J8c1 z-b%0aCv#&NYM=)BTUp&p@-OJO*56W?YxD2cTdvXJLmHgL(m;twoJT@JodVn1nv=7L z?)G=1bo@;weoTXRasDpXPlPNrQJHrz@#@cz{ShnhT2_iX^hr`>-8wxelX8x}SzaQP zE@(Fr?L&=3kPou53C|cqc1H00Z0nRU(hEo`An_!QUuDG~Kvq-Xo@H}?GH_drtQrBq z4_d|1y4(G)OP5*+|B0{R2ApLs0WS>lG!S0_wnhk-q}yAREw1t6)67L{zpc599&`~! ze2@k-EaO{p$Y_ylL)F{P9%!KE6v`RMI&U(^W{}-}{7klWjfIJu!ODSd^tmhPNW2@{ zcvu>WxQd(~E<6YG=dug>5FtA)Ozr`b)M?!GCc4~TW?7)=|0nmDiF;4iu}x!mjRq(= za2;UcTTC?y;Xi9=j(7kZ)`%ke`U;;g1tV@N8Sb!^fO+&Eyoyv5>()#{5_Qr8$TFB^ zsZVeEdeqXbCZgN68{R%w^e5@3)m5bXYZ~H8wjQFqX>aa>@QW7l zS8%C)hJy#E^2-=UlckWg{7v7#ho7&-kEL@*GWdISsuvmJzR6-17N({X67#g!p3jfQ zIu=i*6JHHg$gY}IX@DuYqZ-^fh6tHre$je&I{&%oEKH@o&Amp;VD7;&izkwSFm5$L z<1QA@(I*M9HRmET#PMduBklZsX2R9-W(wD?EBxHbt;m4Me%s0iG|W~aTHcJTl9p)4 z(+#-`O@xqZ$*`qy8^)6U;6ObIW^he53oc^_Z_;whtcngV;LJe)8UF4-?stOBQ~#*` zIF*XyhcMrqPJ_pcn=H5`oA?^Ef3#mBzyht_XvrqL9Y;87xEIR!2}g!=+i5gx_3 z&t!qf8ajLlI#u(1ihZ^6RO^m3Mn+2)--fHl5b^w5#>PM4%xaTxR(^i98Nwd7SbojJ zPnoz|4fshGn{9YmMy?^haK1$!4R^$P-PXcXm`q7HSDs9?;vdHlftuY2cKIV$#<0JHB z>5-S8Y~`P}KzQhD2wq5nUA%x*sj$?`L>tb)coIA3>vT^U5j!$*@LLNsqB=+Z78o{s zXC||8lLY^=ykHQ=6nntU?S#luzV3UJEnNv8#CCO*d^l6T53IkJE{# z3x6;%Yg~Er1r4=;`;vBVg|^1=k6P=E<3KUn#A4DNk%RfC!uQ_xR*LAp5#@r1Z{(TQ~D`xX|M+d_39!@7p%tMZOW zXpR`2Z%?~EFKyX&ihBC{vSYbN%HdsJaW35~M_N|U`LoR}b9m1VE+vEWq?7IKTqDyw z#rEpp#AA6xuR#PY!%GWc^iJSiS;6r$uV#V9wFtk%fO)k{)F2(c-Ub@AJ4X-Xv4F(f z-?A^&Fa^HFH|uQoaId8oH{n{0d(*MUOe#MA+v0NTOHBTorDkjG;4ZSy}(ZU0h} zdql%^=MoX_C4dXzPzcYq5n)`&5C*Z5t%qRp@XLc${D+#B>1V)<+b$#n^4`t%+qfq( zEUdwD&6x7(h~I#rdG2G0-SYWToKk+a+@Yb?qQUvDEGbBE6Le%q!M(ZExe%>}Sg#kL z>Ck+KpZ3ZD?h#vJsrYztaA$qZIf%#+P07*%yUF@GCY&t~r11{Tz00t{%3+J;Q!Ocb zA$f)<hvR-^ zh)CgC#U7t~r7lJ&+__m&aau!y%-zq(hlq$ZJp&v9N%&DrQL8XwmZx(st|A=sQ_Uf#rEA8d5@Y2tq$g;-ogccA`y1chJq{G~7fkdykEY?B z#^7pPmX0NcyUUne1X-lcuHtSyaWn*yUNq5wmG}b~+7T2%e*n?e(oBm*vQSr*yGY!x zJ9f9jMS5kfuk6(;#ZJFn9A&uDKc`>uFOSzXDHNKVd&oGnN!o8Dip6o{40)iZwQvm4 zmS3T3`lrA>qb0i)d^AhCtYq)J=0nWwO=C&JYk0J9;(ORb=Nc?0)3f~$F^&6t3T`RE zFBtGgI-*^=F2$L_4zwhOx3%IKmRu@#_lrY=czcff&i9_0c=om+>T$o0ULeNdlNj$Z z!2aG~#IZ7>NOb6v6(|-k3-j@|^wvIV=t{hhkqK^PIvWMHQu!QB&U016wuhk0gZ9KM z?ky+(g2`z&+UK?0r5$=AF>J}wSne^QFQ}`zpBO^0>a7Efx@qT4Ej-T5TZ>&9&ZLoV z&z++&2)W>Xj^)qO5dTc9tf_NxWh^exaD~R?qqzFonKQTv z46eqo3J;GZ0+_Iq(JuTtM}+)?yJK9C#+FK=i#uzqD1lLa-C&>V(u(Cxm*nOwv&^B< zl8NQ8lW>pItXfR-{hjWT)^BuZ(tspB!7ZU}-!Na~j@B>SKpB4JUSlk0+li3X;(|Gz zP{vIbJ?9cd+~utMdi4JK3t0$AO3c0dr1X0)VlK68Fv>NkOj>9`M(1TIw9+Re4zLiy z>WUvM{D+p{<$4-QfAE6R@&+A#%6NFI-PV*lGXYMo~GPV%Inm`UKVg zFcGzUOgq2Ll8hpyQ0H+6_26-DcMcQ$1?hZK2Dh#%_m%R+Mrw-Xb_)^R^)h8%yn+Sl zY9|L;q!lfv%YvFLXsMJsFvaawJ{I3)Zk4u~xeI9giMHu#3UJnCR{R9L;7W6`bLayU zAiIkEZG|T}YN_%Q64%euy&jy2BL&V^Q{630ReuH8VI@UPnMlH^#6&;~kZr1Q^Ok3r z);qJ6A8X2V$`o9hVQFK;Occde(pl}c`EgeVkg85J;atALaDCuGxLcx+ceV8y>@2QaI~7S;Q}H3)v!biJFGV zBsZ*aOLNXmUw>x5(0pKz-ksBMztM3$_oj^$0(g-KhS6#pCq6wh3*V$yF6DO%xa~~9 zmyZaix0{6X^49d8?!t#hV5wJ|=s`4B&U|-mnji1+|7QD`mGb)`qI_lue{DjN; zY=*&`x!$xU*Uu5@ViuH>VP;Z))#0{#s}=+~h}+>a+@b|s*!Fm2cP;;H`b42_Cn9A( z|DA&=w0D$Ol;cmUh!AGoG$e%h<$`C1lz|CJcQ>ldKj3h#naCBJXvj9o#jyGx&9j^m zxgT^yk@Zyv(YLd8w$oaY!>!Wb@dkb}ZO!Eu(CK)EMbz=9Xzo$H=T#Q?STS5;+hOD* zG#Zbd;+Gh?jb+r&QVyv+&0osQ>uR(DdPQ}3G_UUlYZ|~^IR9W(@Eq{Fo|H0D>ZXoj z{(fT0@a)3NwXDkff=xbmdu>Zy2GQF*z__84{Py&Y=`=BytI=9_WCl}}o%|F|#)WQ! zA~QdU!s5}1PQFiv-?LyLgS!9Nm0Z4F+<;1%o6w=%=VGT8KdrT1W^|Y>CpZ-helawj zRSPd$=M-?qEs5IpxwK_p1sUqhzhrFr^ExuxjbGB^*;VDXatM+`pz$@#Pb=V-XHpMB z^n$Lc^NwxV$Z#8=)CZV3zcq8FC|@uy>+AD>sG1 z*Gk+%H^88}IXdEP^r5~*<9f2^|C-DLT-oan}XFAjZ0FcsJ^ zu{sr&L&IIEc1E}PuKQE@=f)5x(7QC$+SOFoEx3ym74h=H2l(Bir9H&4w@|LX&;F$J zH}uQ9M6YyW67!;%Z~HR$J(`S!@tamcq6WE$LAk|JjJda2u0n@@VDOz8)_b$~1qL*| zVM{7u8QOljCb4n5^)-6IeWP%y(Ry9_!oifL!SusD+!7PfKJc7#qlcSS0)f#)Gtr4t zi-<0~s|vrCiBn*lh$83i2JS+^J+HVGM&bp#rRka)e1vC%P}nHm1B!f$9)+CK(*fX0 z9i@b{=LyyWl>*H=DcNz^UW z@!U}bme*0Ffz5pAMC0BIKn;G@x+^V5#x(B3rz%M~Btq^J7v$$3t)*U8b!-?t%NLEc zJ-6&79crmAPper@?WJ%8b5u42Yn@_^eHZ?M!xvdR-&od}aF{ONjqzm~v`ckz-lcc% zsZPrvg5r2yCH}H;56^v(?G^!+ ztu(oJ5CQSZ9amb5b=zL#0^pBYP8Pr_?0<|1ak>KQv!&c71~0Ybjxy@Lr5=eqL{VFH zR}2jrF66fAK!?|(U*vm8d0f6@!h+|?a#()N#nLH2t-8AFP=2lsZ8}m1w{1q(>63+| z4S{^Z@3dqQ&N%{YuOXt=TAX+s7v_qqcW1Akm^h$^yq+9+b>f^#5~p&xzd4Khlp}e+ z*PM=v#&Z`88|2*n35kB|{dD7Zt6%3n9|K9&V;Pp$xrQ4x+)JEpxi~l1S~_t1kd^yM zX!wk$maKkA`a4HFB6a|+7E2BDuhIKQwV2LO3h^Et5y;=+0wncfgMJ0@78Lwx?qD^* z8va2Se-6iQG;%kUaIc&}c_975UBY5|s@w{pTjnF&JDywUYC((J3Hm1%m2jz#?3@^m^D4s)KY@|H9Rllg6Xn=9cv*Pxs9Y7w`Odar-l z5CHszr$$jfx<^ZYOOfix1_RkxWj}fN5(w3XWD@Sb7Tv(AJa^06qw_D&5*01W$MBiy zeYLs|_{F1T&-Pbgp*(#S5wma3xHUw!%{@F(EP-Tfq<@y)D>My8s`z24 z2v`GySrW`AP_jh-kAY5dq8w@^!oLqRD5uKY0;B^q-O%I_rpKTAthN#nT+R z*q3_d6v=<=Z4#djA?Cz+5T?4*L8*Ot*ad@{~WQ!?-c2hOJ3vBQX7v zwcyY1PvNh;1LYq&h4NqVs@&oKuiZ(~(Er!&{|DOrmqgv~(oIilrKHkb;`fs} z)CZ27-H|*P1kEf`kHmn(K$2y;sIElKc*DUNBWGAJ?b$(j&2Wf_96}q^b3oii0a6Pgg~=@%;Tlg?D})u4Tav`G zaKc4vxD(3O;xrsck#o7ZRcTkeUU{OXVB~O=ks~I0buUhxtJW;5r%R}ZvWy(Og1nVU zmJGDN8R&0t4vILsGNwRC431Yx9vu?ZiUB>)GlVAeKGk1x7T7j8GX?GiUZX9PeVT$2 z61U_nQU|X&41`w4M1ny**)p7>0;i`aXa>0Da!~gy4WsX1Y<4upZZ~bugQHwJq5ci{ zvZ$D8G$v2Th@#h$5q~?J%uoQwtw2e!nhXjxjy9h@eFH5;HO%As29=pf-2eq{aK;Jn zL;BD}5#qm*oSXD#!4UTqKY(sBH$h|%$kc}BVm~ogoR7{!wPu(a^-L2=Pt=PIM1a{O z2EdWFf5?WRPxg6`nukgh<|pKaL}O%;AZ`JsNC>@~*8&X4KA6csK%C4@Q^JDhLv%$l zWYe&SuuVoy%sg_IJo5?av&MJDPyTfvNbyM&$W#5Y(`T;K`gJugr%<1xm;4LJ8C~vF zq8pBQf!MmBJBVttXPmBKcAUX1P#X~Q+nW4Wc|J$ui>qx%%GB9QDoV1;|5-%|CO7Gt z0kP&Omh63#ymFdRDjM?)>U+>R)t8a{X&DePx$8EgpjCURz zpZ#*_dGA!*`o)<4r_%|dOcf3gq|+r; z5~QP2y;SPVRDJix)#Az1dI&Yn0;XOuk)6rEk z>KX^O+g7x#nrOaa1H2c8b=}~fhE><9(m{ZYCd~5xe$2`#ivKV*|GC>!^6wu=j;7|n zX~bU+=DyXbrbjpbQNA$z9GXQ!Q{glpu#AAg7bRgv%pXMfIDjiKWv2@pVqHm|AXEm9e9PxLBX7R9L5|BsEa4s2VpFtJE~p#;yMja+9?XK-7bpia5aNn92|vfyQ22gUA6x zH`FOkv^8Z*LaPD~T8a!P$3A{bP(7~-iZ~X>{m{lBgickQ3T*u)z{EIpTu~8U_>;=K&Yoei6q;Z8>2m-YNsBxx5od>k-02R_FD>_z1&kK{HdBiB`89&35MVc?iF^`lFDE9Qc zBG4H~xu@CV@er87aG=M1W64zTh5Ddk1pER2(grw#$Lp#3y@E!b0NFRHja4fM?19SP z{UDY639bMtqM;_k_69m0*MMXCG_ZLC)FiT3wq+F$utC_HbCw0uZ7+HLNp$qz?tYn2 zm};P6Qa9+rwk7ggWsvcLf`XV@u>#50{)bfRwo2HbSSn5m{={#Nm68cpSi8Bbj zyrh-~2+mz8(5ji6sfT9YlzGawWpy6ZJcRlt-fV=6P$%MK5exfKbajD$fgU_Cx!wI6 z#QZ2P^%ARAI}}?;IhvSH_~yBQ?tjI_?RE{-NCZ4LulGS34ysk$SGrU5Nk?w&eKhRs zXqbhuipxvDg}2E?`Xmf%aE-e6Zz7un49qU500G|*d`~}$65jsV)NgLD1RS_>=ujAn zFQ6=9a5MBtK4w+)Fpb)+@TJ?busw6-q6L(N^jG#GCzc3_aLpmYmgL;N3xU&1*=FfC5 zRwn&dKqy}5B_J)=MSEVbm+U=dA1(?fa~5Eo;Q^wA7VdNnTx&~}uTQ$KP}l)yon;5l z2VRm|yd&v$vP#q|udOAz+|=Q9jitNiW&5GHN$HMtE$%)nTPD)wwiI9y`+x`;MLZFb zD~+%xf%`g;Mgk?07C-QmHvdY;{9fI=p`)pL?nLM6DrV19T)|eOB-{(NalwrPKA>`= zRYLr%PCKO7Ne+~C->uML38F>w>in71a?^}5putPidi)s%G5%> zLK!OB^Ah1ZJckVW2j8E)65S({hbPkta0PCLxsM4k&4)^6VWsIz+1(DCvF1*x~q%358+`!9KJOmLQxGZ5OWZT8G%9%e;qle;_DSq6o zL6abHc&hmw>Q|R_sher;XTb>ei0{*#put43)$@caD}EsFE42_N>8r%wOICh}?-}7# z8>J)8e@`c#FV;drV!4I}^G~%}meRKSAYNwwiA}Pb*JgLf?GX1BqTFbPP=>c=t7KT% zVY_F(#zx}p?RbrrkYD|tZ>IM+h;GBe#1H5xmCAwNpttJ@ujNG=jiC}^Ewgd`B7Qsr z=LA6{q!UhOK(Ism6E#pd@YQDdNjCfQLmE6&SNFLt{}K&%j9pw0L^bLF-lffh#R40N8i8ue){XnUa)TOY zeMT2~7D;Clz}ojEiZW-5pB(l%rxVcl*Dreo>TK{EI#geip<6qVszkb>emoe6# zwMQzbDa@($7r@R1Dka}fDAzR*m$_%1CeU?LVt|>?c#8Ly&rPj6z?8p{Cf)*baNRgP z)Q$-<*OH#C2GsmPIfCsPhkl{w(7@Wr&X)R%UB?B zY&M{BsQ-xJ3-sLe>A|tAPZI8Quc5(h#{=ngS}p&9Zh$7`PN$U{sRC&nE2r#!Ab(Ff z3kMr5!Uxm$t`jeVB!#^;^KIt5Kh-8-KR1vAO+U<-PZygU+0YoYeIj4w_;g4&JV9Oa ze%zo36nCVA2C})d9}Va1Uo*s_DMqyrE9CPGplyvyGJ8)HTWLnV5uKUu(i|zmmvQ2R z>T+g?7w?o8K&4c&^yA3KxhF6>3$M&Dh9IgaN4Ps!^kib4w4MW_`6Rl}F2NG9U4=>p zM@za?r$3!vt}P5hHvorFyP3v`y`CKOLG>m=?3!I>-$g_XUk}z;-(#y>g&$;C=BA71 zBxcCx@F#Rhz8*FLd$B`Sl0}2uy9`S3udBSFdoNvlmV3Fn0Hf}^j3eQWnlM5Wq2S}< zpPDyF7ju@MaN$!LpvEg0(~G&pZ-fsmLoN3F;VHW6fjNV9FfWt!W+AHbmAm$S4azU$ zd3yVaoErR;mTHAE8H;QJ9PtIDv0l6PeB(%*fg8m%en-fK=cL!P@YRc{_!x8tv)mX> z)(zP{;CeOgDVbLM5r-bI-3p9o%dnyKn2Emw?^s0?86N4_PtLw%(-rq})VjnC)Rp#g z`JhUdWNv5Gvb4lqLF}UCb@fQPcT>)Ptk*K0$GvgH8AI(0QgSL@WVvA?hzW`J>Je2 zfp-MK;|{172gCnqdM;rG+VjP{JC`Ube*!_$!Ln!vCsiY;ulC$((>M;Z>Fzcdb^f%MQv=Ts1@R%`%*lg5i1G@&6XMl&mY~o)odfgh3*&ZL3|xoy}^u~ zEJqm7Au#_-@TUeyzuz`VgKiNoFEq1<&!k3^O6=8e@nW=!|I#pcM!uUbztgDdhR!|N ztY)_$7*qGROmr^c7e8$|Glh^(OanGKpPH^#jM(nw=9xCVg4P0qBM5km*QfgtQMBzO z5SuMC`G#%js|&fi)O01cg(W(v4SH`_C3`)CGiTWkuf3aa17pW(;E!ccEAbHx|0?6( zziH{K8BBETCh>lM6iP%6$MB^bzJacJ8*RGS)hPWk%Hu~D*Vqx!&h25aArlHQz2K}_ zjxpTLEKiSS-dW~juiN41pP4n$R@_Hrtv#PRV6?1a<)GTH>28A_j`0n**Ss_e@~QzJ zH8Bs)%~dN(po(R{N<;<_F3#j$rTgdDr{G_Vd?jZF{M{J>UitkRGx(9fi3;KGVTYBj z)pExSG`P52cr;OD$;!azW%kRSANjlKz5C@gG@v!S80_SKurQGmz`ln%KnZ0+@qt0cuFWKI7{R_<(k8 z=g<^+bziSPC9HvSF1 z`brJi+vV2Udc+LNRXX4qJ;__oVOh1dg}YO09H}wM&B__r!oL721eoivhpPBr?R^Ve zROR08yXFFG&E;iSz!C-+XN}At2qUuulu6MwI)j6Pf-+o6Z3YG$<)U1Lvck3{wl%HP ztgI-ltn6;3ZEb66scC6x+cFc)ymV8`$}V>P?;2E8+TVB1x4-@S&N;Dn);bIH-f-rT_BNcM!F4MCW*DGxOjz=sK8h+b znj>q*`2^TgR|F~u^A~T8XTOPs4c6zIrsFd)&xOit?dBmUCF54^aZc>-8b~o0U_N8s z?vg{(?s^LUO(pKq9uvxt^6QS<`K`O?TopMCEf+@@KA>}qlm(7UOm17G2KdWHEQ?}) zh~kcNxqY$ywGh(+Zr|7JN)9jPvi}~^zQ|L>6>bQj3&-afo)25Br^RM3e|9c!_A0Bn zR*mp_h(*otcNVmX890)`+gZGY-ZjzO%KgmDoww=*&ga-C z!t^#RHFE3Vq^CRAnS4fb&9Mkap9s_E8tz$b`@JUzC;yHe6*B$Ec&Y(98vv&s>Vqn^ zg8}RI1mEeT;=9r~pr)-?1PM8Lew-8LeSY2BY?l_#(r!8*Ki~Wqo5qMkLGX2}x}b#y z$w(X@4t2Zl>9!Om)rY*kTUgU-t#w6+eK2LXPZR&l^3JUrj~YI$ag0eaoHIYfbSzch#*v_8~sc~~B7bGTSn%@@H{9Ej@NK-Y<&Wz{I@Es%Z zU6_W&xFMEN{&e6DJTn4+6`#ziQ?rC{13+#Y*hk{{5kp;|cU}SdTtQWlJTLzO>(UTn zDF<3FFyqwKrovidMudHO0wj1MK5L2LIH$&%CQIU(lkAQN=MLrTsZ;pDMSZ;ZQ*v(p z8Gk`T&Jb{*uN}$2j6&!!KvTo-hQr35n_{;1QEc!f4Hs?>W7nGbH5%9gdA)DoPYpCY zinsBWL$QE3iI%Opj>AO1~64b~S~_E-xP?a?}kdV~MQb56a6NQAI0X6_#e(924ihlObO&=bV* z=4X-(o9*8@odMg{YW{33`iYtxeorWW^hUZmSNV43_!FZAGs0g8IPd?RIfEUp2 z0P;r#s3~U-INR|Fm1%T;>*rKiux4LlWxeI7Qv@8*K-AlEbuR;;@ji(`Y5I|GdaY}MXhd^d1>U$-A-9HBm!XW zc?%$rU!vk~p%OsP?0&pA3SgVHCcPq=V^z?w*mk{6I())!_Y*d!;6XYKQMACX>m$LM z1I9~c4QlYO_sCQQgOt{%f)F; zr=o$SiuVn%*_NCN*T^8ki5%`2A3x^g11gL5?a$2jJO*54^x z`1P=e!;dFq%R=r!-Jqp0Ihd;Z*=2HsS+DrwZpL!UEFB9ZWI2!`pYom6%}g7&VO z3hH5)agJv3$UE($kn`0W>{JQ$zWM3A@0C%^lR&(0f+keP7qHjIBfld?NjcqQ1!OqO z*yRegk6QmXMgtU`Ms^v`{S?N2qhiO!xaz~_ez{9>m|!`P(4@8n&Zz*HoB4ShF4qdL zrs_}Q?XjD7aMgWHc>=5jGK=~`!^a=;)(UFa82)hkY_44`?#rG;;dO#=9!vn{YUb8G z&F0CP31A<_L)Dg*0uX^Vt-;Uq*QWX3;w!oDBdyQLTvl57OUEhHsvXfa?_^W!CmCb5 z4>d=?QaKLqQ70L=J7d`?5nSG&%!6{b0ya;Tb7kh|93!atA8M&}<{@Hr(op^lWr2k= z@?gzMeqZ8aEfW7mkM}k?zEnqgLG;q+^X?K#)l`M~_0>yFZ!nXOsk775`8ZT(eQA*M zsQ!92USmR4Uw^~hmdesGWF8+0Q*}ja*&#JI%d)sm@)^Mg6pi)xG#ys~@R0g|uW8qW z&O-pO4Ij<*k79P(PsvhW#FKPj3~A!O#O)DnE5h+}fTIzNXGY^f{&wUr9OJ{k(~whKJSJYBhz%2> zvX%1sU;1{m&7RA*CEb<0EPU0c`ZYMbdG%;qtWl=(59tpZ;OH3t8tWTj(gQLxji%vr zED;+(*_k6R8DMspBuOvNzPPO4UAM|6N9nb$p7uD}s?5rg|iizk-kCvD?!IM)%i;4cei zXWC^{js788;tg3=&@hhw8oPD@<2D>-V&V!n#F~pX&!nyEnA|OhS}pN^-bw*;L;%Qn zHN2J$Nw@FTSi1V+77cVaAwtb!BOYhsv_}2c@umXCuv10#12VJ!@Zw^LCuILhzhd>; zeEd#C)M#v_^&j?CZUJN1k9`fVX};Mb_wV`iT~i1)C0LfXBpJPld?93R1wT`NP;H0fU_h7;9;Sg~fhIgZqw^Ma4x@mD_28_oCu=^{ zx6?&0;182l)v#U_tB%tT#|2Nbo9V#zHPdJy4|O#yez$ZHwPt}*dA8|m{!Tnn?R+S3 zreHUf?#NMw9~Y=-#xf^Pb@qH8a|lXF;*##epU{@}A^1VHb!VbQO9)_N_)hMO&haP( z=Ai2$FmSVgfarC~@z3!KA`Mo&riMxIn(>oEQA+y=_VqrTTm|de{pqmTUoZsav31EA z3%EyF+utawc|KnmRbpG%^(*d-;8ts0iz(_C{eDLKk+6u$o);!`DEXx1LOSYZ<%i09 zVF$!~q`ceGuGWtxc8KZxI1NEoWam@VyX^32TuAFbVJvTJ@ILG>H5p^t^IWEwx!33I z;hyIU-gmqzv-NkjCohQ2nCyLr{gz!kZciU93`^Ev^ewk^u;n|Jc{_OvWB)j$a#M)q zE>Oi{$EMq6v{#qyj7&Zo&2yGClK%$2Zu_T>jk1{^8Wc7#2qtIIT)Eo5Q)Vp;P2G!9 z3J!Czv~xE;*~f6$LPTI1x#zRY59WQQ81KdQxXz#Sb|&LtF%&@9?tav1iIUqqQ2aRc z34BYWa5p75X=OQVgVbwr zB`PMpsoY*qsk_oD_scRD^~uPLKg{PVpQPSXt~9+Awn{QTx9}5yHM=<*HM(}na3jx; zDfr-=4VB}k`csB>k7a(J?6lB==L|v;9yJj8v!6(IPR`qeCS<-V3UA7|0F(T_rgO0V ztQs`e5*ylg+2?7N4>c5Phux|@XOd&{TaJ`)^I!Ds8tact_9uv+o^iYD7*b9)+@aC4 z7~R0%bP*GieKQ5CfXOa=8jD`m7VZsSb)EYk_K^zndpKhVaDWmh=TZI#+?7q`v55}L zdZPqf@ebEB80Tj2XQ^85-M-p%rmM4_D^3#&QG;@S;Y<3tC020VbYfv0_3(Ex>mtfw zkU>#f_#3$sVUATw_M8e=jxtn&+(2#F1d2(9QOpsA_ZV=hS~*1D+81E@%7n#>C7RLA zqG_|v>4AOnsxZrNnlkcv%5iw4%8;ooeJRP2?4g+*bJ7%f&sq4T`s_`}coM)7 z62vF!ZGwg=Z(=8d>V=M3li3lDvqmJZrNwZ^H%M%4Rx0rs!<-M%?r<6N+}sO_K&2%> z+U3{O*EE^5nT{zufXplxK%JGRO+YS);&4jvq;SLLL6i4WZ&# zae(D!HU9{8y9Ss={r#Ymh3jJMw455ud6*?dYu@t2sI1c>xo@vKnNaYXhDdU={iEiN zR!(sZLJqTRa-n*|v&_aNL(862?oX5@g9JM;C&c*3ASMae%cb5r()3FAGE^TP$9p$*gB;&PM&9%%2l0Z~Imr{wVi+I2M554rV`n zcoGEs8}+S0ZvYzM>M?|^6Tq+xuop#cox?5DbYcrHx<$53gS*D+i{janQFtL^7^!po z9EaD96_T_aKh0T(U8_UwIz~9B;E(0LN+$TB-ne$H%QM7sA`O2xKtDc`KfruxZx3xh z)m5(T&n4>cbe-_R=-eklI(Q6#uN%0)2-BNt{AMoI$GbYJjQcQR{xDftDJ2ftx~zT_ zB{U#?R;=@0W(ylnL-?`$2z_+6HCfG`h#<_ISH|YQHP8sKJ0K(j6c%?5!J9^F8oZT$ zd|eb+(t)e!bStOxt@z1&8Yo)&w&5ti-JZ6q)$11KB;UisQJ6}@>>rtmuHzPJ znL`4YYE~hi{(N?vq~u~X?(yEZU<7{@P9WLk5iFwTjy6{%6?Tc+E4HpR%wo(>CyO*2 zcAbAV_NKzAgGn)%iWQ>`^K|$*ruN1@e5|OQW|uP$Di?!YLHRi3;|R5n6Qgs-hXnpE z#8dW{!+(1KpBoChK+<7Skbz^;gZBJDYyB-lfKe6Wb|$3#{=1eQQF(J+50&{jFGe@Pbj6--u8sZ->2m zZ8=_~Ip&BAI=?FOF{W==@l*j0Fe^Ui)O|GszYQ%T&V;wdv?BLS03zUozPOWN?;W=~ zEmNyx?~d>yU#x;!m^)4}Ekz!+-LNoHe-on^>t|Pwv#ywcyfvjnW2qW1EIgU5*Y*n> z!fy=1d$ss0jw%+8YrRXI;+ge^QK|7OJGTY4v`<-86u5Qr6a_QOnk9q5aw!c*6YpIZ zEMxAaHc|VjLr@F@#KGL%*j_KEwk92H%GW=sGHo^`&!?e8ZHcR1%Y00w5G!!QIEy`; zx;}oW`P_yY7p9nbxfL>I{hS!eH$xV`1m~zS-=qzXkD)%*?nh@j@A$gSo<>=dA}bHz z+*LYpUFM6lFhb?5$s8?D3iG-*xv6#aGHTi}y=p7iMKjhRD3Cm%ulu3^toxsKV08*7gG$h0O3tJGxkco^gzgR~8N$gQ8G z(Lbwq^bNOGg(V+O$CEAAzbXpfL;3f}E{xD*j?l2~FzSB$-7@q(^|``PkMA$m=jyoI ziVC-ca>qGKamx0i@#i1gLOIg4`*!oMi!&ddE9Vc`KSXE;1q|!a_F@H?;Pf`N-gH;2-6-P3`RHY8`G~x5wMDUmq1MPB9!$ z0M{j;J&13Ov^=V|+%ksFJ*=>irAMhp^)ItAof%6E zZ>cP~T1ROZ^E26^J(yw|rVC8Wg>AV&Usy&pFQcftl=pQo9f*ChkA}cJ9rwraJDBG( z+E&OMKi$a6Vps1Mp2skQ*H5A^ObyMP8tNJlmTn{{jczQR)z~}Ds&G43i*>;W7@3h~C0GAfk0quC4p&;Ne?P&Xef*wge z&ESyUm7sv#=stp@gRKxjp8>L2lAo2-jW0s&g7|6zwj)BqL`oNMRA32_aBoCq@-kG6 z80jTQ3nWew;v|Se$;bpJY=LTm#S5B*{GfeIxDvszA!0CH*TY=XGiYEGf&Fj-sLcW@ z62b{W1PMO{o+H}FMBy2dCEX){xhpk)Q!bXVgsUQwh1Y8Q!n;94<<<6F5XOaQpb{Yb zAeOG1*Fg+saNJI$5xYxSLgieHzPr2$giFFeLB&CHxP;G>3|Iu8=>=DoK{W>{8W@bC zPXZ@~z^5-3yR^cBI)icttVgYsO-ZwtkaGxzP71nb;t0f!NIb)hR!PGcSUM6K4}!nC z7$Hf7m_q7-FolBnLINo5rH}}K{y$SdB&*$->XBMWnV0A<`@1v#FXzIX{y=`23!^F| zpuZ-9oDA|{)-RKDPpqLVPi>Q1c03`$b^q*tfA+usulrxGxb(kRqrOm$jVh5q{&V`A z9zlFfmV#*YHPYxkh6^&>Tn)b_A27mD9QL%iWR0AnH2 zyrW(nf%w7^Eig&c!w7y+zZm4h*FQj6=b^|TM0LH0P=(kJ4E}INZVOvD9t4^xYJW6H zP800tdKaSXpgRjwv~#L4B?$a!Oxe*-=8T+_AP!o;09lIwo0hd|{r4aS=na5D3iJyA z3`BsZDW~S6tiH&)em=6kkN7~OG!teoeNh$}oJzeTMoH3NV7Tp`-RkNymO4;PEiyLU zw((8GZ%)cr?uaG^_VTI2m(E6uH)jn~j*{fnH{Xd+)7sII-SYUk>pNxE`IPyN)fibH zSU}mvxR<0!L{BI)NznU&ALTk}DpB5Up>UuH5knb3 zVPZzFzYSS)srD@oIH;)GHvE0TOGtc`qO1<&`~;-?%BMfPoyxxxeQi8zQosT#sbKd! zVamaeAFceqQ@b4n3^_f%2%7Vw?`#n@(wer1FF0}mvNhtCg z7fK0YP9$k@`?fD-Zi8~i`Y&m35|h2@=5-xs-$j6&sFxO%jn=eR-$lyJ>r)k_hOQkW$uv+hdZ`xqu&4*8`t;LpLoSVlQn#}k z(}5ufwg+(CU^kjB9r z4__02Ndu08zd;$T+@Y|Vc02~Gr$YGHXeHm^0}gLteKu@nB=AJar>7*%7z-ennoAr9 ziQp2bi~;0*3l#&s$@s0gXsp%fz9^d$bR+=d(caBK&MJhDr>vWiP(c};z*q;GVQpj7 zJCS6}_Ew0al780_5!hDcx?wZmgrQcsUL2!nsQXwaLm7q7BE?E-(kDm&LuAqfI1Tku zi-%U$gOe5Dy91jBR@|Vt_(Knu7J$G7*CJKYsyGz2C5^DlPTUiGPW1O3Wpule3=jDJP;P@xTz6l7eJ)GH^CE@NVTo0LM#btIWWZ z3UHD^9EYR=`i}oT#R#s+>itmouxC+bfe+{DI>{}Jx1gX-d5{0OId>rD6<%B71%Z2L zN{xVnuQXOdukrzQ-)?MsU5+vfpQ6xb)`>FVc11QP_ihGGNVPRu4l1fMs72mw=Alht z?pm`GT#Q#RU6cjZx-CXx{VU_itKHah6tZ_boq1nB^5BxL34u&hvo_x z`pcV7;CuoyoR{t$Hke-og2LX_6zKdHZb#@GD7558D<{pE z+XpqSjbh%1^cfqEUC3GimZ@i{g9Q(-HqyYuUt^Nc&p3w$X|p@FP^fC_=ctj7DEkSu zO$Fy*wn8`;-fumRgn&YqLp%|tzO`VoBC$rS5?iT{hP^BX6jlDDH2?u)?M&+0lXKjk zh^xi(7XjzX10Z-?O%=h3JHNYxMmEncG}>nU7&&fKoSe(wo;gK99f&4M*f5HuzMz3# zTYvI7Y9(c~pGVGToZ;dN>=QH~8p~Eu8Zt&yx)cydKL#DJ?m)syYVqy;$#~@t5`b*a zp4my)`u&{==eDFS#w9cju56U=LxncX%`bQ=0pM*jfGYz04OUC16Z)K&Y~Cb)!AJTj zA>?cZy=MIs2?F;(82K-HYV3eX4-B=ht6XFBY-nW26H^3CL7^0xaaPacDgTHio| zPlhI}_}Q^BWY1!{!tZtdxV#{a$Rv|3q_Pk5M*CNsj{(j@%bXL+g@S6@tuoJFY4pAI zI1+A^d3W{iK`%Tu1?VGqsv3~^K89PyEEw8uZ|)3X&WOvY6R6hxI`S7SB`39bE3?k# zNKpVQ7(i0zmVwY1*-qp2+BvaGPbZB|S${&pXB72Q-u>QTuA`I_R6hOnUC%4LzJdiH z?%lwSW5l}_j~yy4B8eHJdqrixor#7~kT|K?cN;Yrpi8ig4fqQeG6at_=NsgCmR2At zS+8+?K~;W(eE*>IRVx3T8luR-F2b3|C`DtUVVX))TU*yk9fTuzCKNo9fVl?GR4Ioq zSbpIwHSDbCJsN#vJxd9HRq$u0osuokJ1 z7O;tfrJOlMR{}`PuGJ9IXzpn=|1%1dWV&edqxBP1ze}be#EhC(*vc?Co0`Rd2fhB6 z92S9o?b@XMs~<>%vELwlk5A&N~7XA86d`wnm!+7uAWZ-nuuApi_R?q;3qCE3Ew zl;?dKodhR>^_!`pg~*2=WEDqlv)#w8gkD!~UcR+d|AR)Hr#~Ntrsn=0-u}G8->~U= zr3gHun%oCuZEGoCmKM_R9gTKdzed6Ve&rvXJuj9PBILd7oDf%bo( zqF^2bY-m%n=U5b2*b%YtP0DkE7E)>928I3v&7ESoObzn^KpcQ&1aJZYj6M>4_bU4nrEgT$kfFFV+4S!$F6zb=go&5X4q!4hm0Lrcc!(K-_IC|mB zl;=E+KDWM$9QE>o9K&fA^!(r`r!2Cl#JZDmJuLG!<=4vb7%fa1HJ%U9kyj|sNm@`~ z;XzsCL`drN2C-OY1Ru-wbIVpAM>Vz`Y^g?J^f?#BU35d0d|I3I19g(jCb#UMoS*H7 z8resqFRh!YTsVpT0jM|)(6Z__?N8<+i2IJ2P54p3{}L-Hp{_MxZNw^N>TgM!UEhIt zj}hCnW^2uEg|PwaK>NNLbg~;*Sn~7YUMXdpmZ0`)uBz+*I|g2yN3&pQfswA1%CZ^5t0OdIIfSHQz{3z zw)yc}wjtlk;H%P8MGM1J!fcA!y5wiWShWT)ku{y~0-X)kIE^4aI5-^4ux{Wo?Rkg# z2p~A^XJ|M%<9y*d$;lNrjdhl~{z@Gx0nd`2n`j|C7GlQH>hSmRT%88ycK_ttbpU#h zHY)hE##!1aXTIVd=|hR^JRNLsfL(#l_Q5lwL32L$FeLt({?0T<&JY!ugaZnHG*l3(X<+H2H78C;|;L zPnmrRquZ6JgAAXX-~JNd>z)?M#U;sM4r=2Ts>VFBA-3PBhqN66|CzuGRGuM zjg$I-&C!6;jSl|{7~kdGDGJSNx(h*^eMe(x0_*=GXJ5Mjd*VLg}7#8tt%tfrQ}-cC8Aa_{WZ8 zZ5aD&yy{J z(>j2+VH&p1tfq6-bvz{Z1J90&KT}>*gHQG47q!1cS9n`FO+Ot_ z@Ia39po)4IlvDz^mNp*;=8^t@2F`h4D0iJi3s+OSv*?;bj#(;hbtKR~xKGTpQ!JzU zQoGuJ!A3Z(oSPa6E-5PfHJvOt#~_5t+LvCV{AjGKfO($>S@6HVa4iB$f@tydT?FYEZPGr0{vfrxqzfO5_X>`!~5fZ%eO-mOa zW8cB}H|>IBF*{L>Fhc9StMha2FNtt8tsifgxdboh8%g-tjjioN6@mJEyX8=|9A$V$ z(ZcTv;hB(`)%-LVR$)C{!A=VUAC0XUt)%A%47*f@V-c|W->|RCpAQ-UE_aF1J<+vS zIp0G9fMR{m_b?^qB;TM0;2P*QH#nz5to{Ma%PFun?shR?oQju4Mb{BZQEe-yRpq`B zZ%2?F%%yK+qC!er#r6Lz4;U%F}VX+HX6#N_aGs1~Cg< z6N``b=kCz!*T2r4#g$KRsd z!2ZO+*lk&$Za>%*fs9>aYWgp>MiLq!X!xYcE0;$=N)u?IRxSJ@ui5U05uz^YE&V>N zqM|P86*L`-DX_(BVfLlo!&Vp~!8swc_V%AFt$pE0zQX{}Rg>L1>OzEZp{M|VVl@NuT))kXAHbZ_XjzAaB?IGEuHJhgKHL-^4pD1q^t#U{^L{=o zTYp9^^hw+|1HY6ENVDh|{nq43H*%&V_hi=|G{22PnVyrO!g^LXC)a^?1~i93@pv|& z)8sQWY;EW9x+|!o;iVCV2enXs+<9Aq7wAzSV+|fCW@hTFpp8_^is7b3nBN5t zu*He`4JI_)F+;ZS5z6ypC^}??u#!W=H|pS23P!+lI&QZS2)EVxmuc&(3Ye^+9ks#+ zle;OtaIIp&sgBROk~Q@pHR7t49RaZ3pu)r9Xr=WC68KQlTO10U+{yNd@gCYbp9V*Y zmulP%ySTKz+s zuhYali!y5073@Yt36y z%6l_!1xYQEg)8l{*{g@4EBg55oY{Kj%njoL6erP639gQaEkzPUkp>emO2Gr&-cEx< z%2e`1_wh8bK}mB)P=OKlsIy4X1R7zK0o6V!9Qc4s-4@%5YcutS=ajt~@-F8z11ydg zodx&EDcB_cmHxiBY*So5sSLjqp{v{46g;q8d4%H5PO$v2L1Ec@Rx;!M*^s1W@_%(h z>eUR6qGV&?*aET8&fs5NRAf(at2rPlf#^6ne3Gx0zrP2qM-`KSLUb|F>LLIO020D( zv<(Wjn=_Ef#v2ufqBNi@qP^4*^C3ht8Vkx> z!8?jkVAs8PK>?4dFFooW)%~&)j=icc`mFEytY-XFq8JnO0v>$$8XxypJm1r6c<}kF zge@;VO%nYFa%Vjs(E=3UvUFF^J3T-GJuhE#tkU!5wcYjNT@@*q9@T-kIaEws%vY8z zVV8URm4zIvv9DO}Z(){gS_@4MYg9ySbtG7g{q@lpq^)Z}OArn~tD5|6z0{>;f1k^N zOZ5~C^RHU&A0d`GoaKU!1R5%&3F)aeAv(l1&^A!W=q;oz#>=kTdsc8TK=9oDaj4}v zgXL5-cy90fn20ta6ipvKLBtmk;=sH7wO4zK=Wl~9r&xPMvkAuZR6O@oxMiu$a-WGL zJ+iINZ-qSyl*b5YtpRX%TUQeu?S+!D)?X}{%XfN<@q7E_oL$izgAu+P%6+J`oE>6$ zGy}3$_1o{#Uo^e(gWxxkwQH~T7SZRP%UQdokq0B&JAivwZ~1wGW!q4aJ9E}xA;XyG z1--8pV!oaLhg##|901(^0VZc$p%LzFR8;gqAi zF2y)fG-jpS(gydIte?F-^53(2U8>p>wB_?DmU~Je|93xoJdTVw(e$YwkLt)blK*S3 z_7>5fa&(Kc`6d{8wF-e%@pv8&!DRbZPTQ z_L(Mlw~`Ez*`{KYd{yd0?f=1TP?tt zW~Uf0$4>UL*G78r>c2LwOF4Cqx;DzIZeA5LpaC+|ug&A7%l|BkS8S@Tjrf||S4I6_ zEa0B|{c9t?`tDV6U&`N=ReS03Kg;%nseu`Q7_Pc#g}65I-k!KB0sqC!_uO4w8~@dJ zuZsIpF8>cQ@4mwK$F)&kd;hA)|BD&Fh|Vm$0$g@&I{x^%e{FiMF#24ZjB9TH%VY0m zpY$4ak9+^xc&`QoUmJUGPyB1saA{n-XwSYj5xqY0uT4b{L)^9TUv=}Um@7dXzt_RH z$BnDUA!!OEaNoLmm~?J3RTJk=^{r&Oo6!o?v$__N3V_@ZNGiE%%e308X(pJOOx@A< zxG7lSf2P?eS|b(~jw`kY4Ykqt#}6cb1FFV|qF4a`kw5sJE;_12mk5~0kyX+)hvSO- zE##u}()}^w=-_YoltPj2yG5}@g!kZI5{M(+0hi&oDCRW4^Nzvpn(C&q98Y?Qkl*YZ z?P>JY<^{$D9KMnXEmiKC?rYUqGki6L)&3k$n|oSaRk1Ct!96l}nopQz^A%?VuMKJl z_-b5@WfR<)aBoyaNq(A7uvM1_9OFuhZPo4o39oEMS@EptB|^qjUvUw{BL)fvdD<#+ zvKq@u+*K2U?~|}wN{ch5mzLyLR*bH0=?>H8tEq-K3%kS0u=z?H86|@XGs<&8SNI zmXEIPS?|rI0BbBA^uLkLY6qaWGOFE9$a68IGw=uH*!y$6wY?d>+5#azSRYU}widtV zkIMN!)$S&GbE;d)bD;0FUR6fD^pPTA>WuQy4(NmBRRzU`)5~i-ZRM^1)^<6*9Qr@$ z%aHc~^wmGA_gU))n&nY?XBMbJI0)e zWhJ=+wE1ec3)&{>M}nsb#;P=LN!cHcMMZ+AwF1V!?sStrRqd-K{k2;7H;++JZuu?H zzuF)zMc#ihwnJUCmFHwNl?z2xo}6kjH~c%tQj$(Gr=@v|vzq>B{vl~3^B9z4voAPK z{JYvg4#eBw8&uv{Q6j?J=c$Hyc={iuv#=(s!IzUYXl#k61;(}Nvf_VxyX`qQ<&+hB zrkB>Zrg?+o(0?#Tm67xYN{jRB|Go2or%9TViVHH`qnyDxsF(58Q=B2qab=_Gs`4fV z=d|9&tg@O}&7~#(Y#u0U@CmMeKBszuYbO}Pq3>q3l6e~D*amNLZfj{tZiX~(*E%xb zdn>dJX`HR7@wAo;Qu~Cpm3vlxAbr-@X<~rP&+g0fz5wG&o7C3H+&|6Tlov>Eb=Rhm zb)+bc?p`0j`fn8ZJ+5G+D2}Wn%R(<%9*!cvN2QasV+&bUx(i^L=qMn+UF7#TvS=JB zEh_!wD_pwxSvG#qz=7mWL7ueo%$06QU!`x-TdphXg4}>~LByrO^pOiyBg+R?4jf2s zISR=IaXR^s)~n=;C`wm*t{DrZev>=3RA_+PBMZp;X_wMd)N6XAq&v%+VIFC&XtmkO zM>@ul_6&1|6v-^<{fPqx4jcl%oi6ElvT(kbYw4*v_#|`4Ff3E!M^4hW)r7*UV zIghLf%3wVQ>)9CzU%OUMo$V+0L zRGtOoZ3r(%6wAd?Ieu k88cF)4KMLHM|Qi$;8r*0E5H-|C;yKgPXL#oVxjGS0iSD!ApigX literal 0 HcmV?d00001 diff --git a/examples/notebooks/hap/output/data.parquet b/examples/notebooks/hap/output/data.parquet new file mode 100644 index 0000000000000000000000000000000000000000..2aa2e88e829def4f213e82023e7a0ceb54893645 GIT binary patch literal 40282 zcmeFZWmH_vwl3VbyL;pA?(PI9xYGo84esvlPH+oOAhni%nIJSLVwt{NX;S;hDt6YRHKaGH@uWwU8)e5sxl18j-dV+wS;i3{_53tK(%jm(j z<783&c*g8mrn+<5dnaZ?z-@4&)G#$Wq-pjei{WjEeFRInwU%R>BnHLfk?hYDrIXXUq{< zhH_D;;&ml@<9lZD&tIul9FEMgF~7SUHZ=E!ey=GGIs+Ki?2O5`nCP3RVd+HWD^MTU zAsx9=wvrvGqOZjpR=Gwj>;WN56L7hG!WmaQtZwuH^n0ksA@-!hOZ&J)9P3|<%zg#U zF?tTGzL=~9b*1m;t2hXToXN_9xPtC*XZN_)C`+Q8qPrk$H0E4sJpC};Afww;1t*Ry z#fh6$RE+EdQx!tG`eP*u_j1KP-jVE`qM-7x^zbEB>3Cx^5-M*mq~Ibc)o!>1xX{w4 zL_5XR2K%_=_|xBa4~m$rIS1t^u{5*bkp!>!Grg^$=2yD)2rECg+xtyr4j};)3pPvL zh05G(hZ5Ry%jce?9Hh(f#QmDz`~ZAN()-1$E-JJFKtqv0Cyxc#Z~%CO8oXxTbl4H6 zWNxv%qwRpe97q88fDSGI0PuM}63#fJ5{~abjco>E0~*j2 zj)oRk_65Sy!j^t)7J8W-T8?KX+Ok2{!%|{O{o)1u5&aybCo~<+f~Psk$Jn_eF|0d> zgJP3T$ql!b1pzG#>|Gegmmc>vt)&ShA11IIG#H4@A-3Vo6+TY3`<3 ztJ zJ(O4G{~Ln8r8^NGX9w!vof|xaFdP6Hq3Z|IA8`oJ7C&sRw~mIPk;@Qd0EQ<>SBS{$ zXa^W``2K`Q=n3T3w4WIXS*X~dxt6%!3sQ@$NeIbg$!jTdsX@IJ(hW4IH&uGqY~5g& z?^5O3?mh2w6DS@U6zLN;lsKHso{^A)To_pjQz=q4QTMwkr0sWCVQ+OG+0cX9IL}Pj z{OOmXRpdW7M1fn&P-sFIy^}IT_RVwudYG~*iQ`%Qn?v@$ReNof<59=IUT*lVd9L4z zNFDEQObQ?VFNyf@TSx!k02e^_H&?;1{^<07)x3@fh;g9*!@Q9G)4b%R{$X7Gh`8mj zG{bl_{aD2R2SEQ;^#_DF4T%4)`j0T~*np)Y3zEQ=DF~DeIg8)pb$(xYEbcLNqeyT~ z7Tbn0`yCMV0fqp5z%L8bsrv?!1|PTV6F6x}8qtW_+Gg2dk03+lMHbVOgfmFz1RsVq zzzQaPM9<<~A%J_$XhTC{Kz9{aj2TDpR;d+^KFx<>(5w}yWwPv3KZK#uoIoxqce5ch z0#*NPM)*Xa^X>zES=-5Mm_d1h9&f5wF+ zWrr1?-ymyXj~D5W5dYhM(F9>&DgP{RC_K_HP7y>l(O?ZEj6lA!IojcJ6$bM?KZXsQ)9ZIkA;gpUepf+wC=OOlrey=k42wF z0xFhW40NR)TQs}}LV}XG_~Po94b{CB3ci%C5kJQjB0Ib&%KIAOTh}#MGiKM!<8Xi7 zfek8iuLgSyh05HsV3Fl5i;8V3=SZN1m7L{hGl5ueNFoUt1{<4I=q$gC)*7s8JJ2On z`*K#zagMIFEUpVxwHE2_qG8+M_Y(XaP*`G$E8xigtG zP9J^K#axh;l^oL)L`1!u1FtL9N=68z1R3 z(+a6Deg_pq->+yw+qenR;rpK8ZvLQij#D}B4ff#kouip5CqDi`lIi=dq>^BrfX*39v|Nn9R65b{tAEpyz_s>Qa{oC(+#6Y zzk&dYTUfY&TwVWRs}&@;p(r%bG&Jo*um1S|6!PENYy%9=5;!`({bzL40ieJDxL~i( zUzoqYygPU)+J3=VdC9zed0u}$Y| z<~`vP@A8iwk2|_=NMDu??5>6OLRXA_zW=gy40o6D?O|X1vVPV1_%*t|yf6T_c%lAy z;QxMT2!bQ^+WMEnc5hw{zw==>mS-J5c^7!;{9j`vKK#4qfAotF561cD^Xg{*`_;dc z_g_P&lqs=XF}h>}+Zz);v{0#eDXpb6lYc}{atq*F>UjO#MC$ZRh@w2|&QgXyF@&a+ z^`G7qS=GwX0R&Wbw>Ja1{6l4vaJZo`H0jJVg$!W#`ro1a4*>sGRg-i$pD_O2z0x3* zQ2;Ex%c(C?ulIt|F(}AMVVAot+_GI%of0yQhornHSi>)09ePH$DeU(iRhluMl}bupfMcoOD( zhQmd@92ImlrkC`lruo;vBy5d4)iQ-8te5A@LBml?M|z2LDGa>5)&2dX(nT)+sX|6w ztA!EL+{~}&WV{8Fn;+ZNRnSp*Sgfc$ArXO3{FX8o1va11Pzu7LlRf9rWM(GeQp%+F zCL;?+3X?VW4G4ozvj~bhy#xdEVPa13ip-a-a-plK0sTsXDjFyBt_$GRt&cGyTUi)T#{`|QM?!4nJ zr29<`m-#IF$6g<#o6pz@TRuT@Qq%F}olC_Vl#y9T42Saxu5npNdNe#YSQk(1bUzu6 zC4adRJ;r6>@u+_rgEB7jnh{?p%?=^u#OsY5U*w;g`EKb=%t^*|citi`&1$1ggbAws zSd8ZrK+iipF^eWV%(745bH^Nd%1^{l! zOCZtrG;D;6FlEP;cCeS83OUM`8ScdESahdO$@fG*yd8Puuwk}Oa$SvzTm;Eo>86ZU zJ|-b}*(7DBTldgoVvTO5AViOM~v@qu`#E<$cNRhQ><$1Bv z!~p!|d4SkEClN)Pd7-*Lvr);9srbf86VbS|QRiw7I>3yw)wNLKIEEfQfHPniX`|`E z+<{~(x0rk^J%4CP4I&vUggsEqwA6WwLpzKW&I zX9)vbOz6c!Je&x|?veqbu+`^O;cPxUG^&hOh3+5j^Oh=U#1rXhMh>Km6o=76gvDO? zv#+p(TOPECJASY7QHAf^P8M@#HTc%py7bb?Xj`|`3P;SQQl11b#!sFGj*_F z0;;G3!S~s?@#)#wz|~Z(!M;%31LWWiVghQIIsj!{OdZUvUCkYtfRf@selB)4-oHHY z|LTHwT>7736kWs0+R4e<;oZMS(s@SQMQoZy37R{$*GT%mKd!%Z;NVB2C+L57;P4P- z2mnpgllxtC2rwC;A#1p=KZW|kd5N;EBl!csVo>OQ| zGU!UP`T&IMlVQD30mIvFB0Gk2xf*8n9f9Xb?FR}`?j30-ADhpXSPrd*wztKox_jAb zn9Eo?{Mm;nvUWPEF@?Ra%qJ(@pMM>7BNOj`A|YsG3HK8{@KJS4GL;K+A@>e3vQI5D zEj!g}-IgCQ!COhFIV64^gb?_7i^HLrd{>6(A4P(+#k!T`a2(HQ`&@V|k^N}9X()^&_`f9K!~aF-Ykd9hSHWQZ_n))c zRsLlqpyFcj`Wwc-tOURhTD@>I>2GKrWWg(e|BmT@K>D{6sbU-wr`hh`XHu$K*b1PE zWuHY3;G9wlWo!+?&;9I_PZXxHjc5!65y2p^r zTINKq*ly1!pfn1Qqtqby;@p^B%{G_Z1^ESX1Jc7s;{#iU9#FUmhj@%-$*S!}b}(HNUQWBh2)_Do11f*qf07S(#>(bF12*QmlClYidQ zk1BycEcQSyY`oEJeBwHQJrMa#bzB+T)dlhO_Gsk-Uz~y&!`)b!YOBzB^2f3dA(fBN zu*ih@D`GdRSKB_;vhOnSpD%VF#0D^AmNb#)(#j~V>D{u{ex=%ym7SG+EAu*JcgV8+ zoRoCQC6sw=uo++X_!iDVBP}8R5trP@)+p$3S`maRtex(Y=M~KyUPpD+l(IbTTU?a9 zpLsT+ck_jHd41~x#UcBrhYZ5XvctB_%Z&HurTLr`8txz@8W!(ZAI>XM*R$)|L1keq zpIF=Gfu!kO{e!k;D;6S&!N|qy?@pl^wpxb@U;B-PX-^)RRr7uk!(sX{cvVN&GsHVM zv*a0fUeTtv9A*5BNW`kAD({hgT$NoecL*30q;G#x|D4jqbxWi`z(YeZQsGUlCa>62 zp6=51fY7YDG3|*v5Av@Rbw1&m#v=G#)|1?h)XjW5S7vs|{25i2%>=j|g%V1NsvWU7 zK+D2TqF+Zv(|QnlT-o^O!Y~J_qxwf=+-(YE}X#Nzbw9@pOJ@aU(`hx`v^2VhoB zU?GM*qQ_cNXC4+mMc;fVwZBdq9EQe1@{}>e!6br*twID(Ao$*-X?^89?zU?C1l_t? zan%s^{gP(Nvcv49#S!n=E>u~TZf;pGitD~s9N6hh)a`Jxh*0De<&EOx$n+Mgz7Iva z`w$jJ#A4x089ysOhb200dXF2C{bXmV?WylbZfDy+FB|q@eP(|D;Ps68nVEa@d+UC7 zdQ4C>)Q=U3tr2pF5XwsKv+B6F6WDFx%%a0Zofg(McFK?u#wU!5aVlYz9e!GkcXG`3 zs$`qr;!)Kw@v5g**<-tlQb{Zb{bC5t6n3EDshj}h=a?w3c0t8593+J0PmIZTq~Aba zGkI1Wae|_c>!Cnz*J?XeVo=4WusisXIw&zW!wF z`P-IsU6Fyl!XZW`j%-Q{LPG zY-l0sf+|?f3Run=d*`G#tM2|_4`24fMVe@kDm~eT%ZT;}pL^@@>_CzhOi(07jn65{ z7kTj^(QuZLiD`|-M!Jkj#+whAT_LsJtp{-^?#LEB1O3mdJmSN8oZ%RE^K0rp;Ql{t zjV1*7DLY${hllxalmR3OcKsMGef&t`nNAnKmZ89SkoMcYXqwe#i4+|UH42NS0V#Lm zn(G;`f0`?%+7!EtNtxN^$CH>RU_HC@C9e>IMvS#m6_@>^nQI_a1l{{dKZ@iUT5X3} zCz`^s)pCpq(;Zbu;@@e`UA7L}`k{lN_to(Bj9+t4cV$hkNZJQhkYSvw!}KJ-*&oCXH$} z_i|#D8%-27%5_g?foLLpkagNdq@-C&KeQ+mG>{;wcVLko03LR$_Imu~zfVrv{A?RFY}|g*5LH27%XM#$q7}x(n*-1$MffLV`O-A>M8& z#~D{J(*sdA{=(1cUX0GBAr{XVsbDxYIFvcCJ!OyYP$a%+)}UoO$SR~_;XxT<`*3Qe z?^AG6R()8B@fNvNFGj?(h38Noy-avpf3kx7YfJxqa;+a&{k~i90@7+_^k<9wufpT( z5E>68y|pp1?B^i!M}21pOr4d$C=k$Pb>t0XxSgO~>7!6A+Qz3xX2Y0CLTtb`>MR={ zG*ttVay(_99*sgarq4B@c0NmHwVk7VtB(hQTbS?I|Ac-2aHr^m#;oO7VZL7c4kopI zJUH~D2`RpcuJCL0lrZs&6hFw&bDaxyq*L047byLZqp@Cxk}(Iuhcb1)7gNpbPmqL! zv7tZa;(1zYiLPO=!G>P_bZTO=o6PxLAq<^;N5OP}?@L9eGZ|yl>$zD%QEcve;@(k@ zw}@aKv8;OD$Q#gkS4XotS1ZOgZfDc3x~v_ANm;R-&YvB{Z!@-FYhx}nt_^>rk3v>_ zXJH4Xr+I6Xw8YpA$2(U4AoiWqaOuZp49lp7Wx8wWYz4wMPfM+{dEXvQ}F{oib# zKP{H#-xGos%h;b5f24g%Do5Vz;U>y?oo;=>l$At=`+9hW2Xt|b_FddPi|>9(6*KFl zc7_qv&$=>X1V z;^u$Uhyarw8`4pYwnm)Yt}Ewt+@>^K1`>9GeHfv!7{t0c*h<>N!sf6!y$QQ?eHca_ zEFE_$)alg&7$T*IF~!Z&J~1IcFl({+Tx@lsPkiy%#MVQVoXP0@`H-C2fz#@ZZzN$A z;e?;~P_c;nwAeyTDEM$rW4o7p{20KsiJ4ZsqOgST6=YE)BJnSmlmtJQ$|>@_?aiJ? zWr?7Sn(W(mvYqNYEfXoGVra#eSRPRisMlZUQ2A!!%r4B~5F4o(4$|*<)AXvn<0g@OgvCBA_9_F-)s*?v5dgCm0j~8gKy&%n$vy6A zxE^-d8J1zx%;d&ST${;9>ej&;kw)$)6let{F!WQf=~;M_0Km44$e^#K)y!AbQ)T6> zS!)A@UdtT`Mr8)5~NIXfUODp);GOty1pJ3`X4hF43P?KRT1{1Rk* zd`%bMg5yN(WjwtVC$aQP{Kl{KisIgWHwm_?|0+bqLFTXw4qL>+eDkU~9*CDsxWWn> z{ht?Y$Ei6o_w(~IAigB`CP?-J#;4hb)_*5Y@Y;2hkt3dP#Uu7jkZ_uS7qCDSHSiGN zx8OgR6-+F;g5cU_Ds4YuhAUSLAP7$c$I?gCNKwfrbHS#dtBT2qu~0ADpFx4QS$h+# z-GYc=#fQ<&BEzL#e^Ftw<)l^^mM22t95S!SX1UG6{f=YHElL45oMIA4YaS}YjT5!j zoy}FpTSy)%;{^;Z9V(Z4Ow_9PCeO*l3a2?Ib*Xah^h+d{eA&|znp9w{V7xKhDU?%AkI0wfautqyWNh zakEh>oMlZ_-GaRfU)8TQ*{BuUh*|H6o$sx>dpa_z&FSI8_8N)rmOb?Pxx~EKz7*7n ze~D%O3{}T$_03>UHOJE=QV0GG=h5IpNj6`RPjf5IZYTeO7pL2t5p}oaBvU6tIqfpu zSTk8)Y3q~f^UvV3E>ums1S8QsL)>-LDijew;J(@Bw#4->G;_>d)5?)36w%5c-Ncs&T^GkQ9P>W(-3I-V&a zuKIbj;P4r9tRmbs9SkWJ>B*>!qu1kkc1m8K@Uos3p~oU;P!&ZF*FnN}t@(=Zbioq1 zENGK^NHy$5`ZH@<@?h2?I{gK8jh|$f&$U(bIxdS#--=lzdWe2Fs|s3Rh$R4{Cs8p7 zkmuXN?JvUXv~XWMSuvchD`j1;h1jg--JuJO-b%~VsR85IwCs}M3*QLG--t4iD<3sf zp{zZ3)3$VP)c)G%o?F6XqpO!h3?S{I+#2ydBn!SaPeZ3DFPpH;E3)SzVnnkfeu9L< z7X0Fj#vh4AEV<1?EMI6`gW@?Zk!(~&q%yXST>6BSpTdE=GN_3c7t)Q6p93ZDiDi@j2 z3+x^kT)+WY{BDi*j-JROf|zY<;w}V?KxQHflK2JT9I0XP0Erjzv0-HuzlN9zhwb=3|9!Okrz96Zf#W-x4JFQdU*trpV8u@Mf~!IxJJ2 z?v#rEGL=CPgZ&{LXi60v=aQXe(z)trOgs6pvHMZv2GKYl-qoX<3kKr6yHTI`ShoVC zClOpX%j(wlE2Op-ogs}>AcR_k?TVk z0{f*rN&RCoW{eCrC!54xM#B@)SiS>Y(91_))m|FRzNwQ!`~E7aoXNr$?J9WYgjQCO z8)8>~T-%tI!ek zWYd7d=+SLU%7mqRkI*%g<1XBLE7yU`^JbqKX>;mDlG?( zQ&>Z8#r0U0CFl!MxG-2U9G?5@=xb+bmq}oCM$MpnL4CZ#<8;px;W$HSn3-ar4{seSVt5wC*yMlg>Wr?Zo*%~Lq0n@sJmrf#R959iKzjA*bC_`N<>jc8;{ zwAv~U^YxR|CZJOv&7(5=pnPt@K#W%XIksQu95F=Y2k$wG zn?mY|X1lAz?xi>S(ID)P_}LvlA@iyNe#?XnXS^w{RER>Fd=y!Y-{88?*&H#|JGkt? z(1j@He&Nh-f)YsM>hqc7&(q}W$hj^cLem(}(wI~haLnoWG(?;O2XK&Y3pXi{-XbHi z!?^lk7joI&y*miA*$)xrkzcw|R@Kr-bp;3%J}=8ERiZZ05)J<-yy~67t>5%?DyH`v z5rE$GW=S{-dYTjsb}FNc@80&Ij^(`uUA@$|$2X$xC`w*qAp zZ!P4WN<>f)nlTN-B}tF!YbZjm%Ib0}D2U8u@y!g<)}noKa~;{V7!F}#L@XaQcZ`Yz z(bmGD=DXMdYf!-v%^k|*oDR$*{YRj}I9mkoT$q}bl znV&)BeG+6>TBur~J7g*D9kHfY!c-`?wsR7e>RMnrw;RrmsYphKRP{c}%qQLp-s|49 z#FE+rE?GTQoU6{3gSHi{%LXRd7PA6sRkKg@Y?Ef)T1EqrDtQDmjx9Nc^-Yu-=)Yn{ zdyR||>N9A)DM_^(tFUrd+k&eo!xAGWNHJfkz>7Pzm+=HHL~WWa+lB~HOc!(ax!Sr0 zRev&tf^vVwtLrTOb`0Ja9vLsAJzts$cZ&Rbs*v*Na$K%HcxQf#_UA&Ovxhm{iTDUI zyfwqVc76AVuilh{Pm%JLdF3h8cWf|$1a$Fp)i>-x7344@&2zqtLd_6aZZU312(lPs zi#czC_IosF15j{qAq4_#Ej)EKEFF)fekXqyr{RE6x0ho|+z6&Bvv)%cZ)>3`TH>Gm zHL{cYb?o~WWI>nDvv4abejxJ!YX4*>0hf_@gFEb7Z_XW8?U1zjIosfl=P$TvDFi{{ z)0(7FXu-Rm?!_c8@mb@SU`LnZ zq^G_O(5*(X4u|ryxc?SvlU(eN@%Tho#SQ%_abGtebC+w#g0{=D`MTlUswR;kDK@97 zrKbT8LbPvt6vzCW+5%vO8t(XeLM($`sQ7v1E6d#Hp`8AN+$)?-j(oO}@yn{YvpPwn z29;^SwKbglDvo$g$9x3=?_#Sn+w_R&Mr<#S3pwxJx>k}Hmc86J zMW5oqe`jxL!n?6S!)qV5t22hQ$^Lo8Fp+0fkq*to!aG^E0{ff zW!5M~oil>L&^?WyPI_O~{YVr7WkCcb=8?9g#Cf;*gl9L22YYXTL#JGy6|Y;h>{g_w zc4gUZ=K<$L5SHUU$fq~{hZe-wv`fb+hP|`e;{80L&Mw~aj!O${uhYX7*$`V^=H%~n z2kpf?2K@qPSZ&F&{;3OSt91yjtwjN)khbv%bvq|!Mmx!x3ULg#PP6)*8}yT{u(I^M zJ457~vUML+7Z1LFyFb0N?|#Q1x&7-o@u5P6bD6!?CBM0|@j9>eolsFNsK>qw4TEzq zmav18Tc#Ru@ApKr7+$pAem>i=>31?_42T+LyC6=FIH;#uj-Q>rJzK4U9O0e}ezNi& zDso4C8-YSsJ*IKq_rFlTXR~9z>xVi#{7vU)97;zUFOT4*@xhoF;v#YJcT{Ls1t8^T zE$0G12%X_TT(0zhP2qGc;|dlkR4I}Zx?<)CP0t3sXU^(jfBe%XYbQlMCqXj#30j{< zI-&u()Z1f6b0&mR=YAKvyof!^%Mn_Kp1@_so2k@!w+LZ3n&Q&c1aV{(ztifGtgz}$ zn6>#{TMA)8z>V`l^q08%um+2ht|Ro{5z?zD#nY}ja$(T2gl{4JU~8dS`OD~0uNBGQ z{qusl=0#ni)Wb_wB{54-2fxEFpy$BnJ=-5ivpu3oUiJ$lcR6y-DR@9xDDPt$ z6~k|X$@MuTZws=MY+=y%RrP(Ky3Qw~ z!F?@$;&WM|;4wvcNl{&4E+X@^R3hln{*3U&pzfOmrH?Cq+Lu#s^VZe(orjI@0+Qop zf`5pqvhq^)UZF%+DznL{dP){zUY`ZlC1UfIYVZ}~UX$WQG0(FUONX+X!tAuE^f01B zA%|N*X8WBO#j^$@1tf8`zi$ng1KdqG!}7I;!XH6e*&r2PK}6BL}i)~>GZAfTtUn-vh`We(2Avi1N0J*>@KOx>&<9bADH?k=yb+^j%A*Vn$xz0K{w zo8BPs=J!8Zx!RlB*#YewO)XqmfEwUx_8^DXrXUM&BU86OE&q~f@8|&nOEq_Kbanl6 z|ZsFo?543Ou*MF7f;^+p>b!Pbk@(+vu%AUn5=XQSv3hw7sbwD#yaH4}b zSg~ec39sG#MUfMj1#4F~@R@1&-Li{tM0agCbWK;(^ffFx47XL^y8~g%^yW>f- z(+xort@fJH@c*Xx|C{3fx2E{trX<{;;S9qo;ROE6J54_!r_ceoW`-1OwWTOE+!zhm zsMT!eJg6SOMD^N)S`;m%poy{R>`wMlAT|ydn9FWxEb@f-D)ueA~-0ZL*DPIU|_{(ZmZU^LlvuaAL2ZFe;`d-SN=~)_v!%|+N zW#B9$wn_H_u+X8yJ0ggkw0^^JRh63~wxIF<=A0eZY?MZREa*)eEmob1s8o!_{Nb$% zK_9*BWg2aIQY$E?R3Ko7#tgYW6i#1!NcTeElVK^_h%1^V5@uA|%+MZUS%HnCFjV->8;Pm^7NEi+_=1OT z8DU>;t7BKH_@#Np%y{r-o92b61ux4MN%I?Hn>hv(``^sM$mDV#kL)09fyG6+d$A#BR zG83n$p|NSaQ`>13p`q09;gac)7WmMx z<*~GM@zSk{mc7aFqbYT>X|t;s*E(5=CpZn)c=NvtZvGMl_X2*|CGy+9cvD;rkc)@4 zIS2^$r{F|ua8w3|bdUdd|K$b)cLk9qa~*=FVFBXRtNxGh{?;T?p>ZZ5lyGSNnOcnq z*oOpYLLq`-uyX^!!OGnYe!J1F+Q_ zDA~*I1kcC9$F_y;giH)cjPLaN2agJi3NeDSZt^zW~8ak2f?2RI7a4fqW%0&^3}vq3C)aa3Ro}R0T@oG(1j7I(C}Vt?yHXAEGH`UAGPT@ zTphf3FPo98zg`~g_JY`H#RZj*g7Pwh5amfP^f6N)OCb9oBYb>WC3&qc0Q@6nTCTib5x@)b^!p$_)@d_ zQr}-j#q@KATsQ+I&x%pc$*rzknYB~YcZ%)H$(8l_DvdHW^$h;X@@p-dVU#v!edjIg zbyWGy2e!uYLLSFaGABY@nd|u}=^Xid8MD};;XSKW$jpdezq=_$ba^nej+H%7J5KQ` zkq}75g14Q&;I*Wao!Gt@=yyI-lZ%Ngc5=<0_b0i=l=|j;r#HR+if^NGaY6p&X%;JQ zq-}y8*OZHz6A|zC@sB;v}SmL+fON!2GTlqfd z+1^aY{aNc)-vHR|M+u_^&-OLVNlIRGKAG$!B|V$7*uOYKM4RBV@bI7f5K>d@OSD_< z02Umj;R=6+DQeO1xAjY;>yq7yLAh(7_)R&Vm>(`Y!$gjFhtH*khOQw6(Yb9GJJl_R z5kFV8BmLu@olhekqEeME)hKtpV}5m5?se~j6Qd%2zQcTs>KT=|;Joq1MwASVwc_bBP+w>`aba!53ZBKAI_|R-~9D zzfQSBg1Abs{#n$MyrR1GZAW28cZ^D;&sWB?BYCANl9gJqEr6xyHg7Gd$xTfZr3F4g zZn*tU#q>p0RH7xo8sDE70529<*%EVx%@K$E`^>DzaFW?qg-Z;20EPzniI1V-PVd@L z%Xw_3k>*-vz9ctR8z=i_UUsF-_L}2*@WkhMIX>ys>(&p$ZXAufsY-DfoH-?Ev?SjS zR_l^@Pq)Jx+epY5i+g!rTy&sO}UJ?l=SEr z0<29rab=+4i8gH|^G+=tk>vE{kw<2c9({i=sx75UVvj z;zrGo#XFTIC0E+7>EC2j2>M^2p%p2YS~zwkXGS3zA2ZY~<$oGFn}2{N+O4j|i+AhX zYwHz`LfYL}?~lP!F#B=Rb}+LcFGRF!bJi%;vg|p%(XV3gASdp2c7J(|zDw)Ay(ZXm z_b7mN3pV2kEFoXHF-)7hX4+L!qv#l^O*zT7R2KGDvFrS;ZAlm$VzrFq6w8J)JC$Sn zcRs-yjZ>g1e%{}uw=Qs@gPTLwfJ>T<8O_c9X_iTG(%_c z>TEe7Db^Trw0lwZ!o0OQO~+mw$(A6mgY>y zv$f58@6E4SoETQM|Kx)3Vk^lNT}J5xKd^iPer*2y)6E*G#6$woJ917AzaJwkUqjqz zE%%7E?+vCsn#;}_CQkXUWK_dn;Cu!a|9NZ@ZVn|qBiigv;pNt%Z(m}odRFchV)cA_ zeoC3C*?Jd$trb2emQwNkP-Zq&?yx*vEj!bmqDrk5MTktiZKO|V-NqK0kvk!1RWx;0 zB|hv>?}mdYZ1X06MSt5VNI4dN%OSN=W-dJzf5w5&;%|3A z7lovyjc*}>jFJKwsk3kaoui%_^yP!}DO5c1NK!w>bMX?UX~rpLdDu1|Dn7CNF*dE^ z)8%_x7S6T_B0ogiQriJng`%Q9i*5Ug1gIT=?A_`eW(?)F!cJeAmBf}@c9YZ7mR!9d zP>qD2&kVy_KZu&-)ofshx0_8ql>Io8pS*C5K#iwUowI13nW(WIZ3ZVGIW<&8I7_L< zT94-H33J$E$fQtj!ILoxET%8f6{gedsc zUnV&L*V?XA=9_92D$IRK#SRj1GYscF z)enmA-$-^!WqUKUaxoPbT$yWUpsOn^KdxX|rBby&I@1OBi6EW5RKrQr;CIfFj9fU1K6@Kd*x7;Fql&-Wn!0_;%VBe+=ZTsLbzg}lL|#cnLga=HJ`{$ z16{Qc01@nJbFAtQPA3AO$d?vz#hVeEBN9+ha8iIj$yD+J)f0gZDgn8=Yb6hGYybl` zK|V^|fUjdw-%JY7>H@`YXhL3$kpXKuR(iewjO78+%&+YoAJ7WDU-03HBxM_INL(RM zZoF3rU5RT8vp-V*=uqgnis8fi?h>13|FGdbc2oj48Jp>PH%b9EN+IA>~A&#Z2L!oI|*}A z(lLsOa5nSVSR2zQ^qfc=Xw4s@xptcsW+i74^DW?F@prtNA=@dZoeOipfOne2A%D0_-NfinK1E|Jcq=3;ciYMrn}8?8xftN!OD+x{jQhbV!qG*U-k zW)V998_xb^K}=fL$WOc9y$}b5Xc}^ znU1j63YBA&tNH(S>BXYL`miwytXe>~P0NLr+Zo7kK%xJ}K6cOmV6Z#jk;GNVg(&S= z=+2dw&$UjB*TaAVu})^K6RKT|R5@{fCC%n)*?@WKEw`xOr67~ z{*{!K;gx6bV`=3+R>X_Pm5=3F<0TaUQK{_pl1j4y-9>@3kIS z;UzQjl-GDA`NE7tMQVUZe!Fy-V>^siF^S*ZKS0J9AcwCfKY&2+ww|P%I07Wu1XB>GbSy7G0gow5NI@FLe;YMJ9e2VW}V@sY; zIG03kV>>cvwcjkHLY1Y# zN~U@yG9thRPJ|CiAF+ci)t+oW>Z*)bi8l^i>3e&9kf2VZFC9b)?2)bqPQ1l6XlSQ( zd%7}z`*SoUDDMXCBk)QcKZXrElzqm+qjReRr_i0{WI^V6Th+`U8irJ=_dVI}iTN$|iiDcx@}~(Ga^soA$mD(aE@FZ(U+bNQ$`T2G#Flw+=IHG8P6#J1C`)9% z7E#_>%)V}g8iu{b2u(tNBVv}{aHEw9MW z#yrWRCRLBkBif%W(OQmeY4=;Zap^x8Q=-;exy~0M=sS&VX7BQ}k<28x2XYCBq@Os1 zT-O)s2ymxX-BaN@DC$bvGATd`V@P7c5Ch*^XwG`GGY0og&y)Y2|`F7 z&l$jKW8jQIM_AC}pDdX#1#^G+TDKQL+d zT;2N!Y)!Ja(p`PWsY$Ah9?}5eW-4alClgTIWI~E80N%i6R&kHE8F6BNgUJ(R=evWj zs)-1@?{YM4gZegMq1Up--6WE&?pTQ9Ss0TZ{I8uz2MaIKwxdhY%cAHU zgq?GPV~LJ~7(DQ`2ebkni^L7?`k{UUNe7%b-XS<$!v9iI?|0R-pAHp-n=;ys4C8}_ zC|l3Y^fnZ3m+jgtz#<%o-O1TYb_QP3xy#z`v;kIvG-aS-9|@42@^jGm`+KVN zv3EjR3YuAaxZhTu&?VxoXQ1SW#n*BzE+T@$A5;W>sVfK^!-w92`@1Gg$A(y>y#$!NGFmC3R<4j>W<4c7x32cJvPm)pXPNbcpAW(k(?bn zUF|aB=gVK_U{BN4W#LuvQ8w1!e#QUoDW4D#KD>@aTS8~&?-vOgN3)81hmyTk7#TyDL%g_BA|Hu||`rI=fr;zyw6~>ItNS#wXG?+ocDs zP*UU^3^!_5h4b*c^dpKrPUI-w>&c7vR_z+i_|$&li6VuSIW9$rT(U2>A&TOy^erab zK;Yii4r-E|vE5;UL4C6&JssZPazgu{2b=NQ3vG?Pk>=icKRWFVc~&^uD{Z`cQh@Gc zv1V>py^T@Q)Ioejfz!IpYK(-^F2>qaxquu-Zd-BY3}_VHPP~H!{86L%V3^A zyhCR9qleafd?)fxpA`0em3h|xy?#+l?G|<^jqGF2rj+j&Bnr>$u0nSujbg`S_h7ru zC%kXqRsVQivPoZ8%uY9_QtH#O&TY@B?rgVCuXdf*KDJTu&zI1dAYK~>M6i6DG_%M8d${NqCS5ar|6izs-;2xabnto8}=DJ zZtspe)vegR>`|_g*~4mP@A{3B;cLsw1x2&guQnUM7W$~&hgs6>p2(KAhkE|2U4!?n zdREXjFzDoK(>BfJS7`B&mEZ5)217S_+oM{t%hFW?2`ey1OgDM$KYiN!E#dp!?MR(r zdrBAwpEGmFvm(V-*4t<`i6br3878(YLelYL`2+2js(tPT$}-&y&eMz+uBHk3Nx@%` zIwF*}zO!gu5o*WrSbgO9q)&vTfRbNw36JL-2QPz>8++>WAKez(2eaoA)6mVM~us|NmvNwMz74tgKm)b8;)fAs%0 zuwlK)0~_MoV;2YIwjK@MxuK4?)Kjd0n{+QD)2396Ti{Xz%p##r9&~re!x< zw+;KOZ7q;?AvSlJ)%&X@cluY|9r92;ZvDu>W<7>nvi<3(;G+stXDdJLn$PiBg1niU z6NWF6cf}PQ=?Us3^6>3*vO3CT`Pj7Lg!FyKE@Kx<(c-U?<1TM3)(=MBGkJil5;|3E z)Wx$M$LB@FipGCT*xvSzxyHKWY`v8L*Oxl;0};J?wOvQtH&?G-tg7l`>qz(&2kHIko7`A&qWFU zl^l5r#|U!7UR@6>RR@cye6z}tD3t!;{N=`N^^Lg(a!#sM*CQ`+wQC;nVS;PP=-<+) zJ~;WV{!0c=;diYj--$o)>MtFX4+drV76)ES=6@P?rZ-+wcq0dU0a}#QxJ&e;p-N{) z=MTA2_a?=|Jk_emiLlVt+f~|K^w$!J({26)duzW>ciWl%n0&d@MdzYTmDm_Z{Vw$# z^_H;-Ol=s`W6!sL3ATWlWIM;Rd#jZ5jHGpl8=Di)wSO!rO_ZQ3NG$Vm-BFJ$kJ*;k zd|Bcy(bs6}E-R+hY;NL7`RbB|vc;M=zC966xxqGG?eMT=)8XLU%i0pPy)9nCn(gjp zQw6bJX6zYujg{l^HjzfI-`%u3KHucc@{aV9`Y=7gO8*qOD~m1a~Uj?topCpJnxxYZEy85uF~H>_?Wq} zlgi}hP_RNr{p}$86D;o_cg&B36OdPF`BNfIeLAmzJt@8HUnLLk6N)kj{sSa{B z3Mgcm4n{o8iRWfcbIn)H$o@L<;Vpqycx`GdFiXGm*iCH;|$GEaT znnP5{%zv!=JD0&`xyBf?*z=jFH^QNZ@2A@sTDU}8OT0e&xFt?LF7f$mz2jS5%y{jk z3%Jlo6^4b^~ABOHxsfexMBGMWAv92(_DvkduHB| z!6+Jy-gT4SxlT7zCy_^1LMXN<(`mfa_uXVW!od_;QiftA`W7lckXb!aMUQ&7|91F1e4f@e75nRM=Igo#f2 z9%9nBNhXHfsZl%I+`p-P6nj$8+b3%>V{i@U>Q%6JY6wiD;t^eotICeWNS15Ur(Xoz z?t@9oNZww_7xs2ppJtQ%qm=KY2)CgeCiSFqZMwtBv1L-D{2~vA{3XV8D;|7>%b3EJ z$)wiSc5Er+(c`Ja)+>J~H$z-YQb8#xe>T;T$G=+Z|LOg;n_V@{MWOHB>1Zt%eYO3) z$0nzI8=p`&Jej0 z@!QrZ?{;{58(TPkz1+z6?Soa@TKa_st!w&czP1R~t6)stvnXH`gP&}$)G?5?apJpn z!^d3|-$oHvdwP1skdlaF->smJddqaV_PXn3WD<5{RJ|_UeW$nH<`o9Ul9+T!_i+ES z(y7qXq%FO?xrMF*bpiUy$3z}~sQR#v#UECEsN~IQV|!^ON-N0EYCXKg-ADUuh_3L! zt>aqn2X1pI#2R_ND z>vOj5zDpE5xpFGgi}@|jn=(v!!dsRqHV$#y3@ldX1FXH2b8mNLBP}nU#x(lFwqVyY zM@~ogYh88{4_pc-nSFdm9+e&??(N1(@2Ij~ap?*(bAEE2v-d^Q@qnzc&9&pXkLiwe zb`6)jG$Nq4-AnAWnav6?OpF)agvbyw1|ZI6@n9q>*nE>hqS3$$X?RF59qc8+z@W|G zoM=%YBpiT80k{_?JPhQ)M!_*~e@r-;7B!nXEIcFx#AAaop?CecK@{AS?a8BR_E-si#1z0cu=2C%M5a1Ym1PP+zgbBq2z(FDgObSk7pvnO33UCJ! zQKR65&`>}M7mNq^Dm*C+NE8LK(E|Vqj)9h9$l(vRH46tBf^oR;&`<){HwHpe&6EuY z1DIDRg`ozhR7TNMfRh8LITC%!1{f}&w&0mKNO2HO3}C^)+i0M^ zFr_&iV?x0OG(?aV1E&#i0aOx>1h9QTHWGy93Jne?2Eb7y0syKFJaDWp7ArywPfQA8(5EB{&L;#w?F~~ru zxp9D+gK!UXYPZDzNI1ad0lkw6fZ-V&8!#3KzDEY)GK!whTmV#w5NW7VAo&1}4+6?T zpuB}@oQ1AI^#EZB!$95Q!^ul4DT0LgPbmlzbylc(+=hfxLLj60DfoXATCR1N`4?J_ z1E3Ou$>ESK&miUM@>m4l#WTr%c)%=s{}}_6@J}oJrxpIMZiP#tZ?>xlGyy=L`!DoT zBF8x%*!?_XF+b}*w!z8=o@r;riz2ceieO(Ko~L{fsfMLL>_59iUJrzj>hY4Fbyrgj0D0emcsG>iDjp01Yn2Zjv zm&OE~?fm0eFF!5iqhsmIFM^YIxKc}#JD%~Dojdw|d^mGjq@kv=;tIbvE+uu1O>w~{ zTX*|Cd%huEqTjO|{J3_1H|M^1moHeB(>I%cW9I^l-uUH2T|qxe9;d6Bo1Z@$O;=UG9CklDdUwSDR&31|mI!T)&89 z#at0ni=sR4ap|l%6m(odj8{=u*&~G!0uH<>%NG zceb;@sY$EYOpK5n9tCI00!ShL+!621`KI)!vq}+ z;s9<9A=UwKmkN;ODniKrC_gmBF<{zZ1Tf^z8%7x;B82sx1Gz204RV=EK)AsxST)-x z1`g7;Iai(U?;v;%letD|o}+Po3;B`|D za61A4KZnEaQ5QVQC41%zvuyV}W(YC(KS2jeHt)J#CV?{T-@$d>+Gc?;g+ymTGoGw$ znt{Q`$(Zd=w0uoc%5LJ$#7FS#GiFVVKXziLg%r9xB?nKjKo&}~<`JC^6gpHTEAk{{ zQYt_C9+c=keW9X)?J=7jlM|C8)18KZci5zB*Q-bBYx+xXeRv`=5+Q&4Q2%452X#ku zzc0^;R3A4{(9r(`C*~eeZVR>fc3(_S$mLU0dC%Q2&dS$Ql~$LNJ?(}RJ2cBCTGoUc zT#{Utj*so`%)9XXM6O8+#}UKVg*9gBjxFN1&QUbaR1Q^eDIQbt6HTaL(_!LuHMEI=-G+%+1#k9L(R%)@5@` z$L1f2{h~!vlNy~~bFO*wK#FxkuKh%5?IRK60=U4#G8b!C$z-jB*Oq-Y1$8iH_BY!; zzj@*o&%`e$;S;v=>8R3?wW|iRD!z_79QWgW+Q0nL+YRGP+br%HUOq0a?P`=Ne0e#d zr+;h5#zcqcrw6b4RH}p@zgRvZ7M)EyR^wu{Ts>wj(|%TZv2=Ngy&ba^88;{_EUe6& zb}w&uvz%c?rnJ;@o~);LQ#!hLcD~f=RZNRk2^A4m&k;oFl5Q4G_-oae1SjRMfC*>v z%jj}{l3h)gzrA($YF3A#K5gG`fhV>jm!%1vw8L#v`Ly5p*iN1Ua_jIU4ugr)F!{G< zJYBZli)!${w<}mXhdKZ&*UIA%RG(C9cph0a*Gky`EaoTyV}dgHDiQQIeGVu1Y?{ZiB#&hr><60xKq}VC!{C)@wt{` z_8N&!OI+Ezh_uG)`brC?!;0HNPT8JPIe67LWiKl246Ca2q}>Uv&mQS2-0$|Onjv@D zXxZfq$|tl$Ok5MMRUmYC7m9jakS!O?w5+MLT!+Zt(0nRQzGFX{d6%Wwbv2YJx6GyW z*FPanv~{ne+ZgglY~e*tQDXMee5P>ND{LZon{Tel54=U!tIF@QGiqcn)jH)^@kwMC zSq*ddtJykb>9pR`hf}5Zhfki@Moxyk$~wEtf~qGxO6;iR(Giz@o1XU~x|_M|?T~wz zyd#=exz~M&i9JPO^*LNn!#A)6-?K#CSiZ@;mzX_ydTwdOHQl4w+K!DGiG%mMQ}k_{ zKbw+yc-03CGn9`AH

LTP=u`?lWA+C#+M5QXCb_NB2J+^*KpFNIBFcO&&ZKc(1gu zN&Hr&oU(pSWQVYN@ zUP|n^>*0;xs>hO7{7B(j>-9}~__Gv$LS+qim^Cwp(?bzGjK(9Mv+DaD;3+-lZ`DN< z>juq0ut=n6~JK?{&D&4 z*kp1w&H7S-!D`VFnQCNR>1VH_Yl@SQG94-~nt3QQ(QbOxp)173IJDIJ)!!`(8YM>6 zifL=ocvxekI9Y^suj!~XS8eDH9#-zUZ&CO@O6Z}GMo*8{p1zO#LCcfV+P;LZ3{txy zZsGNrEA-}Sg9ud1!JAcy;luJg$GqERR|=JW_6Tko`;TDVS=*(yCzgz_>c zX<3J`x61}fF(!O1UMl&Htv<|xKRx|{Bafq$TGX-aOz!ltbi_!e%Z2+D_wko+>uOBv zd39ZL^XnqK;z7GSGCa_K(SId#mwD(R;GIHn6$2 zR|;Mrko$3mIE#5VjXs-hGEGasA5mhrP7}B%cc52rzu%W!Qa9a5D0}+Z`Yl3-V>Cph zTeh}stnuTX$k`e#Mc8b(YVwtM{BW3URlzOGCm0@!7bCSX(!4_?k&{Pkc8@n*yCPpu zs+)azR{@dT`N+dBQ@3|-5ztOCnras+kv*6>jAFM<_#1m$ub~I63GHknXG-6= zUq4<{*uQnH)V@HThZ_TAHkU1zAj|X9N_` zzTri>2iIMSdsrf6UeJ;;TI_t4$d2e3@sLR;KiP3Ub8? zS`Xa4L?SeA#cgK~I*s8ilDYIo!U*xHU2)F`e%0_KT6SDd z9jtxlxtgsXtm{Tm3x1_#&FHc}?u= z4ck*hth*a(`}Q7s#iYV((vLqe_&y=SNqqZ>O*n(A5x(n_-Zk7kN6v~#GRk;xpg0h{ zTdGA^TK2d?3_WQ}yimd3oI|n4wob-u-uUe%ilZ}r;t;dWm#T^@O@@-^f`v8%frTUrb?)DuXd6E}Z z;fV6PyAfNe;@>p0ST2hsT<^PSi)*ss*eXZB32kt7=aQEli9Gi-sZGfyFz>{b2vUP3 z9Q(W_YOgEqjApn)CF>ysTcHZQ)%?m0a@v1vdt#IB^DI9sct{~y*q~eppCf)~ z#Y#l=O52Lw4s`9;spDG?sdAk441(vl$MuG?vWs4kt55DPcuy+r6(?QETrKh>dhH~? zP}aIuRF;c!i0ud822$WT#S10j>D+fpjXD}Mjk-64*kzE8ZS(i1zv@=cA}DmTR`aa7 z$=keUd#YWtuH4H{x)kS{A43n02CSLxjg^>qkCxt6mv><&c?7H8Q6Al(g1a7xSwDI9 z7X5VNlZffCrX3A>)+wuD?VKsS>)wnVxR<-*hI6jYsgyMrP@dT&;$+)nrE8seS^I-T zl@v)+0u`l9J1*WAlqh7EJkK(gAuOy=kzYdpEQA=3_Gw|)Y{kp9x25yb>{`kQjzYIm zv-(HiN0jR|$6WVu7AA6V3sb{(#CU~stkb`%4defC0PA<-*s>9Iy?Xu~Dz~gMvr;8< zVT$JqXgbZLyf@aLG<9IQy!fYq*Y6^h_33h*swM`tb?&uf`<7THs8hMG1LoA^Z}?sQ z8l|Z}T(%3|czVa{@}srt8;7i{3P_SQ-(c*|`l%KB#y4Fkelozsk{!6x!KVeARbs1o z8)N!ed;GP=SfFRYstm`C=fgL)D9XPMP2+lewEq+5&bJ>|tSlSqMV|Sl#tu^{)z#4- zdM0*w3)MeP>rkTpH=m>E*W-p+`o%esfvCCCBA!JxA`z9=F*un>0HX0&Lo8Z zw|w-CHHK-on+{y&PH$`PG}f!W<+lp2yyGsyRGB6bQ{IDbM<6`*?kcaGDq(g(i3V&XMsiU4w?;oi>?_+ZZwIV3GMw3WwBBSL z`@zyw^c;8hW_|x!>`AXU+X6|)llm{$n^N}b{fLh~8L>CHvM1xlF_Vi1sIztEtx@ZB zb@5l+tS%~FR4Q1zp7Tjv|5hGbxiHJ*^SUB&oU2N0^ah9@lAm{$aUIa{mvd2XDwdAm z{D!jrq+}^+k+CIT=KQwJ`8!lM2p-r{dZ1Be9bNI!V`cR{D?Q39Li;rzc%OEBgDQV< z$@I(wv)hVB_H6h2%dJD9w zogAeCw}fLtz`8cz=%&s1&iyfAVBPcVN-Y&hxKnU=C~&JgMFC_K4-1;}huZ`b;Tmec!Y!$gI~&LoOcfgaC7Jqs$t=P0Z?IZE0IZ)TP#HdU#&UGP zZ(ukzNCIw2W2_Iy62Mym44e{71&i!Kin-Svz?Do5h67JNBNgy?M?$J0(InvJV2L*^ z0F(p1K+x>tKYp=Lf9li!4hl_SCY-W+t*g?uCEMiD4`ZGx&eioR)OYzy$bgKo2r^J|r9lHqZz6JOYyQ7k4H? z+v+n`ak3*|v!}oTPgVO3U`!lyM}E10mr3AFnLb=ybH!u=qDJWntY1h^$Z87rwq8uKCRfw!g~IuccJVSU+VbeoRWQ zBrTEr+ut z7b(@_Z)95wBK}D0&mU+$nqDpX5S3f~Vv0f`3y*4#OYZSIu*BFQ#11JB@M*-j@&l?f z5(I}EEDi-BfBoSG@4Dxi)QSL1h%Y$DBXoA1G&tIzlwG!~i$%7pjRhQTu$ZSm;dCj> z6aLDh4HII61~UpqKny0pCBwYnSV$R|&00W%5yLpHa4vuqB%Oc6-E3yF`A2EZPy5?} zO*4rX9T@Uk`uWE?%%oqi9mia1i0MCBTRuO@?~A7A$IN?iIX`%j-+6xI;&EdxmIG4E zyp8}S0EPUjM;JI~Q+TcL*H!`iJOJw{ey{{0s8$1n7Ac6g8z@82eSWo zIsnk+f2ZSY3CsSsT@|u_*ORGWbNLGJlqAb8xT}O^H4J!7gxEp=(iIaXRKy!2NuuHj zk-j*9HKkC)B=zB5F_J+b3|vVvP{)!=a7oAy0|KbwWZ=n_45NY5ropWm<2pD5T&3(u#x9`!Q@;NTu@@fPx|sMb-9@Sg2)Aw{lV7FOLo5VBK}qP`_tFGnDB2= ze*sy>fDiWZrUi#G#$htF!+2;EGz^DCfz?aFbclxtXtU{HDio@&qoS^&CJ9DF$yxIA zA~jUNKgl?l5ElZCws+H?@u&-ZhO0s!b%ZSffp7$W(8n5q&_W=I2owTAM3@j6VMyd} z@pLHA>}R|k!gA&wd^1X6#Dg$80^|d~P$4G4Cg2;~BM??$APl*|l0pvmG;@w=;JlxO$cylvICZG@3a1RZ= zARs%HAv@XC1x-@ZvZSj2&igMt{p8Cs$W9&TA_D8?8f9rVqbG<5PYRigHAnmZl+DD- z6z^`QhSg9f<48z*hFp+t|9W{Cb>t6p4(f9rpOCyW`dNZEGY@z4OkMwN{LI%I*=)wn zJVQL)B7@vD=IjkYa@2H<#+&QWJl!pcPA)U`hcib3`$a{&YofJrKqg>=Bv5y1c7e0? zySV;CuvAA4I}=ODwm@0v0FYnJe-k;cJPZX+D$zG<3=Ysx}k$`6xV266eI5< zyR-~)rFpoK!z|5EL7o->BCcu6u(73h#Jg#SKz0Uf65<2YlRBN?I?s+NdL#_RH4y(_ z;M2uCAOcHqAvyz@Jt)8i8SQnk9+ynFR9%C<0dMM|HrwvivF=FiKh^W^?PzJcBijXI z+>vxf+ghNbCG5}{V@{^y%t8MYg+zLqqoX|C?V=gHEXC`;*Y20*H%KoE^o>wkR>!)5 z`Xv2Xy)T)L?g9Ew4|luyeb0Q`TcXbA>sy_SH%Ec|=zyPno`qVNrNz8H(!|ONr)j6| z>4bFhbVo+usA#h1jJ+&jo8)MmrYqSJ1>`W-i^7|`2K}wNB-@d3R8L^@E<_iKD;;V_ zo}irq`9koNKkbX0J>9J|Kz}~NH`JyyT*HB0)G+Rze{UZ}hT1bS0!xK>L5DzN;GgvE zpf2p_SPkuHP%kbRXw3RM`*w&=4ZSct&?{C$>tFE*jrVF8H@)z`mA@d*2&{Qz1U^b# z9cO~H_jGd&@^qW&tC#2l%&{7gVOUKZ4eyR1I#XQ9pgsP*cKh3y?n(A^Lz6t*>_Qnj zqb%-Q=Z{fn2ESGT9vWeP>p0-*rp*}ZJxzm&w$3wS&=R_J#WKb?*B~%vM%vDdX-oB4 z6s);kz`q&?JSmY89`66OF%@HuqPuI_1!6U=X6KcQb-5_bQvE)~GYqSN`d8!q0{)lk zSJm-QTlN6Ffc_`~^z%B5e%72!!ckCB1ZZwxVnrvop`vgUJ|E?jtqvT?KaT8gCm%>n`l9E`v7R#Z$^Zs@h;H4Hk8vEn(rgkpi6t`Yvv(9 zP1Rpj72F^Zk&Hkb^v#(6Gj0)9(9e85L-)}U;CTcpj))9ekUt2DpO@bTx?f;#P>?zF zyaAqHz%S!tg0PC#Q&ojzG(!RIfhdfJ6vibZ1roYM&Yu^XLw$n>1r>;ac(F1C`Bfd~ z_(7r>xfb%nC=^Z63j;Q*Y2)rn2V*zXX33t6O8TlMt80jq-ag0mG72|me z8Yh4Cqd@-Jb;@ z4~Eju)2ADB@1o%vg{P1MXUBDD-b2Pxp*cHQ11if6;$J%uls8|WP)^7&<~~~B0=am8 z076Mne%+t>O=jwYk#AmphCMmp!1EdOi~{DTBrxx0B(imwwGZg=6#C5F&+3z5R5SL+ zh-MUuT5|5kh}VPoiFO8sGUWMJ{5WEOUV&bIrcF0NMAS{|{4>hrt{WX!ZXC`_i4Q literal 0 HcmV?d00001 diff --git a/examples/notebooks/hap/output/hap_complete_output.csv b/examples/notebooks/hap/output/hap_complete_output.csv new file mode 100644 index 000000000..5727aa09d --- /dev/null +++ b/examples/notebooks/hap/output/hap_complete_output.csv @@ -0,0 +1,66 @@ +Customer Name,Date,Customer ID,Email Address,Phone Number,Shipping Address,Order Number,Issue Reported,Service Representative,Response Time,Resolution,Follow-up Needed,Customer Feedback,Unnamed: 13,hap_score +Ethan Johnson,11/7/24,782011,ethan.johnson@email.com,-4109,"134 Oak Street, Portland, OR 97204",4578342890," Ethan contacted customer service to report a recurring problem with his washing machine, Model EcoWash 300. He mentioned that during the spin cycle, the machine produces a loud, irregular banging noise. Ethan initially thought the noise might be related to something inside the drum, so he checked the drum for foreign objects like coins or small clothing items. He confirmed that no objects were present. However, the noise continues to persist even with minimal loads. He is concerned that the issue could be related to the suspension system or drum assembly, potentially causing damage to the internal components if not addressed soon. Ethan requested a technician visit to prevent further damage and to get a professional diagnosis. ",Jane Cooper,15 minutes," Jane took the time to walk Ethan through a detailed troubleshooting process, including verifying the load size and checking the balance of the washing machine. She explained that unbalanced loads can sometimes cause unusual noise during the spin cycle. However, since the issue persisted even with small loads, she advised him to check the drum suspension system and ensure that it was properly secured. Jane further explained that if the drum or suspension system had become misaligned or the shock absorbers were worn out, this could cause the banging noise. Realizing the issue might require more complex intervention, Jane scheduled a technician visit to perform a more thorough inspection of the internal components, including the suspension springs and drum motor. ","Yes, technician visit scheduled for drum suspension and motor"," Rating: 4 Comments: ""Service was prompt, but I'm worried it might be a bigger issue with the drum. Hope it gets fixed soon."" Source System: CRM --- ### Customer Service Log 2 Customer Information",,0.00019484490621834993 +Sophia Wilson,11/8/24,785673,sophia.wilson@email.com,-5331,"1025 Pine Avenue, San Francisco, CA 94107",4678452100," Sophia reported an issue with her SmartWash 500, which has been malfunctioning during its washing cycle. Specifically, the machine displays the error code ""E4"" after completing the cycle, which indicates a drainage problem. Despite attempting to troubleshoot on her own, including checking the drain filter and ensuring the drain hose was clear of obstructions, the issue persists. Sophia mentioned that there is also water left in the drum after the cycle finishes. She is concerned that it might be an issue with the pump motor or a deeper internal blockage. ",Peter Foster,30 minutes," Peter explained that the E4 error code typically indicates a drainage issue, often due to a clogged or blocked drain hose or filter. After walking Sophia through the steps to clean the filter and ensure no external blockages were present, he confirmed that the issue was still unresolved. He hypothesized that the issue could be related to the pump motor, which may not be functioning properly to expel the water. Peter emphasized that if the motor had failed, the water would not be drained ebiciently, leading to standing water in the drum. He arranged for a technician to inspect the internal pump motor and check for any blockages or malfunctioning parts that might be abecting the drainage. Peter assured Sophia that the technician would be able to further diagnose the problem and ober a solution. ","Yes, technician visit scheduled to inspect the pump motor and"," Rating: 5 Comments: ""Great help from Peter! Hope the technician resolves the issue."" Source System: CRM --- ### Customer Service Log 3 Customer Information",,0.00015312987670768052 +Liam Smith,11/6/24,781924,liam.smith@email.com,-6553,"432 Birch Road, Los Angeles, CA 90001",4589231843," Liam’s QuickWash 700 washing machine has been experiencing an issue where it intermittently stops mid-cycle and does not complete the wash. He reported that the cycle halts after about 15 minutes, and he is forced to restart the machine manually, only for it to happen again. Liam checked the load balance and even tried running the machine with smaller loads, but the problem persists. He suspects that the issue might be related to the control board or an internal sensor malfunction that is causing the cycle to terminate prematurely. ",Emily Davis,20 minutes," Emily took the time to explain to Liam that one possible cause for a washing machine to stop mid-cycle is a faulty control board or a malfunctioning sensor that fails to communicate the proper cycle status. She walked Liam through a series of checks to verify whether the machine was properly loaded or if any specific wash settings might be triggering the premature shutdown. After ruling out the simpler causes, Emily acknowledged that the problem might lie within the machine’s electronic control system. She advised Liam to keep track of the error codes, if any, that appeared on the display. Emily escalated the issue for a more in-depth inspection of the control board and its connections, as well as a sensor diagnostic. She arranged for a technician to visit and diagnose the problem further. ","Yes, technician visit scheduled to inspect the control board and"," Rating: 3 Comments: ""The service was quick, but I’m still unsure if the issue will be fixed. It keeps happening."" Source System: CRM --- ### Customer Service Log 4 Customer Information",,0.00016854520072229207 +Isabella Martinez,11/9/24,783210,isabella.martinez@email.com,-7775,"785 Oak Drive, Chicago, IL 60601",4599872301," Isabella reached out regarding her SmartWash 200 washing machine, which is failing to spin correctly. After completing the washing cycle, her clothes are still wet and heavy. Isabella mentioned that the drum doesn’t seem to be spinning at all, even when she tries to set the machine to a spin cycle manually. She noted that the drainage seems fine, and there are no unusual noises during the cycle, which led her to believe the problem could be with the spin motor or drive belt. ",Peter Foster,40 minutes," Peter first confirmed that Isabella had checked for any obvious external blockages in the drain and the drum, and all appeared fine. He then walked her through testing the spin cycle to see if the machine would respond when set to high spin speed. After determining that the drum was indeed not turning, Peter suspected that the issue might lie with either the spin motor or the drive belt. He explained that if the motor was malfunctioning, it would prevent the drum from spinning at all, even though the rest of the machine seemed to be operating normally. Given the nature of the issue, Peter scheduled a technician visit to further investigate the motor and drive belt system. ","Yes, technician visit scheduled to inspect the spin motor and drive"," Rating: 5 Comments: ""Excellent service and advice. Hopefully the technician can fix it soon."" Source System: CRM --- ### Customer Service Log 5 Customer Information",,0.00015754287596791983 +Noah White,11/10/24,780512,noah.white@email.com,-10219,"912 Willow Lane, Miami, FL 33101",4602741920," Noah’s SuperClean 800 washing machine has been experiencing intermittent water leakage around the door seal during the wash cycle. He observed that after each wash, water begins to pool around the base of the washing machine. Despite checking the door seal for any visible cracks or damage, he has not been able to identify the source of the leak. He also confirmed that the water inlet valve is functioning correctly. Noah expressed concern that the issue might be due to a defective seal or an issue with the drum assembly causing the leakage . ",Laura Simmons,10 minutes," Laura advised Noah to carefully inspect the door seal and ensure that no detergent or fabric softener had built up in the rubber, as that could cause poor sealing and leakage. After confirming that the seal appeared clean and in good condition, Laura suspected that the issue could be related to improper drum alignment or even a small crack in the seal that was dibicult to detect. She scheduled a technician visit to replace the seal and inspect the drum alignment to prevent future leaks. ","Yes, technician visit scheduled for door seal replacement and drum"," Rating: 2 Comments: ""I’m really frustrated. This issue has been ongoing for weeks and still isn’t resolved."" Source System: CRM --- These logs now include more detailed issues and resolutions, which are useful for analysis and decision-making. Sure! Here are 5 more customer service logs with detailed issues and resolutions, including a variety of feedback: --- ### Customer Service Log 6 Customer Information",,0.0008747439715079963 +Mia Green,11/10/24,787654,mia.green@email.com,-2664,"212 Cedar Way, Austin, TX 78701",4698765432," Mia reported that her EcoWash 1000 washing machine has started to vibrate excessively during the spin cycle. She mentioned that it was noticeable even with smaller loads of laundry, and the vibration causes the machine to move slightly across the floor. Mia checked that the machine was level, and the floor seemed solid, so she suspected that the issue could be related to the shock absorbers or suspension system, which might not be absorbing the vibrations properly. ",Kevin Mitchell,25 minutes," Kevin walked Mia through some troubleshooting steps to check if the machine was perfectly level and if the laundry was balanced. After confirming that the machine was indeed level and the load was balanced, Kevin suspected that the issue could be with the shock absorbers or the suspension springs, which could have worn out over time and were no longer absorbing the vibrations. He scheduled a technician visit to inspect and replace the faulty components. Kevin also recommended that Mia avoid overloading the machine until the issue is resolved, as that could further stress the system. ","Yes, technician visit scheduled to inspect and replace shock"," Rating: 4 Comments: ""The service was helpful, but I’m hoping the issue will be fully resolved with the new parts. It’s been a bit of a nuisance."" Source System: CRM --- ### Customer Service Log 7 Customer Information",,0.015778325498104095 +Daniel Thomas,11/11/24,788901,daniel.thomas@email.com,-9219,"520 Maple Road, Denver, CO 80205",4692345678," Daniel's QuickWash 800 washing machine has started to stop halfway through its wash cycle. He noted that it happens frequently, regardless of the type of load or wash cycle selected. The washing machine will start the cycle normally, but then it freezes and the timer stops progressing. Daniel confirmed that the power supply is stable and the machine is correctly plugged in. He suspected that the control board or timer could be malfunctioning, causing the machine to halt unexpectedly. ",Sarah Lewis,15 minutes," Sarah explained that the issue could indeed be related to a malfunctioning control board or timer, as these are the primary components that regulate the wash cycle. Sarah walked Daniel through a few basic steps, such as turning the machine ob for a few minutes and resetting it, but the problem persisted. She recommended that he check for any error codes or flashing lights on the display that might provide more insight into the issue. Sarah escalated the issue for further inspection and arranged for a technician to diagnose the problem with the control board and timer, as they would need to be thoroughly tested and potentially replaced. ","Yes, technician visit scheduled to inspect the control board and"," Rating: 3 Comments: ""The support was helpful, but I’m frustrated the machine keeps stopping. I’ve been waiting for a technician for a while now."" Source System: CRM --- ### Customer Service Log 8 Customer Information",,0.00022087343677412719 +Charlotte Davis,11/12/24,786502,charlotte.davis@email.com,-11330,"1500 Elm Street, New York, NY 10001",4701293847," Charlotte reported that her SmartWash 200 washing machine is experiencing issues with water leakage during the rinse cycle. She noticed water was dripping from the bottom of the door seal while the cycle was running. Charlotte has inspected the door seal and found no visible tears or cracks, but still, the machine continues to leak water. She is concerned that the issue may lie in the water inlet valve or a possible internal gasket problem. ",James Walker,20 minutes," James acknowledged that water leakage during the rinse cycle is often due to a broken door seal or an issue with the water inlet valve. He walked Charlotte through checking for any detergent residue or buildup that might cause improper sealing around the door, but Charlotte confirmed that the seal appeared clean. James explained that sometimes the gasket or internal seals can wear out or become misaligned, causing water to leak out. He recommended a technician visit to inspect both the water inlet valve and the internal seals for potential issues. James also advised Charlotte to check that the door was properly closed before starting the cycle to avoid further leaks. ","Yes, technician visit scheduled to inspect the water inlet valve and"," Rating: 5 Comments: ""The service was very quick and the representative explained everything clearly. I hope the technician can fix the leak soon."" Source System: CRM --- ### Customer Service Log 9 Customer Information",,0.00017898922669701278 +Benjamin Harris,11/10/24,780893,benjamin.harris@email.com,-5220,"934 Maple Ave, Miami, FL 33132",4691273856," Benjamin’s EcoWash 500 washing machine has been displaying an ""E3"" error code, indicating a sensor issue. The error typically appears mid-cycle, and the washing machine stops working. Benjamin attempted a reset of the machine, but the issue continued. He mentioned that the machine would run the cycle for a while, but then it would halt and display the E3 error code without completing the wash. Benjamin believes the problem may be related to a faulty water level sensor or another internal sensor. ",Olivia Taylor,10 minutes," Olivia explained that the E3 error code often indicates a sensor malfunction, especially related to the water level sensor or the pressure switch, which measures the water levels in the drum during a cycle. Olivia walked Benjamin through a series of troubleshooting steps, such as checking the water supply and ensuring the water pressure was subicient. However, since the error persisted, Olivia arranged for a technician visit to thoroughly inspect and test the sensors and wiring to identify the cause of the issue. She also noted that if a faulty sensor was found, it would need to be replaced to restore proper function. ","Yes, technician visit scheduled to inspect and replace faulty"," Rating: 5 Comments: ""The support was excellent. I’m confident the technician will sort out the issue once and for all."" Source System: CRM --- ### Customer Service Log 10 Customer Information",,0.00023553140636067837 +Grace Clark,11/11/24,789234,grace.clark@email.com,-3998,"600 Oakwood Blvd, San Diego, CA 92101",4697682012," Grace’s QuickWash 900 washing machine has been giving her problems with uneven washes. She reported that no matter what cycle she uses, the clothes come out either soaking wet or still have detergent residue on them. Grace tried adjusting the load size and used the proper detergent, but the problem persisted. She suspects that the issue might be related to an imbalance in the drum or a problem with the dispensing system that isn’t releasing the detergent properly. ",Emma Harris,30 minutes," Emma explained that an uneven wash result could be caused by several factors, including a misaligned drum, water supply issues, or even a clogged detergent dispenser. Emma walked Grace through checking the detergent compartment to make sure it was not clogged or blocked by detergent buildup. She also suggested that Grace check if the load was unbalanced, as this can sometimes cause uneven washing results. Since the issue was more complex, Emma scheduled a technician visit to inspect the drum alignment and the detergent dispensing system for blockages or malfunctions. ","Yes, technician visit scheduled to inspect drum alignment and"," Rating: 2 Comments: ""I’m frustrated with this issue. I’ve tried everything, and the machine still doesn’t wash properly. I hope the technician can figure it out."" Source System: CRM --- ### Customer Service Log 11 Customer Information",,0.0004476681933738291 +Aiden Walker,11/13/24,790321,aiden.walker@email.com,-7664,"2324 Ashwood Drive, Dallas, TX 75201",4703409821," Aiden contacted customer service about his UltraSpin 1200 washing machine, which is not heating the water properly during the wash cycle. He noticed that even after setting the machine to a hot wash, the clothes were still cold and soapy when the cycle finished. Aiden confirmed that the water inlet valve was functioning and that the temperature setting on the machine was correct. He suspects that the issue might be with the heating element or a faulty temperature sensor. ",Michelle Adams,18 minutes," Michelle explained that the issue could be caused by a broken heating element or a malfunctioning temperature sensor, both of which would prevent the water from reaching the correct temperature. She walked Aiden through verifying the settings and ensuring that the water supply was properly connected. Since the issue persisted, Michelle recommended that the heating element be inspected, as it might have failed. She scheduled a technician visit to test the element and the temperature sensor and replace any faulty parts. ","Yes, technician visit scheduled to inspect and replace the heating"," Rating: 4 Comments: ""Good support, but I'm hoping the technician can get the heating issue fixed quickly. The laundry’s been piling up!"" Source System: CRM --- ### Customer Service Log 12 Customer Information",,0.00014801646466366947 +Olivia Thompson,11/14/24,791014,olivia.thompson@email.com,-10219,"1030 Riverbend Street, Orlando, FL 32801",4704891342," Olivia reported that her WashPro 700 washing machine is exhibiting strange odors after completing wash cycles. She mentioned that the odor is sour and unpleasant, even though she has regularly cleaned the drum and used the proper detergent. Olivia is worried that the odor might be caused by mold or mildew buildup, but she hasn’t found any visible signs of it. She is concerned that the smell is coming from the drain pump or an issue with the drum gasket, as these are areas that could trap water and promote mold growth. ",Jacob Carter,25 minutes," Jacob explained that mold and mildew odors are often caused by water remaining stagnant in hidden areas, such as the drain pump, drain hose, or drum gasket, which can trap moisture and detergent residue. He recommended that Olivia run a maintenance wash with hot water and vinegar to help eliminate any buildup inside the drum. Jacob also advised checking the drain filter for any debris that might be obstructing proper drainage. Since the issue persisted, Jacob scheduled a technician visit to inspect the drain pump, gasket, and drain hose for any blockages or signs of mildew buildup. ","Yes, technician visit scheduled to inspect the drain pump, gasket,"," Rating: 3 Comments: ""The advice was helpful, but the odor is still there. Hoping the technician can sort this out quickly."" Source System: CRM --- ### Customer Service Log 13 Customer Information",,0.00745440972968936 +Henry Lewis,11/15/24,792785,henry.lewis@email.com,-7775,"875 Pine Street, Philadelphia, PA 19101",4706765432," Henry’s QuickWash 1000 washing machine has started to not respond to any button presses on the control panel. He reported that the machine powers on normally, but when he tries to start a wash cycle, none of the buttons work. The display screen also appears frozen, showing a blank screen after the initial power-up. Henry suspects that the control board or touchpad might be malfunctioning, and that this is preventing the machine from operating properly. ",Emily Scott,35 minutes," Emily explained that when a washing machine’s control panel or touchpad becomes unresponsive, it’s often a sign of a control board issue or a problem with the wiring connections. Emily recommended that Henry unplug the machine for a few minutes to reset the system, which might help if the issue was due to a temporary electronic glitch. Since this didn’t resolve the problem, Emily scheduled a technician visit to inspect the control board, touchpad, and the wiring connections for any issues. She noted that if the control board was faulty, it would need to be replaced. ","Yes, technician visit scheduled to inspect the control board and"," Rating: 4 Comments: ""Service was ebicient, but I really need this issue fixed soon as the washing machine is practically unusable right now."" Source System: CRM --- ### Customer Service Log 14 Customer Information",,0.0001770696835592389 +Ava Robinson,11/16/24,793214,ava.robinson@email.com,-5331,"520 Pinehurst Road, San Jose, CA 95128",4710923845," Ava reported that her SuperClean 800 washing machine is having trouble with the spin cycle, as it is not reaching full speed. Despite running a heavy-duty cycle, the clothes remain wet after the spin cycle finishes. Ava suspects that the issue could be with the drive motor or the spin belt, which she believes may be slipping or failing to engage properly during the spin cycle. She confirmed that the drainage is working fine and that no blockages are present. ",Sarah Morgan,30 minutes," Sarah explained that a washing machine that doesn’t reach full spin speed is often due to a malfunctioning motor or a broken spin belt, both of which can prevent the drum from reaching the correct RPM. Sarah advised Ava to check the spin cycle settings and make sure the machine was not overloaded, as that could also cause the spin cycle to run inebiciently. Since this didn’t resolve the issue, Sarah scheduled a technician visit to inspect the motor, spin belt, and drum assembly. She also advised Ava to avoid using large, heavy loads until the problem was fixed to avoid damaging the system further. ","Yes, technician visit scheduled to inspect the drive motor and spin"," Rating: 5 Comments: ""Great support. I’m confident the technician will figure out the issue and fix it."" Source System: CRM --- ### Customer Service Log 15 Customer Information",,0.000320453749736771 +Lucas Wright,11/17/24,790539,lucas.wright@email.com,-8997,"874 Birch Street, Miami, FL 33139",4709281365," Lucas reported that his WashPro 900 washing machine is consistently overfilling with water during the wash cycle, even when he sets it to a low water level. He mentioned that the machine will fill up beyond the normal water line and cause water to overflow from the door. Lucas suspects that the issue might be due to a faulty water inlet valve or pressure switch that is not regulating the water flow properly. ",Rachel Bennett,28 minutes," Rachel explained that overfilling during the wash cycle is typically caused by a malfunctioning pressure switch or water inlet valve, both of which control the amount of water entering the drum. She recommended that Lucas check for any blockages in the water inlet valve and ensure that the hose was not kinked or obstructed. Since the issue persisted, Rachel arranged for a technician visit to inspect the pressure switch, water inlet valve, and the drainage system to identify the root cause of the overfilling issue. ","Yes, technician visit scheduled to inspect the water inlet valve and"," Rating: 3 Comments: ""Support was helpful, but I’ve been dealing with this overfilling issue for weeks now. I hope the technician can fix it soon."" Source System: CRM --- ### Customer Service Log 16 Customer Information",,0.00016656539810355753 +Lily Adams,11/18/24,795023,lily.adams@email.com,-11108,"1070 Spruce Lane, Houston, TX 77001",4713485932," Lily’s SmartWash 900 washing machine is displaying a ""water flow error"" during the initial filling cycle. She reported that the water does not fill up to the appropriate level, and the machine stops after a few minutes with the error code displayed. Lily confirmed that the water supply is working in her home, and no issues with the pipes were detected. She suspects a malfunction with the water inlet valve or flow sensor. ",Ethan Clarke,15 minutes," Ethan explained that the water flow error is typically caused by issues with the water inlet valve or a faulty flow sensor, both of which control the water supply to the drum. He walked Lily through checking the inlet hose for kinks or obstructions, which could restrict water flow. Since the issue persisted, Ethan escalated the case to the Field Service Management team for an on-site technician visit. The technician would inspect the water inlet valve and flow sensor to identify any malfunction. ","Yes, technician visit scheduled to inspect and replace water inlet"," Rating: 4 Comments: ""The troubleshooting steps were useful, but I hope the technician can fix the issue soon. I need the machine working properly."" Source System: Field Service Management --- ### Customer Service Log 17 Customer Information",,0.00012659716594498605 +George Marshall,11/19/24,796521,george.marshall@email.com,-6331,"453 Birchwood Ave, Chicago, IL 60602",4712457890," George reported that his WashMax 3000 washing machine has been making a loud, grinding noise during the spin cycle. The noise has become progressively louder, and George is concerned that it could be a serious mechanical issue. He suspects the noise may be coming from the motor, drum bearings, or the drive belt. He noted that the machine still spins, but the noise is very disruptive. ",Natalie Rogers,25 minutes," Natalie explained that a loud grinding noise during the spin cycle could be caused by worn drum bearings, a loose drive belt, or a faulty motor coupling. She walked George through checking the drum alignment and ensuring that the machine was not overloaded, which could exacerbate the noise. Since the issue was mechanical and likely internal, Natalie scheduled a technician visit through SAP to inspect the motor, drum bearings, and drive belt for any signs of wear or damage. ","Yes, technician visit scheduled to inspect motor, drum bearings,"," Rating: 3 Comments: ""The support was helpful, but I’m concerned about the cost of fixing the motor or bearings. It’s been quite a frustrating experience."" Source System: SAP --- ### Customer Service Log 18 Customer Information",,0.002657181117683649 +Zoe Carter,11/20/24,797631,zoe.carter@email.com,-5331,"901 Oak Ridge Blvd, Los Angeles, CA 90001",4714579023," Zoe’s QuickSpin 1000 washing machine has an issue with excessive vibrations during the spin cycle, even with small loads. She mentioned that the machine moves across the floor during the spin, and she has already checked the leveling of the unit, which seems fine. Zoe suspects that the problem could be with the shock absorbers or suspension springs. ",Michael Harris,30 minutes," Michael explained that excessive vibrations during the spin cycle are often due to issues with the shock absorbers or suspension springs, which are responsible for stabilizing the drum during high-speed spinning. He walked Zoe through verifying that the machine was level and balanced, which she confirmed. Michael recommended that Zoe avoid overloading the machine and advised that a technician inspect the shock absorbers and suspension springs for wear or damage. He scheduled the technician visit through Field Service Management to assess and replace the faulty parts. ","Yes, technician visit scheduled to inspect and replace shock"," Rating: 4 Comments: ""The support was helpful, but I hope the technician can resolve the vibration issue soon. It's been annoying trying to balance the machine."" Source System: Field Service Management --- ### Customer Service Log 19 Customer Information",,0.0002705753722693771 +Jack Davis,11/21/24,798234,jack.davis@email.com,-9219,"3500 Maplewood Lane, San Francisco, CA 94105",4720293847," Jack reported that his EcoSpin 600 washing machine is not draining water properly at the end of the wash cycle. The machine completes the cycle, but there is still standing water in the drum. Jack has checked the drain filter and confirmed that it’s clean, but the issue persists. He suspects the drain pump or drain hose may be clogged. ",Rebecca Foster,22 minutes," Rebecca explained that improper drainage is often caused by issues with the drain pump or a clogged drain hose, which could be obstructing the flow of water. She guided Jack through inspecting the drain hose for any visible blockages. Since the hose was clear, Rebecca suggested that the drain pump might have a fault. She escalated the issue through SAP for a technician visit to inspect and possibly replace the drain pump and verify the entire drainage system. ","Yes, technician visit scheduled to inspect and replace drain pump."," Rating: 5 Comments: ""The troubleshooting steps were helpful, and the technician visit is scheduled quickly. I appreciate the fast response."" Source System: SAP --- ### Customer Service Log 20 Customer Information",,0.00015676785551477224 +Sarah Mitchell,11/22/24,799012,sarah.mitchell@email.com,-4331,"2180 Oakwood Ave, Phoenix, AZ 85001",4723128475," Sarah reported that her TurboWash 5000 washing machine is experiencing intermittent power issues. The machine powers ob unexpectedly mid-cycle and sometimes won’t turn on again without multiple attempts. Sarah noted that the power cord and outlet are fine, and there are no visible signs of a power surge. She suspects a problem with the power supply board or internal wiring. ",Thomas Hill,40 minutes," Thomas explained that intermittent power issues are typically caused by a faulty power supply board or wiring issues that cause the machine to lose connection. He advised Sarah to check if there were any error codes displayed when the power failure occurred, but Sarah confirmed there were none. Since the issue seemed related to the internal components, Thomas arranged a technician visit through Field Service Management to inspect the power supply board and internal wiring for faults and perform any necessary replacements. ","Yes, technician visit scheduled to inspect and replace the power"," Rating: 3 Comments: ""The support was good, but I’m concerned about the internal wiring issue. I’ve had problems with this machine since I got it."" Source System: Field Service Management --- ### Customer Service Log 21 Customer Information",,0.00030394032364711165 +Ben Thompson,11/23/24,799748,ben.thompson@email.com,-2321,"1155 Redwood Lane, Austin, TX 73301",4724657892," Ben reported that his TurboClean 900 washing machine is making a strange beeping noise after every wash cycle. The machine operates normally during the cycle, but the beeping happens when the cycle finishes, and it continues until the door is opened. Ben confirmed that there is no error code displayed, and the machine is functioning properly otherwise. He suspects it could be a settings issue or something related to software malfunctions. ",Olivia White,20 minutes," Olivia explained that the beeping could be a result of an alert setting indicating that the cycle has completed. She recommended checking the machine’s alarm settings and confirmed that no errors were logged. After troubleshooting, Olivia suggested that the issue might be due to a minor software glitch and escalated the case to SAP for further analysis. A technician was scheduled to check if a firmware update is needed. ","Yes, technician visit scheduled to check for software issue or"," Rating: 4 Comments: ""The beeping is annoying, but I appreciate the fast response. Hopefully, it’s just a simple fix."" Source System: SAP --- ### Customer Service Log 22 Customer Information",,0.0008498365641571581 +Rachel Clark,11/24/24,800214,rachel.clark@email.com,-6886,"2300 Oak Drive, Boston, MA 02115",4730567884," Rachel reported that her WashPro 600 washing machine has a leaking door seal. She observed that after every wash cycle, there is water around the machine’s door, and it pools on the floor. Rachel confirmed that the door is properly closed during each cycle, but water still leaks out. She suspects that the door gasket is damaged or worn out. ",John Smith,18 minutes," John explained that a leaking door seal is usually caused by a worn gasket or improper sealing. He advised Rachel to check for any visible cracks or damage on the door seal. Since Rachel confirmed that the gasket seemed worn, John scheduled a Field Service Management technician to inspect the seal and replace the damaged gasket. ","Yes, technician visit scheduled to replace door seal."," Rating: 3 Comments: ""The support was helpful, but I’ve had this problem for a while. I really hope the technician fixes it this time."" Source System: Field Service Management --- ### Customer Service Log 23 Customer Information",,0.00033511361107230186 +Lisa Roberts,11/25/24,801658,lisa.roberts@email.com,-5220,"540 Maplewood Ave, Denver, CO 80202",4731245793," Lisa’s SuperWash 1000 washing machine is not spinning correctly during the final rinse. She noted that although the machine agitates during the wash cycle, it fails to spin properly and her clothes are coming out soaking wet. She suspects a malfunction with the spin motor or clutch assembly. ",Charles Miller,22 minutes," Charles explained that the issue is likely related to the spin motor, clutch assembly, or even a broken spin belt. He guided Lisa through checking for any obstructions in the drum or any blockages in the drain hose, but since these appeared clear, he scheduled a technician visit through SAP to inspect the spin motor, clutch, and drain system. ","Yes, technician visit scheduled to inspect spin motor and clutch"," Rating: 4 Comments: ""Good support, but I’m getting frustrated that I can’t use the machine properly. Hope it gets fixed soon."" Source System: SAP --- ### Customer Service Log 24 Customer Information",,0.00020283933554310352 +Amy Wilson,11/26/24,802341,amy.wilson@email.com,-7997,"1805 Pine Ridge Dr, Seattle, WA 98001",4732018439," Amy’s UltraSpin 4000 washing machine is displaying a ""drain error"" after completing a wash cycle. The machine stops draining water and remains stuck in the rinse cycle. Amy mentioned that she has tried checking the filter and confirmed it’s clean, but the issue persists. She suspects a problem with the drain pump or clogged hose. ",Stephanie Evans,40 minutes," Stephanie advised Amy that a drain error is often caused by a clogged drain hose or a faulty drain pump. She suggested checking if the hose was kinked, but since it wasn’t, she escalated the issue to Field Service Management for a technician visit. The technician would inspect the pump and the drain hose for potential clogs or malfunctions. ","Yes, technician visit scheduled to inspect drain pump and hose."," Rating: 5 Comments: ""Great service, and I’m relieved that the issue is being handled. Looking forward to the technician's visit."" Source System: Field Service Management --- ### Customer Service Log 25 Customer Information",,0.0001272695226361975 +Mark Harris,11/27/24,803547,mark.harris@email.com,-665,"784 Cedar Street, New York, NY 10001",4733349576," Mark reported that his QuickWash 1200 washing machine’s drum is wobbling during spin cycles. The wobbling is so severe that the machine shakes and vibrates against the wall. Mark confirmed that the machine is level and that it’s not overloaded, but the problem persists. He suspects it could be an issue with the suspension system or balance rings. ",Paul Jacobs,15 minutes," Paul explained that a wobbling drum during the spin cycle is often due to issues with the suspension system, shock absorbers, or balance rings that keep the drum steady during high-speed spins. He advised Mark to try redistributing the load during a wash cycle, but since the issue continued, he arranged for a technician visit via SAP to inspect and repair the suspension system and balance rings. ","Yes, technician visit scheduled to inspect suspension system and"," Rating: 4 Comments: ""I’m glad the issue is being taken seriously. Hopefully, the technician can resolve the wobbling soon."" Source System: SAP --- ### Customer Service Log 26 Customer Information",,0.006668669171631336 +Samuel Green,11/28/24,804412,samuel.green@email.com,-4553,"678 Elmwood Drive, Phoenix, AZ 85001",4734571035," Samuel’s EcoWash 500 washing machine is having issues with water not draining fully during the spin cycle. He reported that the machine completes the wash cycle but leaves a significant amount of water in the drum. He confirmed that the filter is clear, and there are no visible blockages in the drain hose. Samuel suspects the drain pump may be malfunctioning. ",Karen Williams,28 minutes," Karen explained that the issue could likely be caused by a faulty drain pump, which would prevent water from being pumped out during the spin cycle. She advised Samuel to check the drain filter again and make sure there were no blockages. Since the issue persisted, Karen arranged a Field Service Management technician visit to inspect the drain pump and test its functionality. ","Yes, technician visit scheduled to inspect and replace drain pump."," Rating: 3 Comments: ""The machine isn’t draining properly, and I’m getting frustrated. I hope the technician can fix this soon."" Source System: Field Service Management --- ### Customer Service Log 27 Customer Information",,0.00027618688181973994 +Helen Brown,11/29/24,805273,helen.brown@email.com,-8442,"302 Birch Street, Atlanta, GA 30301",4735621948," Helen reported that her SpeedWash 300 washing machine’s detergent dispenser is not working correctly. The detergent doesn’t fully dispense, and she noticed that the machine is leaving detergent residue in the compartment. Helen suspects an issue with the water inlet valve or the dispenser system. ",Benjamin Cook,25 minutes," Benjamin advised Helen that the problem is often due to a clogged detergent dispenser or a malfunctioning water inlet valve. He walked Helen through cleaning the dispenser tray and ensuring the water supply was working properly. After confirming the issue persisted, he scheduled a technician visit through SAP to inspect the water inlet valve and ensure the dispenser was functioning correctly. ","Yes, technician visit scheduled to inspect and replace water inlet"," Rating: 5 Comments: ""Thanks for your help. I’m glad this is being addressed quickly. I’m hopeful the technician can resolve the issue."" Source System: SAP --- ### Customer Service Log 28 Customer Information",,0.00015080420416779816 +Henry Smith,11/30/24,806485,henry.smith@email.com,-9886,"442 Pinehill Road, San Jose, CA 95101",4736742017," Henry’s MaxSpin 1000 washing machine is stopping mid-cycle. The machine powers ob unexpectedly during the wash cycle, and sometimes, it won’t start at all. Henry tried resetting the machine and unplugging it, but the issue persists. He suspects a problem with the power board or electrical connections. ",Monica Taylor,35 minutes," Monica explained that sudden power losses mid-cycle are often caused by a faulty power board or a short circuit in the machine’s wiring. She recommended checking the error codes on the display, but since there were none, she scheduled a technician visit through Field Service Management to inspect the power board and electrical components. ","Yes, technician visit scheduled to inspect and replace power board."," Rating: 2 Comments: ""I’m really disappointed with the frequent breakdowns of this machine. I’ve had to call for service several times."" Source System: Field Service Management --- ### Customer Service Log 29 Customer Information",,0.00022953079314902425 +Diana Evans,12/1/24,807220,diana.evans@email.com,-7553,"2001 Aspen Court, Dallas, TX 75001",4737853099," Diana’s QuickClean 700 washing machine is displaying a ""water level sensor"" error. The machine seems to be overfilling during cycles, and Diana confirmed that the water level sensor may need to be replaced. She tried recalibrating the machine, but the issue persists. ",Jason Lee,18 minutes," Jason confirmed that a faulty water level sensor could be causing the issue and guided Diana through resetting the sensor. Since resetting didn’t resolve the error, he scheduled a technician visit through SAP to replace the water level sensor and calibrate it. ","Yes, technician visit scheduled to replace water level sensor."," Rating: 4 Comments: ""Good support, but I’m hoping the sensor replacement fixes the problem for good."" Source System: SAP --- ### Customer Service Log 30 Customer Information",,0.0003282120160292834 +Julia Green,12/2/24,808451,julia.green@email.com,-2220,"1250 Hilltop Blvd, Miami, FL 33101",4741102358," Julia’s EcoSpin 2000 washing machine has a persistent odor after each wash cycle, even after running an empty cycle with vinegar. She mentioned that the smell seems to come from the drum, and she suspects there may be mold or mildew buildup, despite cleaning the machine regularly. ",Liam Harris,30 minutes," Liam explained that odors are commonly caused by mold or mildew buildup in the door seal or drum. He advised Julia to check the drain pump filter and door seal for residue and moisture. After confirming the issue persisted, he scheduled a technician visit via SAP to perform a thorough cleaning and inspection of the drum, pump, and seals to eliminate the odor. ","Yes, technician visit scheduled to clean and inspect drum and"," Rating: 3 Comments: ""I’m disappointed that the smell keeps coming back. I hope the technician can fix it for good."" Source System: SAP --- ### Customer Service Log 31 Customer Information",,0.000546164286788553 +Darren King,12/3/24,809276,darren.king@email.com,-3998,"9457 Oakwood Drive, Orlando, FL 32801",4742239081," Darren’s SpinMaster 500 washing machine is experiencing dibiculty completing the wash cycle. The machine often gets stuck on the wash phase, and Darren needs to restart the cycle to complete it. He suspects an issue with the motor or timer mechanism. ",Rachel Scott,25 minutes," Rachel explained that a machine stuck on the wash phase is often due to a faulty timer or motor not completing the cycle. She walked Darren through resetting the machine and checking for any error codes. Since the issue persisted, Rachel escalated the issue to Field Service Management for an on-site technician visit. The technician would inspect the motor, timer, and control board to identify the root cause. ","Yes, technician visit scheduled to inspect motor and timer."," Rating: 4 Comments: ""I’m glad the support team is investigating. It’s frustrating to restart the cycle every time."" Source System: Field Service Management --- ### Customer Service Log 32 Customer Information",,0.00023480135132558644 +Martin Edwards,12/4/24,810112,martin.edwards@email.com,-998,"2030 Birchwood St, San Diego, CA 92101",4743247621," Martin’s QuickClean 1500 washing machine is not agitating during the wash cycle. The drum remains still, and the clothes are not being washed properly. Martin checked the agitator and ensured the drum was not overloaded, but the issue persists. ",Sarah White,18 minutes," Sarah explained that a lack of agitation can be caused by a faulty agitator motor, coupling, or belt. She guided Martin through verifying the belt and the agitator for any visible damage. Since the issue continued, Sarah escalated the case to SAP for a technician to inspect the agitator motor and coupling for potential failure. ","Yes, technician visit scheduled to inspect agitator motor and"," Rating: 3 Comments: ""The support was decent, but I really need the agitator fixed. It’s making washing clothes a hassle."" Source System: SAP --- ### Customer Service Log 33 Customer Information",,0.0013983179815113544 +Emily White,12/5/24,811439,emily.white@email.com,-10441,"1287 Sunset Blvd, Los Angeles, CA 90001",4744365920," Emily’s WashSmart 800 washing machine has a noisy spin cycle. The machine makes a loud, high-pitched noise when it reaches full spin speed. Emily confirmed there are no clothes caught in the drum, and the machine seems level. She suspects a problem with the drum bearings or drive belt. ",Greg Stone,27 minutes," Greg explained that the noise is likely caused by worn drum bearings or a loose drive belt. He walked Emily through checking for any visible issues with the belt and drum but confirmed that these appeared fine. Since the noise persisted, he scheduled a Field Service Management technician visit to inspect and possibly replace the drum bearings or drive belt. ","Yes, technician visit scheduled to inspect and replace drum"," Rating: 4 Comments: ""The noise is getting really annoying. Hopefully, the technician can fix it soon."" Source System: Field Service Management --- ### Customer Service Log 34 Customer Information",,0.00024661299539729953 +Patricia Moore,12/6/24,812876,patricia.moore@email.com,-7442,"3564 Oakhill Drive, Atlanta, GA 30301",4745523477," Patricia reported that her SuperWash 1200 washing machine stopped mid-cycle and won’t restart. She mentioned that the machine was working fine until it suddenly turned ob during the rinse cycle. She has tried pressing the reset button and unplugging the machine, but it still won't power back on. ",Jason Brooks,40 minutes," Jason explained that a machine turning ob mid-cycle could be caused by a power board failure, faulty wiring, or overheating protection. He guided Patricia through checking the plug and outlet for issues but confirmed there were none. He then escalated the case to SAP for a technician visit to inspect and possibly replace the power board and test the internal wiring. ","Yes, technician visit scheduled to inspect and replace power board."," Rating: 2 Comments: ""Very frustrating that the machine just turned ob. I’ve had enough of these power issues."" Source System: SAP --- ### Customer Service Log 35 Customer Information",,0.00038764093187637627 +Vanessa James,12/7/24,813492,vanessa.james@email.com,-2442,"2795 Pine View Rd, Chicago, IL 60602",4746601093," Vanessa’s WashPro 2000 washing machine is draining very slowly during the rinse cycle, and the drum is often left with standing water. She confirmed that the filter is clean, and the drain hose appears clear. She suspects a malfunction in the drain pump or a blockage in the internal pump hose. ",Lisa Clark,33 minutes," Lisa advised Vanessa that slow draining could indicate a faulty drain pump or a blockage in the pump hose. She recommended checking the pump filter for any residual debris. Since the issue persisted, Lisa scheduled a technician visit via Field Service Management to inspect and test the drain pump and pump hose for clogs or faults. ","Yes, technician visit scheduled to inspect and replace drain pump."," Rating: 5 Comments: ""I’m glad the technician is coming soon. I need the machine to drain properly before I can use it again."" Source System: Field Service Management --- ### Customer Service Log 36 Customer Information",,0.00015732910833321512 +Sophia Evans,2024-12,814256,sophia.evans@email.com,-8664,"1985 Crestwood Lane, Houston, TX 77001",4747703817, Sophia reported that her TurboClean 500 washing machine is having an issue with water temperature. The machine is stuck on cold wash even when she selects a hot water setting. She suspects the heating element or temperature sensor is malfunctioning. ,Robert Turner,22 minutes," Robert explained that a washing machine stuck on cold wash is typically due to a faulty heating element or temperature sensor. He guided Sophia through checking the water inlet valve to ensure it's working properly. Since the problem persisted, Robert arranged for a technician visit via SAP to replace the heating element and test the temperature sensor for functionality. ","Yes, technician visit scheduled to inspect and replace heating"," Rating: 4 Comments: ""I appreciate the support. I just need the temperature issue fixed quickly so I can use the machine again."" Source System: SAP --- ### Customer Service Log 37 Customer Information",,0.00029154791263863444 +Carla Peterson,12/9/24,815774,carla.peterson@email.com,-10997,"1368 Grand Avenue, San Francisco, CA 94110",4748865105," Carla’s EcoWash 1200 washing machine is not filling with water properly during the wash cycle. The machine starts, but no water enters the drum, leaving the clothes dry. Carla checked the water supply and the inlet valve and confirmed they are both working correctly. ",Matthew Lee,30 minutes," Matthew explained that no water filling the drum could be caused by a faulty water inlet valve or a clogged filter. He walked Carla through checking the valve filter and the supply hoses for blockages. Since the issue persisted, Matthew arranged for a technician visit via Field Service Management to inspect and replace the water inlet valve. ","Yes, technician visit scheduled to inspect and replace water inlet"," Rating: 3 Comments: ""I’m frustrated that the machine isn’t working. I’m hoping the technician can fix the issue soon."" Source System: Field Service Management --- ### Customer Service Log 38 Customer Information",,0.00021283984824549407 +Edward Ross,12/10/24,816042,edward.ross@email.com,-6442,"4507 Pine Hill Road, Portland, OR 97201",4749981340," Edward reported that his SpinWash 1800 washing machine’s door won’t lock properly. The machine starts a wash cycle but stops after a few minutes, showing a door lock error. Edward confirmed that there is no obstruction in the door mechanism, and the door appears aligned. ",Karen Fisher,24 minutes," Karen explained that a door lock error can be caused by a faulty door latch or door lock mechanism. She advised Edward to check for any visible damage to the door components. Since the issue continued, Karen scheduled a technician visit via SAP to inspect and replace the door lock mechanism. ","Yes, technician visit scheduled to inspect and replace door lock."," Rating: 4 Comments: ""The issue seems to be with the door. I hope the technician can get it fixed without much hassle."" Source System: SAP --- ### Customer Service Log 39 Customer Information",,0.0011916260700672865 +Marcus Johnson,12/11/24,817592,marcus.johnson@email.com,-3220,"1584 Redwood Ave, Sacramento, CA 95814",4751112378," Marcus reported that his TurboClean 500 washing machine was not draining at the end of the cycle. He mentioned that there was standing water in the drum after each wash, even though the drain hose was clear. Marcus suspected an issue with the drain pump or filter. ",Anna Gray,28 minutes," Anna advised Marcus to clean the drain filter and inspect the hose for any visible clogs. Since the issue persisted, she scheduled a technician visit via Field Service Management to inspect and replace the drain pump if necessary. ","Yes, technician visit scheduled to inspect and replace drain pump."," Rating: 4 Comments: ""I’m frustrated with this issue, but the support was decent. Hopefully the technician can finally fix it."" Source System: Field Service Management --- ### Customer Service Log 40 Customer Information",,0.00025551579892635345 +Linda Anderson,12/12/24,818130,linda.anderson@email.com,-7108,"1983 Pinecrest Way, Denver, CO 80202",4752203899," Linda’s WashMaster 3000 washing machine is stopping mid-cycle without completing the wash or spin phases. She has tried resetting the machine multiple times, but it keeps shutting ob randomly. She suspects a problem with the timer or control board. ",Oliver Harris,22 minutes," Oliver walked Linda through checking the error codes and suggested resetting the machine multiple times. After confirming the issue persisted, he escalated the case to SAP for a technician visit to inspect the control board and timer. ","Yes, technician visit scheduled to inspect and replace control"," Rating: 5 Comments: ""Thank you for helping. I hope this issue is resolved. The machine has been a nightmare for the past week."" Source System: SAP --- ### Customer Service Log 41 Customer Information",,0.00023222251911647618 +Frank Lewis,12/13/24,819444,frank.lewis@email.com,-10108,"2156 Horizon St, Chicago, IL 60611",4753401755," Frank’s MaxSpin 2000 washing machine is making a loud banging noise during the spin cycle. Frank mentioned that he made sure the machine was properly leveled, but the noise is still very loud and disruptive. He suspects an issue with the drum bearings or suspension springs. ",Kelly Morgan,18 minutes," Kelly explained that the noise could be due to worn drum bearings or suspension springs. She advised Frank to check the suspension springs for any visible damage. Since the issue persisted, Kelly arranged a technician visit via Field Service Management to inspect and replace the drum bearings. ","Yes, technician visit scheduled to inspect and replace drum"," Rating: 3 Comments: ""The noise is unbearable. I can’t stand it anymore. Please get this fixed ASAP!"" Source System: Field Service Management --- ### Customer Service Log 42 Customer Information",,0.001093846163712442 +Nathan Parker,12/14/24,820289,nathan.parker@email.com,-6886,"3327 Maple St, Phoenix, AZ 85001",4754536791," Nathan reported that his QuickWash 1000 washing machine won’t start after selecting a wash cycle. The control panel lights up, but the machine does not initiate any wash actions. Nathan suspected a malfunction with the control board or start switch. ",Jasmine Lee,35 minutes, Jasmine walked Nathan through checking the control panel and ensuring the door was properly closed. She confirmed the issue persisted and scheduled a technician visit via SAP to inspect and replace the control board or start switch. ,"Yes, technician visit scheduled to inspect and replace control"," Rating: 2 Comments: ""I am so sick of this machine. This is the third time I’ve had problems! I just want a replacement!"" Source System: SAP --- ### Customer Service Log 43 Customer Information",,0.002052868250757456 +Amanda Green,12/15/24,821377,amanda.green@email.com,-9219,"9084 Valley Rd, Portland, OR 97223",4755641032," Amanda’s EcoSpin 2000 washing machine is not spinning properly during the final cycle. The clothes are coming out soaking wet, and the machine appears to be underperforming on the spin cycle. She suspects a problem with the drum motor or drive belt. ",Thomas White,20 minutes," Thomas advised Amanda that a malfunctioning drum motor or drive belt could cause incomplete spinning. After troubleshooting, he escalated the issue to Field Service Management for a technician visit to inspect and replace the drum motor or drive belt. ","Yes, technician visit scheduled to inspect and replace drum motor."," Rating: 5 Comments: ""It’s been frustrating, but the support has been helpful. Hopefully, the technician can get this fixed."" Source System: Field Service Management --- ### Customer Service Log 44 Customer Information",,0.000362074701115489 +Kevin Taylor,12/16/24,822651,kevin.taylor@email.com,-5220,"1223 Oakridge Drive, Las Vegas, NV 89101",4756724389," Kevin’s WashPro 1200 washing machine is not dispensing detergent during the wash cycle. Despite adding detergent to the dispenser, it remains in the compartment, and the clothes don’t seem to be cleaned properly. Kevin suspects a blockage in the detergent dispenser system. ",Olivia Moore,30 minutes," Olivia explained that detergent not dispensing could be caused by a clogged dispenser or a malfunctioning water inlet valve. After guiding Kevin through cleaning the dispenser, she confirmed the issue persisted and escalated the issue to SAP for a technician visit to inspect the dispenser system and water valve. ","Yes, technician visit scheduled to inspect and clean dispenser"," Rating: 4 Comments: ""I’m annoyed that the detergent isn’t dispensing correctly. I expect a better product."" Source System: SAP --- ### Customer Service Log 45 Customer Information",,0.002651396906003356 +Alicia Harris,12/17/24,823493,alicia.harris@email.com,-7553,"3121 Elm St, New York, NY 10001",4757834098," Alicia’s SpinMaster 1000 washing machine is producing a burning smell during the spin cycle. She said the smell is noticeable even when no heavy loads are being washed. Alicia is concerned it might be a sign of a more serious issue, possibly with the motor or wiring. ",Daniel Clark,25 minutes, Daniel recommended that Alicia unplug the machine immediately as a safety precaution and not use it until it’s inspected. He scheduled a technician visit via Field Service Management to inspect and potentially replace the motor or wiring that could be causing the burning smell. ,"Yes, technician visit scheduled to inspect and replace motor."," Rating: 5 Comments: ""I’m scared to use it after that smell. Thank you for the fast response. Please fix this!"" Source System: Field Service Management --- ### Customer Service Log 46 Customer Information",,0.001158195547759533 +Julie Roberts,12/18/24,824006,julie.roberts@email.com,-9886,"2217 Sun Valley Dr, Dallas, TX 75201",4758957320," Julie’s EcoWash 1500 washing machine is leaking water from underneath the machine. The leak is noticeable when the machine starts the wash cycle, and it leaves puddles around the machine. Julie suspects a problem with the water pump or drain hose. ",Nathan Collins,30 minutes," Nathan guided Julie to check the water hoses and pump filter for any visible leaks or damage. Since the leak continued, he scheduled a technician visit via SAP to inspect and replace the water pump or hose. ","Yes, technician visit scheduled to inspect and replace pump."," Rating: 1 Comments: ""This is ridiculous! The machine has been leaking for days and no one is fixing it. I’m done with this brand!"" Source System: SAP --- ### Customer Service Log 47 Customer Information",,0.513248085975647 +Brian Mitchell,12/19/24,825137,brian.mitchell@email.com,-1442,"2571 Briarwood Dr, Austin, TX 73301",4760018825," Brian’s SmartWash 700 washing machine won’t drain properly at the end of the cycle. Water remains in the drum after completing the cycle, and the clothes are still soaked. Brian tried checking the drain hose and filter but the problem persists. ",Helen Scott,25 minutes," Helen explained that the issue could be caused by a faulty drain pump or blockage in the drain hose. After troubleshooting, she scheduled a technician visit via Field Service Management to inspect and replace the drain pump. ","Yes, technician visit scheduled to inspect and replace pump."," Rating: 3 Comments: ""I’ve had enough of these problems. If the technician doesn’t fix it soon, I’ll demand a refund."" Source System: Field Service Management --- ### Customer Service Log 48 Customer Information",,0.00032178364926949143 +Thomas Williams,12/20/24,826255,thomas.williams@email.com,-10886,"1152 Ashwood Dr, Minneapolis, MN 55123",4761134120," Thomas reported that his UltraWash 3000 washing machine is vibrating excessively during the spin cycle, causing the whole unit to shake and move across the floor. He mentioned that he had leveled the machine but the issue continues. ",Mark Johnson,22 minutes," Mark explained that an unbalanced load or worn suspension springs could be causing the vibration. He guided Thomas through checking the load distribution and inspecting the suspension springs. Since the issue persisted, Mark scheduled a technician visit via SAP to inspect and replace the suspension springs. ","Yes, technician visit scheduled to inspect and replace suspension"," Rating: 4 Comments: ""I’m glad someone’s coming to fix it, but I’ve had enough of this. I just want the machine to work properly!"" Source System: SAP --- ### Customer Service Log 49 Customer Information",,0.00022851758694741875 +Jessica Bryant,12/21/24,827404,jessica.bryant@email.com,-2442,"4632 Birchwood Rd, San Diego, CA 92103",4762231882," Jessica’s SmartWash 500 washing machine is not agitating properly during the wash cycle. The drum spins slowly, and clothes are not being cleaned as expected. Jessica believes the agitator or motor is malfunctioning. ",Nancy Hall,30 minutes," Nancy walked Jessica through checking the agitator for blockages and confirmed it was moving improperly. Since the issue wasn’t resolved, she escalated the case via Field Service Management for a technician visit to inspect and replace the agitator or motor. ","Yes, technician visit scheduled to inspect and replace agitator."," Rating: 5 Comments: ""The support team was helpful, but I’m really annoyed this happened right after the warranty expired."" Source System: Field Service Management --- ### Customer Service Log 50 Customer Information",,0.00014407523849513382 +Natalie Cooper,12/22/24,828552,natalie.cooper@email.com,-6331,"1056 Kings Rd, Orlando, FL 32801",4763347501," Natalie’s TurboSpin 1200 washing machine is leaking water from the bottom panel during the wash cycle. The water pools on the floor around the machine, and Natalie suspects a problem with the water pump or seal. ",Leo Davis,26 minutes," Leo explained that a water leak could be caused by a damaged water seal or faulty water pump. He advised Natalie to inspect the water hose and drain filter for any visible damage. Since the leak persisted, Leo scheduled a technician visit via SAP to inspect and replace the water pump or seal. ","Yes, technician visit scheduled to inspect and replace water pump."," Rating: 2 Comments: ""This is the second time my machine is leaking. I’m getting tired of this brand. FIX IT."" Source System: SAP --- ### Customer Service Log 51 Customer Information",,0.0002608941576909274 +Adam Rodriguez,12/23/24,829701,adam.rodriguez@email.com,-4331,"2987 Oak Valley Dr, Miami, FL 33101",4764450613," Adam’s QuickWash 1500 washing machine is not spinning at all during the final spin cycle. The clothes are coming out wet and heavy, and the drum does not rotate. Adam has checked the door lock and load balance, but the problem persists. ",Carla Moore,24 minutes," Carla explained that the issue could be due to a broken drive belt or faulty motor. She advised Adam to inspect the drive belt for any visible damage. Since the issue wasn’t resolved, Carla scheduled a technician visit via Field Service Management to inspect and replace the drive belt or motor. ","Yes, technician visit scheduled to inspect and replace drive belt."," Rating: 5 Comments: ""I’m hoping this gets fixed soon. I’ve already had too many issues with this machine."" Source System: Field Service Management --- ### Customer Service Log 52 Customer Information",,0.00043143669608980417 +Barbara Grant,12/24/24,830014,barbara.grant@email.com,-5664,"5743 Riverside Ave, Austin, TX 73301",4765564392," Barbara’s EcoSpin 1000 washing machine is overheating during the wash cycle. The temperature sensor seems to be malfunctioning as the water gets hotter than expected, potentially damaging clothes. Barbara has confirmed that the thermostat and water inlet valve are functioning. ",Daniel Scott,33 minutes, Daniel explained that overheating can be caused by a faulty temperature sensor or malfunctioning control board. He escalated the issue via SAP and scheduled a technician visit to inspect and replace the temperature sensor or control board. ,"Yes, technician visit scheduled to inspect and replace temperature"," Rating: 3 Comments: ""The machine shouldn’t be overheating like this. It’s frustrating, especially since it’s brand new!"" Source System: SAP --- ### Customer Service Log 53 Customer Information",,0.0005323936929926276 +Eric Foster,12/25/24,831179,eric.foster@email.com,-2553,"7455 Hilltop Ln, Los Angeles, CA 90001",4766678245," Eric’s SpinClean 800 washing machine is not draining water at the end of the cycle. The water remains in the drum, and the clothes are still soaking wet. Eric has already cleaned the drain filter and checked the drain hose but the issue persists. ",Helen Taylor,20 minutes, Helen explained that the issue could be with the drain pump or a blockage in the drain hose. She arranged a technician visit via Field Service Management to inspect the pump and hose for any obstructions. ,"Yes, technician visit scheduled to inspect and replace drain pump."," Rating: 1 Comments: ""I’ve had enough of this junk! It’s been a month of issues and no one’s fixing it! Get this thing replaced."" Source System: Field Service Management --- ### Customer Service Log 54 Customer Information",,0.8846611380577087 +Samuel Lewis,12/26/24,832254,samuel.lewis@email.com,-998,"1586 Willow Creek Rd, Dallas, TX 75201",4767789337," Samuel’s WashMaster 4000 washing machine is producing a strange odor during the wash cycle. He suspects the drain pump or filter might be clogged, but he cannot find any visible issues. ",Anna Walker,30 minutes," Anna advised Samuel to clean the drain pump and filter thoroughly. After confirming the issue persisted, she scheduled a technician visit via SAP to inspect the pump and filter system for any internal clogs. ","Yes, technician visit scheduled to inspect and clean pump."," Rating: 4 Comments: ""The odor is unbearable, but at least I have someone coming to fix it."" Source System: SAP --- ### Customer Service Log 55 Customer Information",,0.0009222364169545472 +Rachel White,12/27/24,833411,rachel.white@email.com,-3998,"3185 Creekside Dr, San Francisco, CA 94105",4768890246," Rachel’s MaxSpin 2000 washing machine is not completing the wash cycle. The machine shuts ob halfway through, and she needs to restart it multiple times to finish the cycle. She suspects an issue with the timer or control board. ",Jennifer Miller,29 minutes," Jennifer advised Rachel to check for any error codes or obstructions. Since the issue persisted, she escalated the issue to Field Service Management for a technician visit to inspect and replace the timer or control board. ","Yes, technician visit scheduled to inspect and replace control"," Rating: 2 Comments: ""I don’t care about the technician visit. Just fix this machine! It’s been down too long."" Source System: Field Service Management --- ### Customer Service Log 56 Customer Information",,0.00040924124186858535 +Peter Knight,12/28/24,834558,peter.knight@email.com,-8553,"2450 Glenwood Rd, Seattle, WA 98101",4770003942," Peter’s TurboClean 1500 washing machine is not starting when he presses the power button. The lights on the control panel come on, but it doesn’t respond when he selects a cycle. Peter suspects a problem with the start button or control board. ",Brian King,27 minutes," Brian walked Peter through checking the power connections and error codes. After confirming the issue persisted, he escalated the case via SAP and scheduled a technician visit to inspect the control board. ","Yes, technician visit scheduled to inspect and replace control"," Rating: 4 Comments: ""I’m beyond frustrated at this point. This machine is brand new and it’s already broken!"" Source System: SAP --- ### Customer Service Log 57 Customer Information",,0.0005144605529494584 +Erica Matthews,12/29/24,835744,erica.matthews@email.com,-5331,"9725 Maplewood St, Portland, OR 97205",4771123567, Erica's EcoWash 1000 washing machine is making a loud grinding noise during the wash cycle. She suspects an issue with the drive belt or motor. ,James Carter,28 minutes, James explained that the grinding noise could be due to a worn drive belt or damaged motor bearings. He scheduled a technician visit via SAP to inspect and potentially replace the drive belt or motor. ,"Yes, technician visit scheduled to inspect and replace drive belt."," Rating: 3 Comments: ""It’s been so frustrating. I don’t know how this machine got so loud so quickly. Let’s see if the technician can fix it."" Source System: SAP --- ### Customer Service Log 58 Customer Information",,0.00046848144847899675 +Michael Spencer,12/30/24,836011,michael.spencer@email.com,-10219,"1156 Eagle St, Denver, CO 80203",4772234568," Michael reported that his SpinMaster 2500 washing machine isn’t spinning at all. The drum remains stationary, and clothes are still soaked after the cycle. ",Emily White,24 minutes," Emily suggested that the drum motor could be faulty, or there could be an issue with the drive belt. She scheduled a technician visit via Field Service Management to inspect and replace the motor or drive belt. ","Yes, technician visit scheduled to inspect and replace motor."," Rating: 5 Comments: ""Quick and ebicient response. I hope the technician can fix it soon."" Source System: Field Service Management --- ### Customer Service Log 59 Customer Information",,0.00019660884572658688 +Jennifer Thomas,12/31/24,836895,jennifer.thomas@email.com,-7442,"3446 Sunridge Ave, Phoenix, AZ 85001",4773345569," Jennifer's MaxSpin 3000 washing machine is not draining properly, and water is left standing in the drum after each cycle. She believes there may be a blockage in the drain hose or filter. ",Laura Phillips,21 minutes," Laura advised Jennifer to clean the drain filter and inspect the hose for obstructions. When the issue wasn’t resolved, Laura escalated the case via SAP to schedule a technician visit to check the drain pump and hose. ","Yes, technician visit scheduled to inspect and clean the drain pump."," Rating: 4 Comments: ""The technician better sort this out. I’ve cleaned everything and it’s still broken."" Source System: SAP --- ### Customer Service Log 60 Customer Information",,0.00048811433953233063 +Steve Harvey,1/1/25,837112,steve.harvey@email.com,-6664,"7900 Northview Dr, Seattle, WA 98109",4774456670, Steve reported that his QuickClean 5000 washing machine’s display screen is flickering and showing random error codes. He’s unable to start a cycle because of this malfunction. ,Sam Collins,19 minutes, Sam explained that the issue could be related to the control board or wiring. He scheduled a technician visit via Field Service Management to inspect and replace the control board. ,"Yes, technician visit scheduled to inspect and replace control"," Rating: 2 Comments: ""How hard is it to make a washing machine that actually works? I’m sick of having to call for repairs."" Source System: Field Service Management --- ### Customer Service Log 61 Customer Information",,0.00046817120164632797 +Karen Brooks,1/2/25,838225,karen.brooks@email.com,-2553,"6715 Valley Dr, Chicago, IL 60601",4775567781," Karen’s EcoSpin 2000 washing machine is overheating during the wash cycle. The water temperature is rising too high, potentially ruining clothes. ",Patricia Lee,32 minutes, Patricia explained that overheating could be caused by a malfunctioning thermostat or control board. She scheduled a technician visit via SAP to inspect and replace the thermostat or control board. ,"Yes, technician visit scheduled to inspect and replace thermostat."," Rating: 3 Comments: ""This is not what I expected from a high-end machine. Can you get this sorted out quickly?"" Source System: SAP --- ### Customer Service Log 62 Customer Information",,0.00014980968262534589 +David King,1/3/25,839478,david.king@email.com,-3775,"8903 Pinehurst Dr, Houston, TX 77001",4776678892," David’s TurboWash 800 washing machine is not starting at all. The power button doesn’t respond, and there are no lights on the control panel. ",Brian Martin,25 minutes," Brian suggested checking the power supply and circuit breaker. Since the machine still wouldn’t start, he escalated the case via Field Service Management to schedule a technician visit to inspect the control board and power connections. ","Yes, technician visit scheduled to inspect and replace control"," Rating: 1 Comments: ""This is insane. I’ve only had this machine for a year, and it’s already broken. This brand is trash."" Source System: Field Service Management --- ### Customer Service Log 63 Customer Information",,0.5792514085769653 +Angela Hill,1/4/25,840325,angela.hill@email.com,-8997,"1108 Crestwood Rd, Los Angeles, CA 90015",4777789921, Angela’s SmartWash 7000 washing machine is not spinning clothes dry. The machine goes through the wash cycle but leaves the clothes soaking wet. ,Kevin Clark,27 minutes, Kevin suggested that the spin motor or pump could be malfunctioning. He arranged a technician visit via SAP to inspect and replace the spin motor or pump. ,"Yes, technician visit scheduled to inspect and replace motor."," Rating: 4 Comments: ""I need this fixed. I can’t abord to keep waiting for a solution."" Source System: SAP --- ### Customer Service Log 64 Customer Information",,0.0003836100222542882 +Richard Young,1/5/25,841196,richard.young@email.com,-9553,"4822 Oakview Dr, Dallas, TX 75209",4778891104, Richard’s SpinForce 2500 washing machine is stopping midway through the wash cycle and not draining the water . ,Olivia Stone,34 minutes," Olivia suggested checking for any blockages in the drain hose and pump. Since the issue wasn’t resolved, she escalated the case via Field Service Management to schedule a technician visit to inspect the pump and drain hose. ","Yes, technician visit scheduled to inspect pump."," Rating: 2 Comments: ""I’m so done with this machine. I can’t trust it to work. Just get it fixed already."" Source System: Field Service Management --- ### Customer Service Log 65 Customer Information",,0.0002852928882930428 +Lisa Green,1/6/25,842401,lisa.green@email.com,-5775,"5224 Ashford Ave, Miami, FL 33165",4779912335, Lisa’s TurboClean 1500 washing machine is vibrating excessively during the spin cycle and making an unusual screeching noise. ,Scott Gray,29 minutes," Scott explained that the issue could be related to unbalanced loads or a faulty suspension system. Since the issue persisted, he escalated the case via SAP and scheduled a technician visit to inspect the suspension springs and balance. ","Yes, technician visit scheduled to inspect suspension springs."," Rating: 3 Comments: ""I’ve leveled the machine, but it’s still vibrating like crazy. Something’s clearly wrong."" Source System: SAP ",,0.05251740291714668 diff --git a/examples/notebooks/hap/output/hap_filtered_output.csv b/examples/notebooks/hap/output/hap_filtered_output.csv new file mode 100644 index 000000000..13f38a25e --- /dev/null +++ b/examples/notebooks/hap/output/hap_filtered_output.csv @@ -0,0 +1,66 @@ +Customer Feedback,hap_score +" Rating: 4 Comments: ""Service was prompt, but I'm worried it might be a bigger issue with the drum. Hope it gets fixed soon."" Source System: CRM --- ### Customer Service Log 2 Customer Information",0.00019484490621834993 +" Rating: 5 Comments: ""Great help from Peter! Hope the technician resolves the issue."" Source System: CRM --- ### Customer Service Log 3 Customer Information",0.00015312987670768052 +" Rating: 3 Comments: ""The service was quick, but I’m still unsure if the issue will be fixed. It keeps happening."" Source System: CRM --- ### Customer Service Log 4 Customer Information",0.00016854520072229207 +" Rating: 5 Comments: ""Excellent service and advice. Hopefully the technician can fix it soon."" Source System: CRM --- ### Customer Service Log 5 Customer Information",0.00015754287596791983 +" Rating: 2 Comments: ""I’m really frustrated. This issue has been ongoing for weeks and still isn’t resolved."" Source System: CRM --- These logs now include more detailed issues and resolutions, which are useful for analysis and decision-making. Sure! Here are 5 more customer service logs with detailed issues and resolutions, including a variety of feedback: --- ### Customer Service Log 6 Customer Information",0.0008747439715079963 +" Rating: 4 Comments: ""The service was helpful, but I’m hoping the issue will be fully resolved with the new parts. It’s been a bit of a nuisance."" Source System: CRM --- ### Customer Service Log 7 Customer Information",0.015778325498104095 +" Rating: 3 Comments: ""The support was helpful, but I’m frustrated the machine keeps stopping. I’ve been waiting for a technician for a while now."" Source System: CRM --- ### Customer Service Log 8 Customer Information",0.00022087343677412719 +" Rating: 5 Comments: ""The service was very quick and the representative explained everything clearly. I hope the technician can fix the leak soon."" Source System: CRM --- ### Customer Service Log 9 Customer Information",0.00017898922669701278 +" Rating: 5 Comments: ""The support was excellent. I’m confident the technician will sort out the issue once and for all."" Source System: CRM --- ### Customer Service Log 10 Customer Information",0.00023553140636067837 +" Rating: 2 Comments: ""I’m frustrated with this issue. I’ve tried everything, and the machine still doesn’t wash properly. I hope the technician can figure it out."" Source System: CRM --- ### Customer Service Log 11 Customer Information",0.0004476681933738291 +" Rating: 4 Comments: ""Good support, but I'm hoping the technician can get the heating issue fixed quickly. The laundry’s been piling up!"" Source System: CRM --- ### Customer Service Log 12 Customer Information",0.00014801646466366947 +" Rating: 3 Comments: ""The advice was helpful, but the odor is still there. Hoping the technician can sort this out quickly."" Source System: CRM --- ### Customer Service Log 13 Customer Information",0.00745440972968936 +" Rating: 4 Comments: ""Service was ebicient, but I really need this issue fixed soon as the washing machine is practically unusable right now."" Source System: CRM --- ### Customer Service Log 14 Customer Information",0.0001770696835592389 +" Rating: 5 Comments: ""Great support. I’m confident the technician will figure out the issue and fix it."" Source System: CRM --- ### Customer Service Log 15 Customer Information",0.000320453749736771 +" Rating: 3 Comments: ""Support was helpful, but I’ve been dealing with this overfilling issue for weeks now. I hope the technician can fix it soon."" Source System: CRM --- ### Customer Service Log 16 Customer Information",0.00016656539810355753 +" Rating: 4 Comments: ""The troubleshooting steps were useful, but I hope the technician can fix the issue soon. I need the machine working properly."" Source System: Field Service Management --- ### Customer Service Log 17 Customer Information",0.00012659716594498605 +" Rating: 3 Comments: ""The support was helpful, but I’m concerned about the cost of fixing the motor or bearings. It’s been quite a frustrating experience."" Source System: SAP --- ### Customer Service Log 18 Customer Information",0.002657181117683649 +" Rating: 4 Comments: ""The support was helpful, but I hope the technician can resolve the vibration issue soon. It's been annoying trying to balance the machine."" Source System: Field Service Management --- ### Customer Service Log 19 Customer Information",0.0002705753722693771 +" Rating: 5 Comments: ""The troubleshooting steps were helpful, and the technician visit is scheduled quickly. I appreciate the fast response."" Source System: SAP --- ### Customer Service Log 20 Customer Information",0.00015676785551477224 +" Rating: 3 Comments: ""The support was good, but I’m concerned about the internal wiring issue. I’ve had problems with this machine since I got it."" Source System: Field Service Management --- ### Customer Service Log 21 Customer Information",0.00030394032364711165 +" Rating: 4 Comments: ""The beeping is annoying, but I appreciate the fast response. Hopefully, it’s just a simple fix."" Source System: SAP --- ### Customer Service Log 22 Customer Information",0.0008498365641571581 +" Rating: 3 Comments: ""The support was helpful, but I’ve had this problem for a while. I really hope the technician fixes it this time."" Source System: Field Service Management --- ### Customer Service Log 23 Customer Information",0.00033511361107230186 +" Rating: 4 Comments: ""Good support, but I’m getting frustrated that I can’t use the machine properly. Hope it gets fixed soon."" Source System: SAP --- ### Customer Service Log 24 Customer Information",0.00020283933554310352 +" Rating: 5 Comments: ""Great service, and I’m relieved that the issue is being handled. Looking forward to the technician's visit."" Source System: Field Service Management --- ### Customer Service Log 25 Customer Information",0.0001272695226361975 +" Rating: 4 Comments: ""I’m glad the issue is being taken seriously. Hopefully, the technician can resolve the wobbling soon."" Source System: SAP --- ### Customer Service Log 26 Customer Information",0.006668669171631336 +" Rating: 3 Comments: ""The machine isn’t draining properly, and I’m getting frustrated. I hope the technician can fix this soon."" Source System: Field Service Management --- ### Customer Service Log 27 Customer Information",0.00027618688181973994 +" Rating: 5 Comments: ""Thanks for your help. I’m glad this is being addressed quickly. I’m hopeful the technician can resolve the issue."" Source System: SAP --- ### Customer Service Log 28 Customer Information",0.00015080420416779816 +" Rating: 2 Comments: ""I’m really disappointed with the frequent breakdowns of this machine. I’ve had to call for service several times."" Source System: Field Service Management --- ### Customer Service Log 29 Customer Information",0.00022953079314902425 +" Rating: 4 Comments: ""Good support, but I’m hoping the sensor replacement fixes the problem for good."" Source System: SAP --- ### Customer Service Log 30 Customer Information",0.0003282120160292834 +" Rating: 3 Comments: ""I’m disappointed that the smell keeps coming back. I hope the technician can fix it for good."" Source System: SAP --- ### Customer Service Log 31 Customer Information",0.000546164286788553 +" Rating: 4 Comments: ""I’m glad the support team is investigating. It’s frustrating to restart the cycle every time."" Source System: Field Service Management --- ### Customer Service Log 32 Customer Information",0.00023480135132558644 +" Rating: 3 Comments: ""The support was decent, but I really need the agitator fixed. It’s making washing clothes a hassle."" Source System: SAP --- ### Customer Service Log 33 Customer Information",0.0013983179815113544 +" Rating: 4 Comments: ""The noise is getting really annoying. Hopefully, the technician can fix it soon."" Source System: Field Service Management --- ### Customer Service Log 34 Customer Information",0.00024661299539729953 +" Rating: 2 Comments: ""Very frustrating that the machine just turned ob. I’ve had enough of these power issues."" Source System: SAP --- ### Customer Service Log 35 Customer Information",0.00038764093187637627 +" Rating: 5 Comments: ""I’m glad the technician is coming soon. I need the machine to drain properly before I can use it again."" Source System: Field Service Management --- ### Customer Service Log 36 Customer Information",0.00015732910833321512 +" Rating: 4 Comments: ""I appreciate the support. I just need the temperature issue fixed quickly so I can use the machine again."" Source System: SAP --- ### Customer Service Log 37 Customer Information",0.00029154791263863444 +" Rating: 3 Comments: ""I’m frustrated that the machine isn’t working. I’m hoping the technician can fix the issue soon."" Source System: Field Service Management --- ### Customer Service Log 38 Customer Information",0.00021283984824549407 +" Rating: 4 Comments: ""The issue seems to be with the door. I hope the technician can get it fixed without much hassle."" Source System: SAP --- ### Customer Service Log 39 Customer Information",0.0011916260700672865 +" Rating: 4 Comments: ""I’m frustrated with this issue, but the support was decent. Hopefully the technician can finally fix it."" Source System: Field Service Management --- ### Customer Service Log 40 Customer Information",0.00025551579892635345 +" Rating: 5 Comments: ""Thank you for helping. I hope this issue is resolved. The machine has been a nightmare for the past week."" Source System: SAP --- ### Customer Service Log 41 Customer Information",0.00023222251911647618 +" Rating: 3 Comments: ""The noise is unbearable. I can’t stand it anymore. Please get this fixed ASAP!"" Source System: Field Service Management --- ### Customer Service Log 42 Customer Information",0.001093846163712442 +" Rating: 2 Comments: ""I am so sick of this machine. This is the third time I’ve had problems! I just want a replacement!"" Source System: SAP --- ### Customer Service Log 43 Customer Information",0.002052868250757456 +" Rating: 5 Comments: ""It’s been frustrating, but the support has been helpful. Hopefully, the technician can get this fixed."" Source System: Field Service Management --- ### Customer Service Log 44 Customer Information",0.000362074701115489 +" Rating: 4 Comments: ""I’m annoyed that the detergent isn’t dispensing correctly. I expect a better product."" Source System: SAP --- ### Customer Service Log 45 Customer Information",0.002651396906003356 +" Rating: 5 Comments: ""I’m scared to use it after that smell. Thank you for the fast response. Please fix this!"" Source System: Field Service Management --- ### Customer Service Log 46 Customer Information",0.001158195547759533 +" Rating: 1 Comments: ""This is ridiculous! The machine has been leaking for days and no one is fixing it. I’m done with this brand!"" Source System: SAP --- ### Customer Service Log 47 Customer Information",0.513248085975647 +" Rating: 3 Comments: ""I’ve had enough of these problems. If the technician doesn’t fix it soon, I’ll demand a refund."" Source System: Field Service Management --- ### Customer Service Log 48 Customer Information",0.00032178364926949143 +" Rating: 4 Comments: ""I’m glad someone’s coming to fix it, but I’ve had enough of this. I just want the machine to work properly!"" Source System: SAP --- ### Customer Service Log 49 Customer Information",0.00022851758694741875 +" Rating: 5 Comments: ""The support team was helpful, but I’m really annoyed this happened right after the warranty expired."" Source System: Field Service Management --- ### Customer Service Log 50 Customer Information",0.00014407523849513382 +" Rating: 2 Comments: ""This is the second time my machine is leaking. I’m getting tired of this brand. FIX IT."" Source System: SAP --- ### Customer Service Log 51 Customer Information",0.0002608941576909274 +" Rating: 5 Comments: ""I’m hoping this gets fixed soon. I’ve already had too many issues with this machine."" Source System: Field Service Management --- ### Customer Service Log 52 Customer Information",0.00043143669608980417 +" Rating: 3 Comments: ""The machine shouldn’t be overheating like this. It’s frustrating, especially since it’s brand new!"" Source System: SAP --- ### Customer Service Log 53 Customer Information",0.0005323936929926276 +" Rating: 1 Comments: ""I’ve had enough of this junk! It’s been a month of issues and no one’s fixing it! Get this thing replaced."" Source System: Field Service Management --- ### Customer Service Log 54 Customer Information",0.8846611380577087 +" Rating: 4 Comments: ""The odor is unbearable, but at least I have someone coming to fix it."" Source System: SAP --- ### Customer Service Log 55 Customer Information",0.0009222364169545472 +" Rating: 2 Comments: ""I don’t care about the technician visit. Just fix this machine! It’s been down too long."" Source System: Field Service Management --- ### Customer Service Log 56 Customer Information",0.00040924124186858535 +" Rating: 4 Comments: ""I’m beyond frustrated at this point. This machine is brand new and it’s already broken!"" Source System: SAP --- ### Customer Service Log 57 Customer Information",0.0005144605529494584 +" Rating: 3 Comments: ""It’s been so frustrating. I don’t know how this machine got so loud so quickly. Let’s see if the technician can fix it."" Source System: SAP --- ### Customer Service Log 58 Customer Information",0.00046848144847899675 +" Rating: 5 Comments: ""Quick and ebicient response. I hope the technician can fix it soon."" Source System: Field Service Management --- ### Customer Service Log 59 Customer Information",0.00019660884572658688 +" Rating: 4 Comments: ""The technician better sort this out. I’ve cleaned everything and it’s still broken."" Source System: SAP --- ### Customer Service Log 60 Customer Information",0.00048811433953233063 +" Rating: 2 Comments: ""How hard is it to make a washing machine that actually works? I’m sick of having to call for repairs."" Source System: Field Service Management --- ### Customer Service Log 61 Customer Information",0.00046817120164632797 +" Rating: 3 Comments: ""This is not what I expected from a high-end machine. Can you get this sorted out quickly?"" Source System: SAP --- ### Customer Service Log 62 Customer Information",0.00014980968262534589 +" Rating: 1 Comments: ""This is insane. I’ve only had this machine for a year, and it’s already broken. This brand is trash."" Source System: Field Service Management --- ### Customer Service Log 63 Customer Information",0.5792514085769653 +" Rating: 4 Comments: ""I need this fixed. I can’t abord to keep waiting for a solution."" Source System: SAP --- ### Customer Service Log 64 Customer Information",0.0003836100222542882 +" Rating: 2 Comments: ""I’m so done with this machine. I can’t trust it to work. Just get it fixed already."" Source System: Field Service Management --- ### Customer Service Log 65 Customer Information",0.0002852928882930428 +" Rating: 3 Comments: ""I’ve leveled the machine, but it’s still vibrating like crazy. Something’s clearly wrong."" Source System: SAP ",0.05251740291714668 diff --git a/examples/notebooks/hap/output/metadata.json b/examples/notebooks/hap/output/metadata.json new file mode 100644 index 000000000..b9f7c11a3 --- /dev/null +++ b/examples/notebooks/hap/output/metadata.json @@ -0,0 +1,56 @@ +{ + "pipeline": "pipeline_id", + "job details": { + "job category": "preprocessing", + "job name": "hap", + "job type": "pure python", + "job id": "job_id", + "start_time": "2024-11-29 13:41:19", + "end_time": "2024-11-29 13:41:21", + "status": "success" + }, + "code": { + "github": "github", + "commit_hash": "12345", + "path": "path" + }, + "job_input_params": { + "model_name_or_path": "ibm-granite/granite-guardian-hap-38m", + "annotation_column": "hap_score", + "doc_text_column": "Customer Feedback", + "inference_engine": "CPU", + "max_length": 512, + "batch_size": 128, + "checkpointing": false, + "max_files": -1, + "random_samples": -1, + "files_to_use": [ + ".parquet" + ], + "num_processors": 0 + }, + "execution_stats": { + "cpus": 35.7, + "gpus": 0, + "memory": 9.43, + "object_store": 0, + "execution time, min": 0.037 + }, + "job_output_stats": { + "source_files": 1, + "source_size": 47309, + "result_files": 1, + "result_size": 40282, + "processing_time": 0.419, + "source_doc_count": 65, + "result_doc_count": 65 + }, + "source": { + "name": "/Users/aisha/Documents/GitHub/Personal/DPK/examples/notebooks/hap/input", + "type": "path" + }, + "target": { + "name": "/Users/aisha/Documents/GitHub/Personal/DPK/examples/notebooks/hap/output", + "type": "path" + } +} \ No newline at end of file diff --git a/transforms/universal/hap/python/src/hap_config.py b/transforms/universal/hap/python/src/hap_config.py new file mode 100644 index 000000000..3dc3568c4 --- /dev/null +++ b/transforms/universal/hap/python/src/hap_config.py @@ -0,0 +1,23 @@ +# hap_config.py +import os + +def get_hap_params(custom_params=None): + """ + Returns HAP parameters. + If `custom_params` are provided (e.g., from a notebook), they override the defaults. + """ + default_hap_params = { + "model_name_or_path": "ibm-granite/granite-guardian-hap-38m", + "annotation_column": "hap_score", + "doc_text_column": "contents", # Default column for documents + "inference_engine": "CPU", + "max_length": 512, + "batch_size": 128, + } + + # Merge custom_params into defaults, if provided + if custom_params: + default_hap_params.update(custom_params) + + print(f"Final HAP Parameters: {default_hap_params}") # Debugging log + return default_hap_params From 3d123a4131e3d00934f7cef42ee3a859002728eb Mon Sep 17 00:00:00 2001 From: Aisha Mohammed Farooq Darga Date: Fri, 29 Nov 2024 14:16:36 +0530 Subject: [PATCH 11/31] Update hap_local_python.py Signed-off-by: Aisha Mohammed Farooq Darga --- .../hap/python/src/hap_local_python.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/transforms/universal/hap/python/src/hap_local_python.py b/transforms/universal/hap/python/src/hap_local_python.py index 8e79dc583..2dfe8ab2c 100644 --- a/transforms/universal/hap/python/src/hap_local_python.py +++ b/transforms/universal/hap/python/src/hap_local_python.py @@ -18,8 +18,14 @@ from hap_transform_python import HAPPythonTransformConfiguration # create parameters -input_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../test-data/input")) -output_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../output")) +# input_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../test-data/input")) +# output_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../output")) + +input_folder = os.getenv('INPUT_FOLDER', os.path.abspath(os.path.join(os.path.dirname(__file__), "../test-data/input"))) +output_folder = os.getenv('OUTPUT_FOLDER', os.path.abspath(os.path.join(os.path.dirname(__file__), "../output")) +) +print("PATHS: ", input_folder, output_folder) + local_conf = { "input_folder": input_folder, "output_folder": output_folder, @@ -35,18 +41,18 @@ hap_params = { - "model_name_or_path": 'ibm-granite/granite-guardian-hap-38m', - "annotation_column": "hap_score", - "doc_text_column": "contents", - "inference_engine": "CPU", - "max_length": 512, - "batch_size": 128, + "model_name_or_path": os.getenv('MODEL_NAME_OR_PATH', 'ibm-granite/granite-guardian-hap-38m'), + "annotation_column": os.getenv('ANNOTATION_COLUMN', "hap_score"), + "doc_text_column": os.getenv('DOC_TEXT_COLUMN', "contents"), + "inference_engine": os.getenv('INFERENCE_ENGINE', "CPU"), + "max_length": int(os.getenv('MAX_LENGTH', 512)), + "batch_size": int(os.getenv('BATCH_SIZE', 128)), } - if __name__ == "__main__": # Set the simulated command line args sys.argv = ParamsUtils.dict_to_req(d=params | hap_params) + print("PARAMs: ", hap_params) # create launcher launcher = PythonTransformLauncher(runtime_config=HAPPythonTransformConfiguration()) # Launch the ray actor(s) to process the input From fedf0c38ede74e777ba84c52576481b85d724e7a Mon Sep 17 00:00:00 2001 From: SHAHROKH DAIJAVAD Date: Wed, 4 Dec 2024 10:48:14 -0800 Subject: [PATCH 12/31] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 716f3b0f2..b63ac2d5d 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ The matrix below shows the the combination of modules and supported runtimes. Al | [Web to Parquet](transforms/universal/web2parquet/README.md) | :white_check_mark: | | | | | **Universal (Code & Language)** | | | | | | [Exact dedup filter](transforms/universal/ededup/ray/README.md) | :white_check_mark: | :white_check_mark: | | :white_check_mark: | -| [Fuzzy dedup filter](transforms/universal/fdedup/ray/README.md) | | :white_check_mark: | | :white_check_mark: | +| [Fuzzy dedup filter](transforms/universal/fdedup/ray/README.md) | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | | [Unique ID annotation](transforms/universal/doc_id/ray/README.md) | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | | [Filter on annotations](transforms/universal/filter/python/README.md) | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | | [Profiler](transforms/universal/profiler/ray/README.md) | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | From 06599a502a819193461b8835bb52ab73ce77f047 Mon Sep 17 00:00:00 2001 From: Aisha Darga <37473748+AishaDarga@users.noreply.github.com> Date: Fri, 29 Nov 2024 14:19:28 +0530 Subject: [PATCH 13/31] Update hap_local_python.py Signed-off-by: Aisha Mohammed Farooq Darga --- transforms/universal/hap/python/src/hap_local_python.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/transforms/universal/hap/python/src/hap_local_python.py b/transforms/universal/hap/python/src/hap_local_python.py index 2dfe8ab2c..ca5beb144 100644 --- a/transforms/universal/hap/python/src/hap_local_python.py +++ b/transforms/universal/hap/python/src/hap_local_python.py @@ -17,14 +17,9 @@ from data_processing.utils import ParamsUtils from hap_transform_python import HAPPythonTransformConfiguration -# create parameters -# input_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../test-data/input")) -# output_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../output")) - input_folder = os.getenv('INPUT_FOLDER', os.path.abspath(os.path.join(os.path.dirname(__file__), "../test-data/input"))) output_folder = os.getenv('OUTPUT_FOLDER', os.path.abspath(os.path.join(os.path.dirname(__file__), "../output")) ) -print("PATHS: ", input_folder, output_folder) local_conf = { "input_folder": input_folder, @@ -39,7 +34,6 @@ "runtime_code_location": ParamsUtils.convert_to_ast(code_location), } - hap_params = { "model_name_or_path": os.getenv('MODEL_NAME_OR_PATH', 'ibm-granite/granite-guardian-hap-38m'), "annotation_column": os.getenv('ANNOTATION_COLUMN', "hap_score"), @@ -52,8 +46,7 @@ if __name__ == "__main__": # Set the simulated command line args sys.argv = ParamsUtils.dict_to_req(d=params | hap_params) - print("PARAMs: ", hap_params) # create launcher launcher = PythonTransformLauncher(runtime_config=HAPPythonTransformConfiguration()) # Launch the ray actor(s) to process the input - launcher.launch() \ No newline at end of file + launcher.launch() From 0bdc9a1f38a06313e31060734eeec708b2843afe Mon Sep 17 00:00:00 2001 From: Aisha Mohammed Farooq Darga Date: Wed, 4 Dec 2024 18:07:13 +0530 Subject: [PATCH 14/31] Colab compatible Signed-off-by: Aisha Mohammed Farooq Darga --- .../hap/generate_hap_score_csv.ipynb | 506 +++++++++++++++++- examples/notebooks/hap/requirements.txt | 4 + 2 files changed, 501 insertions(+), 9 deletions(-) create mode 100644 examples/notebooks/hap/requirements.txt diff --git a/examples/notebooks/hap/generate_hap_score_csv.ipynb b/examples/notebooks/hap/generate_hap_score_csv.ipynb index 497218c2e..a4717ac70 100644 --- a/examples/notebooks/hap/generate_hap_score_csv.ipynb +++ b/examples/notebooks/hap/generate_hap_score_csv.ipynb @@ -40,13 +40,501 @@ "4. View and analyze the results.\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Open this notebook in Google Colab" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Click link to open notebook in google colab: [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/IBM/data-prep-kit/blob/ed990c3445d0187062dbc3ca3913a46674c7a7e3/examples/notebooks/hap/generate_hap_score_csv.ipynb)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Install dependencies for Google Colab environment" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: data-prep-connector in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (0.2.3)\n", + "Requirement already satisfied: scrapy>=2.11.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-connector) (2.12.0)\n", + "Requirement already satisfied: pydantic>=2.8.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-connector) (2.10.2)\n", + "Requirement already satisfied: tldextract>=5.1.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-connector) (5.1.3)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic>=2.8.1->data-prep-connector) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.27.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic>=2.8.1->data-prep-connector) (2.27.1)\n", + "Requirement already satisfied: typing-extensions>=4.12.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic>=2.8.1->data-prep-connector) (4.12.2)\n", + "Requirement already satisfied: Twisted>=21.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.11.0)\n", + "Requirement already satisfied: cryptography>=37.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (44.0.0)\n", + "Requirement already satisfied: cssselect>=0.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.2.0)\n", + "Requirement already satisfied: itemloaders>=1.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.3.2)\n", + "Requirement already satisfied: parsel>=1.5.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.9.1)\n", + "Requirement already satisfied: pyOpenSSL>=22.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.3.0)\n", + "Requirement already satisfied: queuelib>=1.4.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.7.0)\n", + "Requirement already satisfied: service-identity>=18.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.2.0)\n", + "Requirement already satisfied: w3lib>=1.17.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (2.2.1)\n", + "Requirement already satisfied: zope.interface>=5.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (7.2)\n", + "Requirement already satisfied: protego>=0.1.15 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (0.3.1)\n", + "Requirement already satisfied: itemadapter>=0.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (0.10.0)\n", + "Requirement already satisfied: packaging in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.2)\n", + "Requirement already satisfied: lxml>=4.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (4.9.4)\n", + "Requirement already satisfied: defusedxml>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (0.7.1)\n", + "Requirement already satisfied: PyDispatcher>=2.0.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (2.0.7)\n", + "Requirement already satisfied: idna in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (3.10)\n", + "Requirement already satisfied: requests>=2.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (2.32.3)\n", + "Requirement already satisfied: requests-file>=1.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (2.1.0)\n", + "Requirement already satisfied: filelock>=3.0.8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (3.16.1)\n", + "Requirement already satisfied: cffi>=1.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from cryptography>=37.0.0->scrapy>=2.11.2->data-prep-connector) (1.17.1)\n", + "Requirement already satisfied: jmespath>=0.9.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from itemloaders>=1.0.1->scrapy>=2.11.2->data-prep-connector) (1.0.1)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.1.0->tldextract>=5.1.2->data-prep-connector) (3.4.0)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.1.0->tldextract>=5.1.2->data-prep-connector) (2.2.3)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.1.0->tldextract>=5.1.2->data-prep-connector) (2024.8.30)\n", + "Requirement already satisfied: attrs>=19.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from service-identity>=18.1.0->scrapy>=2.11.2->data-prep-connector) (24.2.0)\n", + "Requirement already satisfied: pyasn1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from service-identity>=18.1.0->scrapy>=2.11.2->data-prep-connector) (0.6.1)\n", + "Requirement already satisfied: pyasn1-modules in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from service-identity>=18.1.0->scrapy>=2.11.2->data-prep-connector) (0.4.1)\n", + "Requirement already satisfied: automat>=24.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (24.8.1)\n", + "Requirement already satisfied: constantly>=15.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (23.10.4)\n", + "Requirement already satisfied: hyperlink>=17.1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (21.0.0)\n", + "Requirement already satisfied: incremental>=24.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (24.7.2)\n", + "Requirement already satisfied: setuptools in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from zope.interface>=5.1.0->scrapy>=2.11.2->data-prep-connector) (75.1.0)\n", + "Requirement already satisfied: pycparser in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from cffi>=1.12->cryptography>=37.0.0->scrapy>=2.11.2->data-prep-connector) (2.22)\n", + "Requirement already satisfied: data-prep-toolkit==0.2.2.dev1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]==0.2.2.dev1) (0.2.2.dev1)\n", + "Requirement already satisfied: numpy<1.29.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.26.4)\n", + "Requirement already satisfied: pyarrow==16.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (16.1.0)\n", + "Requirement already satisfied: boto3==1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.34.69)\n", + "Collecting argparse (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1)\n", + " Using cached argparse-1.4.0-py2.py3-none-any.whl.metadata (2.8 kB)\n", + "Requirement already satisfied: mmh3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (4.1.0)\n", + "Requirement already satisfied: psutil in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (5.9.0)\n", + "Requirement already satisfied: ray==2.36.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.36.1)\n", + "Requirement already satisfied: fastapi>=0.110.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]==0.2.2.dev1) (0.115.5)\n", + "Requirement already satisfied: pillow>=10.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]==0.2.2.dev1) (10.4.0)\n", + "Requirement already satisfied: botocore<1.35.0,>=1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.34.162)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (0.10.4)\n", + "Requirement already satisfied: click>=7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (8.1.7)\n", + "Requirement already satisfied: filelock in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.16.1)\n", + "Requirement already satisfied: jsonschema in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (4.23.0)\n", + "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.1.0)\n", + "Requirement already satisfied: packaging in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (24.2)\n", + "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (5.29.0)\n", + "Requirement already satisfied: pyyaml in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (6.0.2)\n", + "Requirement already satisfied: aiosignal in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.3.1)\n", + "Requirement already satisfied: frozenlist in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.5.0)\n", + "Requirement already satisfied: requests in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.32.3)\n", + "Requirement already satisfied: aiohttp>=3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.11.8)\n", + "Requirement already satisfied: aiohttp-cors in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.7.0)\n", + "Requirement already satisfied: colorful in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.5.6)\n", + "Requirement already satisfied: py-spy>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.4.0)\n", + "Requirement already satisfied: opencensus in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.11.4)\n", + "Requirement already satisfied: pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.10.2)\n", + "Requirement already satisfied: prometheus-client>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.21.0)\n", + "Requirement already satisfied: smart-open in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (7.0.5)\n", + "Requirement already satisfied: virtualenv!=20.21.1,>=20.0.24 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (20.28.0)\n", + "Requirement already satisfied: grpcio>=1.42.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.68.0)\n", + "Requirement already satisfied: memray in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.14.0)\n", + "Requirement already satisfied: starlette<0.42.0,>=0.40.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (0.41.3)\n", + "Requirement already satisfied: typing-extensions>=4.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (4.12.2)\n", + "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.4.3)\n", + "Requirement already satisfied: attrs>=17.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (24.2.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (6.1.0)\n", + "Requirement already satisfied: propcache>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.2.0)\n", + "Requirement already satisfied: yarl<2.0,>=1.17.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.18.0)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from botocore<1.35.0,>=1.34.69->boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (2.9.0.post0)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from botocore<1.35.0,>=1.34.69->boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (2.2.3)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.27.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.27.1)\n", + "Requirement already satisfied: anyio<5,>=3.4.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from starlette<0.42.0,>=0.40.0->fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (4.6.2.post1)\n", + "Requirement already satisfied: distlib<1,>=0.3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.3.9)\n", + "Requirement already satisfied: platformdirs<5,>=3.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (4.3.6)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2024.10.1)\n", + "Requirement already satisfied: referencing>=0.28.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.35.1)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.21.0)\n", + "Requirement already satisfied: jinja2>=2.9 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.1.4)\n", + "Requirement already satisfied: rich>=11.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (13.9.4)\n", + "Requirement already satisfied: textual>=0.41.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.87.1)\n", + "Requirement already satisfied: opencensus-context>=0.1.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.1.3)\n", + "Requirement already satisfied: six~=1.16 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.16.0)\n", + "Requirement already satisfied: google-api-core<3.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.23.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.4.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.10)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2024.8.30)\n", + "Requirement already satisfied: wrapt in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from smart-open->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.17.0)\n", + "Requirement already satisfied: sniffio>=1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from anyio<5,>=3.4.0->starlette<0.42.0,>=0.40.0->fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (1.3.1)\n", + "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.66.0)\n", + "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.25.0)\n", + "Requirement already satisfied: google-auth<3.0.dev0,>=2.14.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.36.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jinja2>=2.9->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.1.5)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from rich>=11.2.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.0.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from rich>=11.2.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.18.0)\n", + "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (5.5.0)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.4.1)\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (4.9)\n", + "Requirement already satisfied: mdurl~=0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich>=11.2.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.1.2)\n", + "Requirement already satisfied: linkify-it-py<3,>=1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.0.3)\n", + "Requirement already satisfied: mdit-py-plugins in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.4.2)\n", + "Requirement already satisfied: uc-micro-py in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.0.3)\n", + "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.6.1)\n", + "Using cached argparse-1.4.0-py2.py3-none-any.whl (23 kB)\n", + "Installing collected packages: argparse\n", + "Successfully installed argparse-1.4.0\n", + "Requirement already satisfied: data-prep-toolkit-transforms==0.2.2.dev1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.2.dev1)\n", + "Requirement already satisfied: data-prep-toolkit>=0.2.2.dev1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.2.dev1)\n", + "Requirement already satisfied: bs4==0.0.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.0.2)\n", + "Requirement already satisfied: transformers==4.38.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.38.2)\n", + "Requirement already satisfied: parameterized in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.0)\n", + "Requirement already satisfied: pandas in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.2.2)\n", + "Requirement already satisfied: docling-core==1.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0)\n", + "Requirement already satisfied: llama-index-core<0.12.0,>=0.11.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.11.23)\n", + "Requirement already satisfied: fasttext==0.9.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.2)\n", + "Requirement already satisfied: langcodes==3.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3.0)\n", + "Requirement already satisfied: huggingface-hub<1.0.0,>=0.21.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.26.3)\n", + "Requirement already satisfied: numpy==1.26.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.26.4)\n", + "Requirement already satisfied: sentence-transformers==3.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.0.1)\n", + "Requirement already satisfied: docling-ibm-models==1.1.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.7)\n", + "Requirement already satisfied: deepsearch-glm==0.21.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.0)\n", + "Requirement already satisfied: docling==1.11.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.11.0)\n", + "Requirement already satisfied: filetype<2.0.0,>=1.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.2.0)\n", + "Requirement already satisfied: nltk==3.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.9.1)\n", + "Requirement already satisfied: torch==2.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.4.1)\n", + "Requirement already satisfied: mmh3>=4.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.1.0)\n", + "Requirement already satisfied: xxhash==3.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.4.1)\n", + "Requirement already satisfied: duckdb>=0.10.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.3)\n", + "Requirement already satisfied: networkx==3.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3)\n", + "Requirement already satisfied: colorlog==6.8.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (6.8.2)\n", + "Requirement already satisfied: func-timeout==4.3.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.3.5)\n", + "Requirement already satisfied: emerge-viz==2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.0)\n", + "Requirement already satisfied: beautifulsoup4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from bs4==0.0.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.12.3)\n", + "Requirement already satisfied: pyarrow==16.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (16.1.0)\n", + "Requirement already satisfied: boto3==1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.34.69)\n", + "Collecting argparse (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1)\n", + " Using cached argparse-1.4.0-py2.py3-none-any.whl.metadata (2.8 kB)\n", + "Requirement already satisfied: psutil in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.9.0)\n", + "Requirement already satisfied: docutils!=0.21 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.2)\n", + "Requirement already satisfied: matplotlib<4.0.0,>=3.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.9.2)\n", + "Requirement already satisfied: netwulf<0.2.0,>=0.1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.5)\n", + "Requirement already satisfied: numerize<0.13,>=0.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.12)\n", + "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.1)\n", + "Requirement already satisfied: rich<14.0.0,>=13.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (13.9.4)\n", + "Requirement already satisfied: tabulate>=0.8.9 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.0)\n", + "Requirement already satisfied: tqdm<5.0.0,>=4.64.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.67.1)\n", + "Requirement already satisfied: certifi>=2024.7.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.8.30)\n", + "Requirement already satisfied: docling-parse<2.0.0,>=1.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.6.2)\n", + "Requirement already satisfied: easyocr<2.0,>=1.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.7.2)\n", + "Requirement already satisfied: pydantic<3.0.0,>=2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.10.2)\n", + "Requirement already satisfied: pydantic-settings<3.0.0,>=2.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.6.1)\n", + "Requirement already satisfied: pypdfium2<5.0.0,>=4.30.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.30.0)\n", + "Requirement already satisfied: requests<3.0.0,>=2.32.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.32.3)\n", + "Requirement already satisfied: rtree<2.0.0,>=1.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0)\n", + "Requirement already satisfied: scipy<2.0.0,>=1.14.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.14.1)\n", + "Requirement already satisfied: json-schema-for-humans<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.4)\n", + "Requirement already satisfied: jsonref<2.0.0,>=1.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.0)\n", + "Requirement already satisfied: jsonschema<5.0.0,>=4.16.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.23.0)\n", + "Requirement already satisfied: pyproject-toml<0.0.11,>=0.0.10 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.0.10)\n", + "Requirement already satisfied: Pillow<11.0.0,>=10.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (10.4.0)\n", + "Requirement already satisfied: jsonlines<4.0.0,>=3.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.0)\n", + "Requirement already satisfied: lxml<5.0.0,>=4.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.9.4)\n", + "Requirement already satisfied: mean_average_precision<2022.0.0.0,>=2021.4.26.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2021.4.26.0)\n", + "Requirement already satisfied: onnxruntime<2.0.0,>=1.16.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.20.1)\n", + "Requirement already satisfied: opencv-python-headless<5.0.0.0,>=4.9.0.80 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.10.0.84)\n", + "Requirement already satisfied: torchvision<1,>=0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.19.1)\n", + "Requirement already satisfied: wheel in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.44.0)\n", + "Requirement already satisfied: autopep8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.3.1)\n", + "Requirement already satisfied: coloredlogs in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (15.0.1)\n", + "Requirement already satisfied: interrogate in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.7.0)\n", + "Requirement already satisfied: scikit-learn in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.5.2)\n", + "Requirement already satisfied: prettytable in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.12.0)\n", + "Requirement already satisfied: py in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.11.0)\n", + "Requirement already satisfied: pycodestyle in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.12.1)\n", + "Requirement already satisfied: pylint in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3.1)\n", + "Requirement already satisfied: pyparsing in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.2.0)\n", + "Requirement already satisfied: python-louvain in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.16)\n", + "Requirement already satisfied: PyYAML in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (6.0.2)\n", + "Requirement already satisfied: PyDriller in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.7)\n", + "Requirement already satisfied: pyperclip in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.9.0)\n", + "Requirement already satisfied: pybind11>=2.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fasttext==0.9.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.13.6)\n", + "Requirement already satisfied: setuptools>=0.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fasttext==0.9.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (75.1.0)\n", + "Requirement already satisfied: click in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (8.1.7)\n", + "Requirement already satisfied: joblib in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.4.2)\n", + "Requirement already satisfied: regex>=2021.8.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.11.6)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.2)\n", + "Requirement already satisfied: filelock in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.16.1)\n", + "Requirement already satisfied: typing-extensions>=4.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.12.2)\n", + "Requirement already satisfied: sympy in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.13.3)\n", + "Requirement already satisfied: jinja2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.4)\n", + "Requirement already satisfied: fsspec in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.10.0)\n", + "Requirement already satisfied: packaging>=20.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (24.2)\n", + "Requirement already satisfied: tokenizers<0.19,>=0.14 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.15.2)\n", + "Requirement already satisfied: safetensors>=0.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.5)\n", + "Requirement already satisfied: botocore<1.35.0,>=1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.34.162)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.10.4)\n", + "Requirement already satisfied: ray==2.36.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.36.1)\n", + "Requirement already satisfied: fastapi>=0.110.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.115.5)\n", + "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.0)\n", + "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.29.0)\n", + "Requirement already satisfied: aiosignal in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.1)\n", + "Requirement already satisfied: frozenlist in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.5.0)\n", + "Requirement already satisfied: aiohttp>=3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.11.8)\n", + "Requirement already satisfied: aiohttp-cors in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.7.0)\n", + "Requirement already satisfied: colorful in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.5.6)\n", + "Requirement already satisfied: py-spy>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.0)\n", + "Requirement already satisfied: opencensus in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.11.4)\n", + "Requirement already satisfied: prometheus-client>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.0)\n", + "Requirement already satisfied: smart-open in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (7.0.5)\n", + "Requirement already satisfied: virtualenv!=20.21.1,>=20.0.24 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (20.28.0)\n", + "Requirement already satisfied: grpcio>=1.42.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.68.0)\n", + "Requirement already satisfied: memray in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.14.0)\n", + "Requirement already satisfied: SQLAlchemy>=1.4.49 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.36)\n", + "Requirement already satisfied: dataclasses-json in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.6.7)\n", + "Requirement already satisfied: deprecated>=1.2.9.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.2.15)\n", + "Requirement already satisfied: dirtyjson<2.0.0,>=1.0.8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.8)\n", + "Requirement already satisfied: httpx in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.28.0)\n", + "Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.6.0)\n", + "Requirement already satisfied: tenacity!=8.4.0,<9.0.0,>=8.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (8.5.0)\n", + "Requirement already satisfied: tiktoken>=0.3.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.8.0)\n", + "Requirement already satisfied: typing-inspect>=0.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.0)\n", + "Requirement already satisfied: wrapt in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.17.0)\n", + "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.4.3)\n", + "Requirement already satisfied: attrs>=17.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (24.2.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (6.1.0)\n", + "Requirement already satisfied: propcache>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.0)\n", + "Requirement already satisfied: yarl<2.0,>=1.17.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.18.0)\n", + "Requirement already satisfied: scikit-image in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.24.0)\n", + "Requirement already satisfied: python-bidi in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.6.3)\n", + "Requirement already satisfied: Shapely in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.6)\n", + "Requirement already satisfied: pyclipper in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0.post6)\n", + "Requirement already satisfied: ninja in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.11.1.2)\n", + "Requirement already satisfied: starlette<0.42.0,>=0.40.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fastapi>=0.110.2->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.41.3)\n", + "Requirement already satisfied: MarkupSafe<3.0.0,>=2.1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.1.5)\n", + "Requirement already satisfied: Pygments<3.0.0,>=2.18.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.18.0)\n", + "Requirement already satisfied: htmlmin2<0.2.0,>=0.1.13 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.13)\n", + "Requirement already satisfied: markdown2<3.0.0,>=2.5.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.5.1)\n", + "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from dataclasses-json->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.23.1)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.16.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.10.1)\n", + "Requirement already satisfied: referencing>=0.28.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.16.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.35.1)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.16.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.0)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.1)\n", + "Requirement already satisfied: cycler>=0.10 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.55.0)\n", + "Requirement already satisfied: kiwisolver>=1.3.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.4.7)\n", + "Requirement already satisfied: simplejson>=3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from netwulf<0.2.0,>=0.1.5->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.19.3)\n", + "Requirement already satisfied: flatbuffers in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from onnxruntime<2.0.0,>=1.16.2->docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (24.3.25)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.27.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.27.1)\n", + "Requirement already satisfied: toml in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pyproject-toml<0.0.11,>=0.0.10->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.10.2)\n", + "Requirement already satisfied: six>=1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.16.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests<3.0.0,>=2.32.3->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.4.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests<3.0.0,>=2.32.3->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.10)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests<3.0.0,>=2.32.3->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.2.3)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from rich<14.0.0,>=13.7.0->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.0.0)\n", + "Requirement already satisfied: greenlet!=0.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.1)\n", + "Requirement already satisfied: mypy-extensions>=0.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from typing-inspect>=0.8.0->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.0)\n", + "Requirement already satisfied: soupsieve>1.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from beautifulsoup4->bs4==0.0.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.6)\n", + "Requirement already satisfied: humanfriendly>=9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from coloredlogs->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (10.0)\n", + "Requirement already satisfied: anyio in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.6.2.post1)\n", + "Requirement already satisfied: httpcore==1.* in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.7)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpcore==1.*->httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.14.0)\n", + "Requirement already satisfied: colorama in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from interrogate->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.6)\n", + "Requirement already satisfied: wcwidth in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from prettytable->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.13)\n", + "Requirement already satisfied: gitpython in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.43)\n", + "Requirement already satisfied: types-pytz in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.2.0.20241003)\n", + "Requirement already satisfied: lizard in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.17.13)\n", + "Requirement already satisfied: platformdirs>=2.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.3.6)\n", + "Requirement already satisfied: astroid<=3.4.0-dev0,>=3.3.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3.5)\n", + "Requirement already satisfied: isort!=5.13.0,<6,>=4.2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.13.2)\n", + "Requirement already satisfied: mccabe<0.8,>=0.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.7.0)\n", + "Requirement already satisfied: tomlkit>=0.10.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.13.2)\n", + "Requirement already satisfied: dill>=0.3.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.3.9)\n", + "Requirement already satisfied: threadpoolctl>=3.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-learn->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.5.0)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from sympy->torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0)\n", + "Requirement already satisfied: mdurl~=0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich<14.0.0,>=13.7.0->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.2)\n", + "Requirement already satisfied: sniffio>=1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from anyio->httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.1)\n", + "Requirement already satisfied: distlib<1,>=0.3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.3.9)\n", + "Requirement already satisfied: gitdb<5,>=4.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from gitpython->PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.0.11)\n", + "Requirement already satisfied: textual>=0.41.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.87.1)\n", + "Requirement already satisfied: opencensus-context>=0.1.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.3)\n", + "Requirement already satisfied: google-api-core<3.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.23.0)\n", + "Requirement already satisfied: imageio>=2.33 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-image->easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.36.1)\n", + "Requirement already satisfied: tifffile>=2022.8.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-image->easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.9.20)\n", + "Requirement already satisfied: lazy-loader>=0.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-image->easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4)\n", + "Requirement already satisfied: smmap<6,>=3.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from gitdb<5,>=4.0.1->gitpython->PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.0.1)\n", + "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.66.0)\n", + "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.25.0)\n", + "Requirement already satisfied: google-auth<3.0.dev0,>=2.14.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.36.0)\n", + "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.5.0)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.1)\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.9)\n", + "Requirement already satisfied: linkify-it-py<3,>=1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.3)\n", + "Requirement already satisfied: mdit-py-plugins in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.2)\n", + "Requirement already satisfied: uc-micro-py in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.3)\n", + "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.6.1)\n", + "Using cached argparse-1.4.0-py2.py3-none-any.whl (23 kB)\n", + "Installing collected packages: argparse\n", + "Successfully installed argparse-1.4.0\n", + "Requirement already satisfied: jupyterlab in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (4.3.2)\n", + "Requirement already satisfied: ipykernel in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (6.29.5)\n", + "Requirement already satisfied: ipywidgets in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (8.1.5)\n", + "Requirement already satisfied: async-lru>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (2.0.4)\n", + "Requirement already satisfied: httpx~=0.28.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (0.28.0)\n", + "Requirement already satisfied: jinja2>=3.0.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (3.1.4)\n", + "Requirement already satisfied: jupyter-core in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (5.7.2)\n", + "Requirement already satisfied: jupyter-lsp>=2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (2.2.5)\n", + "Requirement already satisfied: jupyter-server<3,>=2.4.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (2.14.2)\n", + "Requirement already satisfied: jupyterlab-server<3,>=2.27.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (2.27.3)\n", + "Requirement already satisfied: notebook-shim>=0.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (0.2.4)\n", + "Requirement already satisfied: packaging in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (24.2)\n", + "Requirement already satisfied: setuptools>=40.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (75.1.0)\n", + "Requirement already satisfied: tornado>=6.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (6.4.1)\n", + "Requirement already satisfied: traitlets in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (5.14.3)\n", + "Requirement already satisfied: appnope in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipykernel) (0.1.4)\n", + "Requirement already satisfied: comm>=0.1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipykernel) (0.2.2)\n", + "Requirement already satisfied: debugpy>=1.6.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipykernel) (1.6.7)\n", + "Requirement already satisfied: ipython>=7.23.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipykernel) (8.29.0)\n", + "Requirement already satisfied: jupyter-client>=6.1.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipykernel) (8.6.3)\n", + "Requirement already satisfied: matplotlib-inline>=0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipykernel) (0.1.7)\n", + "Requirement already satisfied: nest-asyncio in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipykernel) (1.6.0)\n", + "Requirement already satisfied: psutil in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipykernel) (5.9.0)\n", + "Requirement already satisfied: pyzmq>=24 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipykernel) (25.1.2)\n", + "Requirement already satisfied: widgetsnbextension~=4.0.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipywidgets) (4.0.13)\n", + "Requirement already satisfied: jupyterlab-widgets~=3.0.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipywidgets) (3.0.13)\n", + "Requirement already satisfied: anyio in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpx~=0.28.0->jupyterlab) (4.6.2.post1)\n", + "Requirement already satisfied: certifi in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpx~=0.28.0->jupyterlab) (2024.8.30)\n", + "Requirement already satisfied: httpcore==1.* in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpx~=0.28.0->jupyterlab) (1.0.7)\n", + "Requirement already satisfied: idna in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpx~=0.28.0->jupyterlab) (3.10)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpcore==1.*->httpx~=0.28.0->jupyterlab) (0.14.0)\n", + "Requirement already satisfied: decorator in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel) (5.1.1)\n", + "Requirement already satisfied: jedi>=0.16 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel) (0.19.2)\n", + "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel) (3.0.48)\n", + "Requirement already satisfied: pygments>=2.4.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel) (2.18.0)\n", + "Requirement already satisfied: stack-data in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel) (0.6.2)\n", + "Requirement already satisfied: typing-extensions>=4.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel) (4.12.2)\n", + "Requirement already satisfied: pexpect>4.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel) (4.9.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jinja2>=3.0.3->jupyterlab) (2.1.5)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-client>=6.1.12->ipykernel) (2.9.0.post0)\n", + "Requirement already satisfied: platformdirs>=2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-core->jupyterlab) (4.3.6)\n", + "Requirement already satisfied: argon2-cffi>=21.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (23.1.0)\n", + "Requirement already satisfied: jupyter-events>=0.9.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.10.0)\n", + "Requirement already satisfied: jupyter-server-terminals>=0.4.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.5.3)\n", + "Requirement already satisfied: nbconvert>=6.4.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (7.16.4)\n", + "Requirement already satisfied: nbformat>=5.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (5.10.4)\n", + "Requirement already satisfied: overrides>=5.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (7.7.0)\n", + "Requirement already satisfied: prometheus-client>=0.9 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.21.0)\n", + "Requirement already satisfied: send2trash>=1.8.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (1.8.3)\n", + "Requirement already satisfied: terminado>=0.8.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.18.1)\n", + "Requirement already satisfied: websocket-client>=1.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (1.8.0)\n", + "Requirement already satisfied: babel>=2.10 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.27.1->jupyterlab) (2.16.0)\n", + "Requirement already satisfied: json5>=0.9.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.27.1->jupyterlab) (0.10.0)\n", + "Requirement already satisfied: jsonschema>=4.18.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.27.1->jupyterlab) (4.23.0)\n", + "Requirement already satisfied: requests>=2.31 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.27.1->jupyterlab) (2.32.3)\n", + "Requirement already satisfied: sniffio>=1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from anyio->httpx~=0.28.0->jupyterlab) (1.3.1)\n", + "Requirement already satisfied: argon2-cffi-bindings in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from argon2-cffi>=21.1->jupyter-server<3,>=2.4.0->jupyterlab) (21.2.0)\n", + "Requirement already satisfied: parso<0.9.0,>=0.8.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jedi>=0.16->ipython>=7.23.1->ipykernel) (0.8.4)\n", + "Requirement already satisfied: attrs>=22.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.27.1->jupyterlab) (24.2.0)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.27.1->jupyterlab) (2024.10.1)\n", + "Requirement already satisfied: referencing>=0.28.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.27.1->jupyterlab) (0.35.1)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.27.1->jupyterlab) (0.21.0)\n", + "Requirement already satisfied: python-json-logger>=2.0.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (2.0.7)\n", + "Requirement already satisfied: pyyaml>=5.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (6.0.2)\n", + "Requirement already satisfied: rfc3339-validator in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (0.1.4)\n", + "Requirement already satisfied: rfc3986-validator>=0.1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (0.1.1)\n", + "Requirement already satisfied: beautifulsoup4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (4.12.3)\n", + "Requirement already satisfied: bleach!=5.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (6.2.0)\n", + "Requirement already satisfied: defusedxml in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.7.1)\n", + "Requirement already satisfied: jupyterlab-pygments in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.3.0)\n", + "Requirement already satisfied: mistune<4,>=2.0.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (3.0.2)\n", + "Requirement already satisfied: nbclient>=0.5.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.10.1)\n", + "Requirement already satisfied: pandocfilters>=1.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (1.5.1)\n", + "Requirement already satisfied: tinycss2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (1.4.0)\n", + "Requirement already satisfied: fastjsonschema>=2.15 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nbformat>=5.3.0->jupyter-server<3,>=2.4.0->jupyterlab) (2.21.1)\n", + "Requirement already satisfied: ptyprocess>=0.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pexpect>4.3->ipython>=7.23.1->ipykernel) (0.7.0)\n", + "Requirement already satisfied: wcwidth in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython>=7.23.1->ipykernel) (0.2.13)\n", + "Requirement already satisfied: six>=1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from python-dateutil>=2.8.2->jupyter-client>=6.1.12->ipykernel) (1.16.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.31->jupyterlab-server<3,>=2.27.1->jupyterlab) (3.4.0)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.31->jupyterlab-server<3,>=2.27.1->jupyterlab) (2.2.3)\n", + "Requirement already satisfied: executing>=1.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from stack-data->ipython>=7.23.1->ipykernel) (2.1.0)\n", + "Requirement already satisfied: asttokens>=2.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from stack-data->ipython>=7.23.1->ipykernel) (2.4.1)\n", + "Requirement already satisfied: pure-eval in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from stack-data->ipython>=7.23.1->ipykernel) (0.2.3)\n", + "Requirement already satisfied: webencodings in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from bleach!=5.0.0->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.5.1)\n", + "Requirement already satisfied: fqdn in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.5.1)\n", + "Requirement already satisfied: isoduration in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (20.11.0)\n", + "Requirement already satisfied: jsonpointer>1.13 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (3.0.0)\n", + "Requirement already satisfied: uri-template in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.3.0)\n", + "Requirement already satisfied: webcolors>=24.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (24.11.1)\n", + "Requirement already satisfied: cffi>=1.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from argon2-cffi-bindings->argon2-cffi>=21.1->jupyter-server<3,>=2.4.0->jupyterlab) (1.17.1)\n", + "Requirement already satisfied: soupsieve>1.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from beautifulsoup4->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (2.6)\n", + "Requirement already satisfied: pycparser in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi>=21.1->jupyter-server<3,>=2.4.0->jupyterlab) (2.22)\n", + "Requirement already satisfied: arrow>=0.15.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from isoduration->jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.3.0)\n", + "Requirement already satisfied: types-python-dateutil>=2.8.10 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from arrow>=0.15.0->isoduration->jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (2.9.0.20241003)\n", + "Requirement already satisfied: nltk==3.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from -r requirements.txt (line 1)) (3.9.1)\n", + "Requirement already satisfied: transformers==4.38.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from -r requirements.txt (line 2)) (4.38.2)\n", + "Requirement already satisfied: torch<=2.4.1,>=2.2.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from -r requirements.txt (line 3)) (2.4.1)\n", + "Requirement already satisfied: pandas==2.2.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from -r requirements.txt (line 4)) (2.2.2)\n", + "Requirement already satisfied: click in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->-r requirements.txt (line 1)) (8.1.7)\n", + "Requirement already satisfied: joblib in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->-r requirements.txt (line 1)) (1.4.2)\n", + "Requirement already satisfied: regex>=2021.8.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->-r requirements.txt (line 1)) (2024.11.6)\n", + "Requirement already satisfied: tqdm in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->-r requirements.txt (line 1)) (4.67.1)\n", + "Requirement already satisfied: filelock in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (3.16.1)\n", + "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (0.26.3)\n", + "Requirement already satisfied: numpy>=1.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (1.26.4)\n", + "Requirement already satisfied: packaging>=20.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (24.2)\n", + "Requirement already satisfied: pyyaml>=5.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (6.0.2)\n", + "Requirement already satisfied: requests in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (2.32.3)\n", + "Requirement already satisfied: tokenizers<0.19,>=0.14 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (0.15.2)\n", + "Requirement already satisfied: safetensors>=0.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (0.4.5)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas==2.2.2->-r requirements.txt (line 4)) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas==2.2.2->-r requirements.txt (line 4)) (2024.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas==2.2.2->-r requirements.txt (line 4)) (2024.2)\n", + "Requirement already satisfied: typing-extensions>=4.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (4.12.2)\n", + "Requirement already satisfied: sympy in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (1.13.3)\n", + "Requirement already satisfied: networkx in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (3.3)\n", + "Requirement already satisfied: jinja2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (3.1.4)\n", + "Requirement already satisfied: fsspec in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (2024.10.0)\n", + "Requirement already satisfied: six>=1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas==2.2.2->-r requirements.txt (line 4)) (1.16.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jinja2->torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (2.1.5)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->transformers==4.38.2->-r requirements.txt (line 2)) (3.4.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->transformers==4.38.2->-r requirements.txt (line 2)) (3.10)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->transformers==4.38.2->-r requirements.txt (line 2)) (2.2.3)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->transformers==4.38.2->-r requirements.txt (line 2)) (2024.8.30)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from sympy->torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (1.3.0)\n" + ] + } + ], + "source": [ + "! pip install data-prep-connector\n", + "! pip install 'data-prep-toolkit[ray]==0.2.2.dev1'\n", + "! pip install 'data-prep-toolkit-transforms[ray,all]==0.2.2.dev1'\n", + "! pip install -r requirements.txt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Import necessary libraries" + ] + }, { "cell_type": "code", - "execution_count": 491, + "execution_count": 500, "metadata": {}, "outputs": [], "source": [ - "# Import necessary libraries\n", "import os\n", "import pandas as pd\n", "import subprocess\n", @@ -64,7 +552,7 @@ }, { "cell_type": "code", - "execution_count": 492, + "execution_count": 501, "metadata": {}, "outputs": [], "source": [ @@ -75,7 +563,7 @@ }, { "cell_type": "code", - "execution_count": 493, + "execution_count": 502, "metadata": {}, "outputs": [ { @@ -112,7 +600,7 @@ }, { "cell_type": "code", - "execution_count": 494, + "execution_count": 503, "metadata": {}, "outputs": [ { @@ -148,7 +636,7 @@ }, { "cell_type": "code", - "execution_count": 495, + "execution_count": 504, "metadata": {}, "outputs": [ { @@ -175,7 +663,7 @@ }, { "cell_type": "code", - "execution_count": 496, + "execution_count": 505, "metadata": {}, "outputs": [ { @@ -223,7 +711,7 @@ }, { "cell_type": "code", - "execution_count": 497, + "execution_count": 506, "metadata": {}, "outputs": [ { @@ -289,7 +777,7 @@ }, { "cell_type": "code", - "execution_count": 498, + "execution_count": 507, "metadata": {}, "outputs": [ { diff --git a/examples/notebooks/hap/requirements.txt b/examples/notebooks/hap/requirements.txt new file mode 100644 index 000000000..079767b7a --- /dev/null +++ b/examples/notebooks/hap/requirements.txt @@ -0,0 +1,4 @@ +nltk==3.9.1 +transformers==4.38.2 +torch>=2.2.2,<=2.4.1 +pandas==2.2.2 From dcdc2f75434a8b2121fc37685e9301e050edf509 Mon Sep 17 00:00:00 2001 From: Aisha Mohammed Farooq Darga Date: Wed, 4 Dec 2024 18:09:34 +0530 Subject: [PATCH 15/31] Removed the output cells Signed-off-by: Aisha Mohammed Farooq Darga --- .../hap/generate_hap_score_csv.ipynb | 649 +----------------- 1 file changed, 15 insertions(+), 634 deletions(-) diff --git a/examples/notebooks/hap/generate_hap_score_csv.ipynb b/examples/notebooks/hap/generate_hap_score_csv.ipynb index a4717ac70..58c44b94d 100644 --- a/examples/notebooks/hap/generate_hap_score_csv.ipynb +++ b/examples/notebooks/hap/generate_hap_score_csv.ipynb @@ -65,456 +65,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: data-prep-connector in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (0.2.3)\n", - "Requirement already satisfied: scrapy>=2.11.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-connector) (2.12.0)\n", - "Requirement already satisfied: pydantic>=2.8.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-connector) (2.10.2)\n", - "Requirement already satisfied: tldextract>=5.1.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-connector) (5.1.3)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic>=2.8.1->data-prep-connector) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.27.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic>=2.8.1->data-prep-connector) (2.27.1)\n", - "Requirement already satisfied: typing-extensions>=4.12.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic>=2.8.1->data-prep-connector) (4.12.2)\n", - "Requirement already satisfied: Twisted>=21.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.11.0)\n", - "Requirement already satisfied: cryptography>=37.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (44.0.0)\n", - "Requirement already satisfied: cssselect>=0.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.2.0)\n", - "Requirement already satisfied: itemloaders>=1.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.3.2)\n", - "Requirement already satisfied: parsel>=1.5.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.9.1)\n", - "Requirement already satisfied: pyOpenSSL>=22.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.3.0)\n", - "Requirement already satisfied: queuelib>=1.4.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.7.0)\n", - "Requirement already satisfied: service-identity>=18.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.2.0)\n", - "Requirement already satisfied: w3lib>=1.17.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (2.2.1)\n", - "Requirement already satisfied: zope.interface>=5.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (7.2)\n", - "Requirement already satisfied: protego>=0.1.15 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (0.3.1)\n", - "Requirement already satisfied: itemadapter>=0.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (0.10.0)\n", - "Requirement already satisfied: packaging in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.2)\n", - "Requirement already satisfied: lxml>=4.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (4.9.4)\n", - "Requirement already satisfied: defusedxml>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (0.7.1)\n", - "Requirement already satisfied: PyDispatcher>=2.0.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (2.0.7)\n", - "Requirement already satisfied: idna in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (3.10)\n", - "Requirement already satisfied: requests>=2.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (2.32.3)\n", - "Requirement already satisfied: requests-file>=1.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (2.1.0)\n", - "Requirement already satisfied: filelock>=3.0.8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (3.16.1)\n", - "Requirement already satisfied: cffi>=1.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from cryptography>=37.0.0->scrapy>=2.11.2->data-prep-connector) (1.17.1)\n", - "Requirement already satisfied: jmespath>=0.9.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from itemloaders>=1.0.1->scrapy>=2.11.2->data-prep-connector) (1.0.1)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.1.0->tldextract>=5.1.2->data-prep-connector) (3.4.0)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.1.0->tldextract>=5.1.2->data-prep-connector) (2.2.3)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.1.0->tldextract>=5.1.2->data-prep-connector) (2024.8.30)\n", - "Requirement already satisfied: attrs>=19.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from service-identity>=18.1.0->scrapy>=2.11.2->data-prep-connector) (24.2.0)\n", - "Requirement already satisfied: pyasn1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from service-identity>=18.1.0->scrapy>=2.11.2->data-prep-connector) (0.6.1)\n", - "Requirement already satisfied: pyasn1-modules in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from service-identity>=18.1.0->scrapy>=2.11.2->data-prep-connector) (0.4.1)\n", - "Requirement already satisfied: automat>=24.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (24.8.1)\n", - "Requirement already satisfied: constantly>=15.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (23.10.4)\n", - "Requirement already satisfied: hyperlink>=17.1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (21.0.0)\n", - "Requirement already satisfied: incremental>=24.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (24.7.2)\n", - "Requirement already satisfied: setuptools in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from zope.interface>=5.1.0->scrapy>=2.11.2->data-prep-connector) (75.1.0)\n", - "Requirement already satisfied: pycparser in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from cffi>=1.12->cryptography>=37.0.0->scrapy>=2.11.2->data-prep-connector) (2.22)\n", - "Requirement already satisfied: data-prep-toolkit==0.2.2.dev1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]==0.2.2.dev1) (0.2.2.dev1)\n", - "Requirement already satisfied: numpy<1.29.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.26.4)\n", - "Requirement already satisfied: pyarrow==16.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (16.1.0)\n", - "Requirement already satisfied: boto3==1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.34.69)\n", - "Collecting argparse (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1)\n", - " Using cached argparse-1.4.0-py2.py3-none-any.whl.metadata (2.8 kB)\n", - "Requirement already satisfied: mmh3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (4.1.0)\n", - "Requirement already satisfied: psutil in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (5.9.0)\n", - "Requirement already satisfied: ray==2.36.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.36.1)\n", - "Requirement already satisfied: fastapi>=0.110.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]==0.2.2.dev1) (0.115.5)\n", - "Requirement already satisfied: pillow>=10.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]==0.2.2.dev1) (10.4.0)\n", - "Requirement already satisfied: botocore<1.35.0,>=1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.34.162)\n", - "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.0.1)\n", - "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (0.10.4)\n", - "Requirement already satisfied: click>=7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (8.1.7)\n", - "Requirement already satisfied: filelock in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.16.1)\n", - "Requirement already satisfied: jsonschema in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (4.23.0)\n", - "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.1.0)\n", - "Requirement already satisfied: packaging in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (24.2)\n", - "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (5.29.0)\n", - "Requirement already satisfied: pyyaml in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (6.0.2)\n", - "Requirement already satisfied: aiosignal in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.3.1)\n", - "Requirement already satisfied: frozenlist in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.5.0)\n", - "Requirement already satisfied: requests in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.32.3)\n", - "Requirement already satisfied: aiohttp>=3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.11.8)\n", - "Requirement already satisfied: aiohttp-cors in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.7.0)\n", - "Requirement already satisfied: colorful in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.5.6)\n", - "Requirement already satisfied: py-spy>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.4.0)\n", - "Requirement already satisfied: opencensus in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.11.4)\n", - "Requirement already satisfied: pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.10.2)\n", - "Requirement already satisfied: prometheus-client>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.21.0)\n", - "Requirement already satisfied: smart-open in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (7.0.5)\n", - "Requirement already satisfied: virtualenv!=20.21.1,>=20.0.24 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (20.28.0)\n", - "Requirement already satisfied: grpcio>=1.42.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.68.0)\n", - "Requirement already satisfied: memray in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.14.0)\n", - "Requirement already satisfied: starlette<0.42.0,>=0.40.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (0.41.3)\n", - "Requirement already satisfied: typing-extensions>=4.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (4.12.2)\n", - "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.4.3)\n", - "Requirement already satisfied: attrs>=17.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (24.2.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (6.1.0)\n", - "Requirement already satisfied: propcache>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.2.0)\n", - "Requirement already satisfied: yarl<2.0,>=1.17.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.18.0)\n", - "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from botocore<1.35.0,>=1.34.69->boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (2.9.0.post0)\n", - "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from botocore<1.35.0,>=1.34.69->boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (2.2.3)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.27.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.27.1)\n", - "Requirement already satisfied: anyio<5,>=3.4.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from starlette<0.42.0,>=0.40.0->fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (4.6.2.post1)\n", - "Requirement already satisfied: distlib<1,>=0.3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.3.9)\n", - "Requirement already satisfied: platformdirs<5,>=3.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (4.3.6)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2024.10.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.35.1)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.21.0)\n", - "Requirement already satisfied: jinja2>=2.9 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.1.4)\n", - "Requirement already satisfied: rich>=11.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (13.9.4)\n", - "Requirement already satisfied: textual>=0.41.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.87.1)\n", - "Requirement already satisfied: opencensus-context>=0.1.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.1.3)\n", - "Requirement already satisfied: six~=1.16 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.16.0)\n", - "Requirement already satisfied: google-api-core<3.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.23.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.4.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.10)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2024.8.30)\n", - "Requirement already satisfied: wrapt in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from smart-open->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.17.0)\n", - "Requirement already satisfied: sniffio>=1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from anyio<5,>=3.4.0->starlette<0.42.0,>=0.40.0->fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (1.3.1)\n", - "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.66.0)\n", - "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.25.0)\n", - "Requirement already satisfied: google-auth<3.0.dev0,>=2.14.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.36.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jinja2>=2.9->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.1.5)\n", - "Requirement already satisfied: markdown-it-py>=2.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from rich>=11.2.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.0.0)\n", - "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from rich>=11.2.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.18.0)\n", - "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (5.5.0)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.4.1)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (4.9)\n", - "Requirement already satisfied: mdurl~=0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich>=11.2.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.1.2)\n", - "Requirement already satisfied: linkify-it-py<3,>=1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.0.3)\n", - "Requirement already satisfied: mdit-py-plugins in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.4.2)\n", - "Requirement already satisfied: uc-micro-py in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.0.3)\n", - "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.6.1)\n", - "Using cached argparse-1.4.0-py2.py3-none-any.whl (23 kB)\n", - "Installing collected packages: argparse\n", - "Successfully installed argparse-1.4.0\n", - "Requirement already satisfied: data-prep-toolkit-transforms==0.2.2.dev1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.2.dev1)\n", - "Requirement already satisfied: data-prep-toolkit>=0.2.2.dev1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.2.dev1)\n", - "Requirement already satisfied: bs4==0.0.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.0.2)\n", - "Requirement already satisfied: transformers==4.38.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.38.2)\n", - "Requirement already satisfied: parameterized in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.0)\n", - "Requirement already satisfied: pandas in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.2.2)\n", - "Requirement already satisfied: docling-core==1.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0)\n", - "Requirement already satisfied: llama-index-core<0.12.0,>=0.11.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.11.23)\n", - "Requirement already satisfied: fasttext==0.9.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.2)\n", - "Requirement already satisfied: langcodes==3.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3.0)\n", - "Requirement already satisfied: huggingface-hub<1.0.0,>=0.21.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.26.3)\n", - "Requirement already satisfied: numpy==1.26.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.26.4)\n", - "Requirement already satisfied: sentence-transformers==3.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.0.1)\n", - "Requirement already satisfied: docling-ibm-models==1.1.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.7)\n", - "Requirement already satisfied: deepsearch-glm==0.21.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.0)\n", - "Requirement already satisfied: docling==1.11.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.11.0)\n", - "Requirement already satisfied: filetype<2.0.0,>=1.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.2.0)\n", - "Requirement already satisfied: nltk==3.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.9.1)\n", - "Requirement already satisfied: torch==2.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.4.1)\n", - "Requirement already satisfied: mmh3>=4.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.1.0)\n", - "Requirement already satisfied: xxhash==3.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.4.1)\n", - "Requirement already satisfied: duckdb>=0.10.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.3)\n", - "Requirement already satisfied: networkx==3.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3)\n", - "Requirement already satisfied: colorlog==6.8.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (6.8.2)\n", - "Requirement already satisfied: func-timeout==4.3.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.3.5)\n", - "Requirement already satisfied: emerge-viz==2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.0)\n", - "Requirement already satisfied: beautifulsoup4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from bs4==0.0.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.12.3)\n", - "Requirement already satisfied: pyarrow==16.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (16.1.0)\n", - "Requirement already satisfied: boto3==1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.34.69)\n", - "Collecting argparse (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1)\n", - " Using cached argparse-1.4.0-py2.py3-none-any.whl.metadata (2.8 kB)\n", - "Requirement already satisfied: psutil in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.9.0)\n", - "Requirement already satisfied: docutils!=0.21 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.2)\n", - "Requirement already satisfied: matplotlib<4.0.0,>=3.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.9.2)\n", - "Requirement already satisfied: netwulf<0.2.0,>=0.1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.5)\n", - "Requirement already satisfied: numerize<0.13,>=0.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.12)\n", - "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.1)\n", - "Requirement already satisfied: rich<14.0.0,>=13.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (13.9.4)\n", - "Requirement already satisfied: tabulate>=0.8.9 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.0)\n", - "Requirement already satisfied: tqdm<5.0.0,>=4.64.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.67.1)\n", - "Requirement already satisfied: certifi>=2024.7.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.8.30)\n", - "Requirement already satisfied: docling-parse<2.0.0,>=1.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.6.2)\n", - "Requirement already satisfied: easyocr<2.0,>=1.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.7.2)\n", - "Requirement already satisfied: pydantic<3.0.0,>=2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.10.2)\n", - "Requirement already satisfied: pydantic-settings<3.0.0,>=2.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.6.1)\n", - "Requirement already satisfied: pypdfium2<5.0.0,>=4.30.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.30.0)\n", - "Requirement already satisfied: requests<3.0.0,>=2.32.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.32.3)\n", - "Requirement already satisfied: rtree<2.0.0,>=1.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0)\n", - "Requirement already satisfied: scipy<2.0.0,>=1.14.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.14.1)\n", - "Requirement already satisfied: json-schema-for-humans<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.4)\n", - "Requirement already satisfied: jsonref<2.0.0,>=1.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.0)\n", - "Requirement already satisfied: jsonschema<5.0.0,>=4.16.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.23.0)\n", - "Requirement already satisfied: pyproject-toml<0.0.11,>=0.0.10 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.0.10)\n", - "Requirement already satisfied: Pillow<11.0.0,>=10.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (10.4.0)\n", - "Requirement already satisfied: jsonlines<4.0.0,>=3.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.0)\n", - "Requirement already satisfied: lxml<5.0.0,>=4.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.9.4)\n", - "Requirement already satisfied: mean_average_precision<2022.0.0.0,>=2021.4.26.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2021.4.26.0)\n", - "Requirement already satisfied: onnxruntime<2.0.0,>=1.16.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.20.1)\n", - "Requirement already satisfied: opencv-python-headless<5.0.0.0,>=4.9.0.80 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.10.0.84)\n", - "Requirement already satisfied: torchvision<1,>=0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.19.1)\n", - "Requirement already satisfied: wheel in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.44.0)\n", - "Requirement already satisfied: autopep8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.3.1)\n", - "Requirement already satisfied: coloredlogs in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (15.0.1)\n", - "Requirement already satisfied: interrogate in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.7.0)\n", - "Requirement already satisfied: scikit-learn in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.5.2)\n", - "Requirement already satisfied: prettytable in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.12.0)\n", - "Requirement already satisfied: py in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.11.0)\n", - "Requirement already satisfied: pycodestyle in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.12.1)\n", - "Requirement already satisfied: pylint in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3.1)\n", - "Requirement already satisfied: pyparsing in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.2.0)\n", - "Requirement already satisfied: python-louvain in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.16)\n", - "Requirement already satisfied: PyYAML in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (6.0.2)\n", - "Requirement already satisfied: PyDriller in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.7)\n", - "Requirement already satisfied: pyperclip in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.9.0)\n", - "Requirement already satisfied: pybind11>=2.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fasttext==0.9.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.13.6)\n", - "Requirement already satisfied: setuptools>=0.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fasttext==0.9.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (75.1.0)\n", - "Requirement already satisfied: click in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (8.1.7)\n", - "Requirement already satisfied: joblib in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.4.2)\n", - "Requirement already satisfied: regex>=2021.8.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.11.6)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.9.0.post0)\n", - "Requirement already satisfied: pytz>=2020.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.2)\n", - "Requirement already satisfied: tzdata>=2022.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.2)\n", - "Requirement already satisfied: filelock in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.16.1)\n", - "Requirement already satisfied: typing-extensions>=4.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.12.2)\n", - "Requirement already satisfied: sympy in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.13.3)\n", - "Requirement already satisfied: jinja2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.4)\n", - "Requirement already satisfied: fsspec in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.10.0)\n", - "Requirement already satisfied: packaging>=20.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (24.2)\n", - "Requirement already satisfied: tokenizers<0.19,>=0.14 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.15.2)\n", - "Requirement already satisfied: safetensors>=0.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.5)\n", - "Requirement already satisfied: botocore<1.35.0,>=1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.34.162)\n", - "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.1)\n", - "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.10.4)\n", - "Requirement already satisfied: ray==2.36.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.36.1)\n", - "Requirement already satisfied: fastapi>=0.110.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.115.5)\n", - "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.0)\n", - "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.29.0)\n", - "Requirement already satisfied: aiosignal in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.1)\n", - "Requirement already satisfied: frozenlist in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.5.0)\n", - "Requirement already satisfied: aiohttp>=3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.11.8)\n", - "Requirement already satisfied: aiohttp-cors in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.7.0)\n", - "Requirement already satisfied: colorful in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.5.6)\n", - "Requirement already satisfied: py-spy>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.0)\n", - "Requirement already satisfied: opencensus in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.11.4)\n", - "Requirement already satisfied: prometheus-client>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.0)\n", - "Requirement already satisfied: smart-open in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (7.0.5)\n", - "Requirement already satisfied: virtualenv!=20.21.1,>=20.0.24 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (20.28.0)\n", - "Requirement already satisfied: grpcio>=1.42.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.68.0)\n", - "Requirement already satisfied: memray in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.14.0)\n", - "Requirement already satisfied: SQLAlchemy>=1.4.49 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.36)\n", - "Requirement already satisfied: dataclasses-json in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.6.7)\n", - "Requirement already satisfied: deprecated>=1.2.9.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.2.15)\n", - "Requirement already satisfied: dirtyjson<2.0.0,>=1.0.8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.8)\n", - "Requirement already satisfied: httpx in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.28.0)\n", - "Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.6.0)\n", - "Requirement already satisfied: tenacity!=8.4.0,<9.0.0,>=8.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (8.5.0)\n", - "Requirement already satisfied: tiktoken>=0.3.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.8.0)\n", - "Requirement already satisfied: typing-inspect>=0.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.0)\n", - "Requirement already satisfied: wrapt in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.17.0)\n", - "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.4.3)\n", - "Requirement already satisfied: attrs>=17.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (24.2.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (6.1.0)\n", - "Requirement already satisfied: propcache>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.0)\n", - "Requirement already satisfied: yarl<2.0,>=1.17.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.18.0)\n", - "Requirement already satisfied: scikit-image in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.24.0)\n", - "Requirement already satisfied: python-bidi in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.6.3)\n", - "Requirement already satisfied: Shapely in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.6)\n", - "Requirement already satisfied: pyclipper in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0.post6)\n", - "Requirement already satisfied: ninja in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.11.1.2)\n", - "Requirement already satisfied: starlette<0.42.0,>=0.40.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fastapi>=0.110.2->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.41.3)\n", - "Requirement already satisfied: MarkupSafe<3.0.0,>=2.1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.1.5)\n", - "Requirement already satisfied: Pygments<3.0.0,>=2.18.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.18.0)\n", - "Requirement already satisfied: htmlmin2<0.2.0,>=0.1.13 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.13)\n", - "Requirement already satisfied: markdown2<3.0.0,>=2.5.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.5.1)\n", - "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from dataclasses-json->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.23.1)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.16.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.10.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.16.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.35.1)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.16.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.0)\n", - "Requirement already satisfied: contourpy>=1.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.1)\n", - "Requirement already satisfied: cycler>=0.10 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.12.1)\n", - "Requirement already satisfied: fonttools>=4.22.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.55.0)\n", - "Requirement already satisfied: kiwisolver>=1.3.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.4.7)\n", - "Requirement already satisfied: simplejson>=3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from netwulf<0.2.0,>=0.1.5->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.19.3)\n", - "Requirement already satisfied: flatbuffers in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from onnxruntime<2.0.0,>=1.16.2->docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (24.3.25)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.27.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.27.1)\n", - "Requirement already satisfied: toml in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pyproject-toml<0.0.11,>=0.0.10->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.10.2)\n", - "Requirement already satisfied: six>=1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.16.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests<3.0.0,>=2.32.3->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.4.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests<3.0.0,>=2.32.3->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.10)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests<3.0.0,>=2.32.3->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.2.3)\n", - "Requirement already satisfied: markdown-it-py>=2.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from rich<14.0.0,>=13.7.0->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.0.0)\n", - "Requirement already satisfied: greenlet!=0.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.1)\n", - "Requirement already satisfied: mypy-extensions>=0.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from typing-inspect>=0.8.0->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.0)\n", - "Requirement already satisfied: soupsieve>1.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from beautifulsoup4->bs4==0.0.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.6)\n", - "Requirement already satisfied: humanfriendly>=9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from coloredlogs->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (10.0)\n", - "Requirement already satisfied: anyio in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.6.2.post1)\n", - "Requirement already satisfied: httpcore==1.* in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.7)\n", - "Requirement already satisfied: h11<0.15,>=0.13 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpcore==1.*->httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.14.0)\n", - "Requirement already satisfied: colorama in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from interrogate->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.6)\n", - "Requirement already satisfied: wcwidth in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from prettytable->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.13)\n", - "Requirement already satisfied: gitpython in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.43)\n", - "Requirement already satisfied: types-pytz in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.2.0.20241003)\n", - "Requirement already satisfied: lizard in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.17.13)\n", - "Requirement already satisfied: platformdirs>=2.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.3.6)\n", - "Requirement already satisfied: astroid<=3.4.0-dev0,>=3.3.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3.5)\n", - "Requirement already satisfied: isort!=5.13.0,<6,>=4.2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.13.2)\n", - "Requirement already satisfied: mccabe<0.8,>=0.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.7.0)\n", - "Requirement already satisfied: tomlkit>=0.10.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.13.2)\n", - "Requirement already satisfied: dill>=0.3.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.3.9)\n", - "Requirement already satisfied: threadpoolctl>=3.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-learn->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.5.0)\n", - "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from sympy->torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0)\n", - "Requirement already satisfied: mdurl~=0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich<14.0.0,>=13.7.0->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.2)\n", - "Requirement already satisfied: sniffio>=1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from anyio->httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.1)\n", - "Requirement already satisfied: distlib<1,>=0.3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.3.9)\n", - "Requirement already satisfied: gitdb<5,>=4.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from gitpython->PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.0.11)\n", - "Requirement already satisfied: textual>=0.41.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.87.1)\n", - "Requirement already satisfied: opencensus-context>=0.1.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.3)\n", - "Requirement already satisfied: google-api-core<3.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.23.0)\n", - "Requirement already satisfied: imageio>=2.33 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-image->easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.36.1)\n", - "Requirement already satisfied: tifffile>=2022.8.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-image->easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.9.20)\n", - "Requirement already satisfied: lazy-loader>=0.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-image->easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4)\n", - "Requirement already satisfied: smmap<6,>=3.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from gitdb<5,>=4.0.1->gitpython->PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.0.1)\n", - "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.66.0)\n", - "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.25.0)\n", - "Requirement already satisfied: google-auth<3.0.dev0,>=2.14.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.36.0)\n", - "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.5.0)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.1)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.9)\n", - "Requirement already satisfied: linkify-it-py<3,>=1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.3)\n", - "Requirement already satisfied: mdit-py-plugins in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.2)\n", - "Requirement already satisfied: uc-micro-py in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.3)\n", - "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.6.1)\n", - "Using cached argparse-1.4.0-py2.py3-none-any.whl (23 kB)\n", - "Installing collected packages: argparse\n", - "Successfully installed argparse-1.4.0\n", - "Requirement already satisfied: jupyterlab in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (4.3.2)\n", - "Requirement already satisfied: ipykernel in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (6.29.5)\n", - "Requirement already satisfied: ipywidgets in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (8.1.5)\n", - "Requirement already satisfied: async-lru>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (2.0.4)\n", - "Requirement already satisfied: httpx~=0.28.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (0.28.0)\n", - "Requirement already satisfied: jinja2>=3.0.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (3.1.4)\n", - "Requirement already satisfied: jupyter-core in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (5.7.2)\n", - "Requirement already satisfied: jupyter-lsp>=2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (2.2.5)\n", - "Requirement already satisfied: jupyter-server<3,>=2.4.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (2.14.2)\n", - "Requirement already satisfied: jupyterlab-server<3,>=2.27.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (2.27.3)\n", - "Requirement already satisfied: notebook-shim>=0.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (0.2.4)\n", - "Requirement already satisfied: packaging in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (24.2)\n", - "Requirement already satisfied: setuptools>=40.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (75.1.0)\n", - "Requirement already satisfied: tornado>=6.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (6.4.1)\n", - "Requirement already satisfied: traitlets in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab) (5.14.3)\n", - "Requirement already satisfied: appnope in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipykernel) (0.1.4)\n", - "Requirement already satisfied: comm>=0.1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipykernel) (0.2.2)\n", - "Requirement already satisfied: debugpy>=1.6.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipykernel) (1.6.7)\n", - "Requirement already satisfied: ipython>=7.23.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipykernel) (8.29.0)\n", - "Requirement already satisfied: jupyter-client>=6.1.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipykernel) (8.6.3)\n", - "Requirement already satisfied: matplotlib-inline>=0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipykernel) (0.1.7)\n", - "Requirement already satisfied: nest-asyncio in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipykernel) (1.6.0)\n", - "Requirement already satisfied: psutil in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipykernel) (5.9.0)\n", - "Requirement already satisfied: pyzmq>=24 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipykernel) (25.1.2)\n", - "Requirement already satisfied: widgetsnbextension~=4.0.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipywidgets) (4.0.13)\n", - "Requirement already satisfied: jupyterlab-widgets~=3.0.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipywidgets) (3.0.13)\n", - "Requirement already satisfied: anyio in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpx~=0.28.0->jupyterlab) (4.6.2.post1)\n", - "Requirement already satisfied: certifi in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpx~=0.28.0->jupyterlab) (2024.8.30)\n", - "Requirement already satisfied: httpcore==1.* in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpx~=0.28.0->jupyterlab) (1.0.7)\n", - "Requirement already satisfied: idna in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpx~=0.28.0->jupyterlab) (3.10)\n", - "Requirement already satisfied: h11<0.15,>=0.13 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpcore==1.*->httpx~=0.28.0->jupyterlab) (0.14.0)\n", - "Requirement already satisfied: decorator in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel) (5.1.1)\n", - "Requirement already satisfied: jedi>=0.16 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel) (0.19.2)\n", - "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel) (3.0.48)\n", - "Requirement already satisfied: pygments>=2.4.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel) (2.18.0)\n", - "Requirement already satisfied: stack-data in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel) (0.6.2)\n", - "Requirement already satisfied: typing-extensions>=4.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel) (4.12.2)\n", - "Requirement already satisfied: pexpect>4.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ipython>=7.23.1->ipykernel) (4.9.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jinja2>=3.0.3->jupyterlab) (2.1.5)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-client>=6.1.12->ipykernel) (2.9.0.post0)\n", - "Requirement already satisfied: platformdirs>=2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-core->jupyterlab) (4.3.6)\n", - "Requirement already satisfied: argon2-cffi>=21.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (23.1.0)\n", - "Requirement already satisfied: jupyter-events>=0.9.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.10.0)\n", - "Requirement already satisfied: jupyter-server-terminals>=0.4.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.5.3)\n", - "Requirement already satisfied: nbconvert>=6.4.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (7.16.4)\n", - "Requirement already satisfied: nbformat>=5.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (5.10.4)\n", - "Requirement already satisfied: overrides>=5.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (7.7.0)\n", - "Requirement already satisfied: prometheus-client>=0.9 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.21.0)\n", - "Requirement already satisfied: send2trash>=1.8.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (1.8.3)\n", - "Requirement already satisfied: terminado>=0.8.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (0.18.1)\n", - "Requirement already satisfied: websocket-client>=1.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-server<3,>=2.4.0->jupyterlab) (1.8.0)\n", - "Requirement already satisfied: babel>=2.10 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.27.1->jupyterlab) (2.16.0)\n", - "Requirement already satisfied: json5>=0.9.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.27.1->jupyterlab) (0.10.0)\n", - "Requirement already satisfied: jsonschema>=4.18.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.27.1->jupyterlab) (4.23.0)\n", - "Requirement already satisfied: requests>=2.31 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyterlab-server<3,>=2.27.1->jupyterlab) (2.32.3)\n", - "Requirement already satisfied: sniffio>=1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from anyio->httpx~=0.28.0->jupyterlab) (1.3.1)\n", - "Requirement already satisfied: argon2-cffi-bindings in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from argon2-cffi>=21.1->jupyter-server<3,>=2.4.0->jupyterlab) (21.2.0)\n", - "Requirement already satisfied: parso<0.9.0,>=0.8.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jedi>=0.16->ipython>=7.23.1->ipykernel) (0.8.4)\n", - "Requirement already satisfied: attrs>=22.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.27.1->jupyterlab) (24.2.0)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.27.1->jupyterlab) (2024.10.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.27.1->jupyterlab) (0.35.1)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema>=4.18.0->jupyterlab-server<3,>=2.27.1->jupyterlab) (0.21.0)\n", - "Requirement already satisfied: python-json-logger>=2.0.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (2.0.7)\n", - "Requirement already satisfied: pyyaml>=5.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (6.0.2)\n", - "Requirement already satisfied: rfc3339-validator in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (0.1.4)\n", - "Requirement already satisfied: rfc3986-validator>=0.1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (0.1.1)\n", - "Requirement already satisfied: beautifulsoup4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (4.12.3)\n", - "Requirement already satisfied: bleach!=5.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (6.2.0)\n", - "Requirement already satisfied: defusedxml in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.7.1)\n", - "Requirement already satisfied: jupyterlab-pygments in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.3.0)\n", - "Requirement already satisfied: mistune<4,>=2.0.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (3.0.2)\n", - "Requirement already satisfied: nbclient>=0.5.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.10.1)\n", - "Requirement already satisfied: pandocfilters>=1.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (1.5.1)\n", - "Requirement already satisfied: tinycss2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (1.4.0)\n", - "Requirement already satisfied: fastjsonschema>=2.15 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nbformat>=5.3.0->jupyter-server<3,>=2.4.0->jupyterlab) (2.21.1)\n", - "Requirement already satisfied: ptyprocess>=0.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pexpect>4.3->ipython>=7.23.1->ipykernel) (0.7.0)\n", - "Requirement already satisfied: wcwidth in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython>=7.23.1->ipykernel) (0.2.13)\n", - "Requirement already satisfied: six>=1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from python-dateutil>=2.8.2->jupyter-client>=6.1.12->ipykernel) (1.16.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.31->jupyterlab-server<3,>=2.27.1->jupyterlab) (3.4.0)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.31->jupyterlab-server<3,>=2.27.1->jupyterlab) (2.2.3)\n", - "Requirement already satisfied: executing>=1.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from stack-data->ipython>=7.23.1->ipykernel) (2.1.0)\n", - "Requirement already satisfied: asttokens>=2.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from stack-data->ipython>=7.23.1->ipykernel) (2.4.1)\n", - "Requirement already satisfied: pure-eval in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from stack-data->ipython>=7.23.1->ipykernel) (0.2.3)\n", - "Requirement already satisfied: webencodings in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from bleach!=5.0.0->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (0.5.1)\n", - "Requirement already satisfied: fqdn in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.5.1)\n", - "Requirement already satisfied: isoduration in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (20.11.0)\n", - "Requirement already satisfied: jsonpointer>1.13 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (3.0.0)\n", - "Requirement already satisfied: uri-template in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.3.0)\n", - "Requirement already satisfied: webcolors>=24.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (24.11.1)\n", - "Requirement already satisfied: cffi>=1.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from argon2-cffi-bindings->argon2-cffi>=21.1->jupyter-server<3,>=2.4.0->jupyterlab) (1.17.1)\n", - "Requirement already satisfied: soupsieve>1.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from beautifulsoup4->nbconvert>=6.4.4->jupyter-server<3,>=2.4.0->jupyterlab) (2.6)\n", - "Requirement already satisfied: pycparser in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi>=21.1->jupyter-server<3,>=2.4.0->jupyterlab) (2.22)\n", - "Requirement already satisfied: arrow>=0.15.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from isoduration->jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (1.3.0)\n", - "Requirement already satisfied: types-python-dateutil>=2.8.10 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from arrow>=0.15.0->isoduration->jsonschema[format-nongpl]>=4.18.0->jupyter-events>=0.9.0->jupyter-server<3,>=2.4.0->jupyterlab) (2.9.0.20241003)\n", - "Requirement already satisfied: nltk==3.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from -r requirements.txt (line 1)) (3.9.1)\n", - "Requirement already satisfied: transformers==4.38.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from -r requirements.txt (line 2)) (4.38.2)\n", - "Requirement already satisfied: torch<=2.4.1,>=2.2.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from -r requirements.txt (line 3)) (2.4.1)\n", - "Requirement already satisfied: pandas==2.2.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from -r requirements.txt (line 4)) (2.2.2)\n", - "Requirement already satisfied: click in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->-r requirements.txt (line 1)) (8.1.7)\n", - "Requirement already satisfied: joblib in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->-r requirements.txt (line 1)) (1.4.2)\n", - "Requirement already satisfied: regex>=2021.8.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->-r requirements.txt (line 1)) (2024.11.6)\n", - "Requirement already satisfied: tqdm in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->-r requirements.txt (line 1)) (4.67.1)\n", - "Requirement already satisfied: filelock in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (3.16.1)\n", - "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (0.26.3)\n", - "Requirement already satisfied: numpy>=1.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (1.26.4)\n", - "Requirement already satisfied: packaging>=20.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (24.2)\n", - "Requirement already satisfied: pyyaml>=5.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (6.0.2)\n", - "Requirement already satisfied: requests in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (2.32.3)\n", - "Requirement already satisfied: tokenizers<0.19,>=0.14 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (0.15.2)\n", - "Requirement already satisfied: safetensors>=0.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (0.4.5)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas==2.2.2->-r requirements.txt (line 4)) (2.9.0.post0)\n", - "Requirement already satisfied: pytz>=2020.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas==2.2.2->-r requirements.txt (line 4)) (2024.2)\n", - "Requirement already satisfied: tzdata>=2022.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas==2.2.2->-r requirements.txt (line 4)) (2024.2)\n", - "Requirement already satisfied: typing-extensions>=4.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (4.12.2)\n", - "Requirement already satisfied: sympy in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (1.13.3)\n", - "Requirement already satisfied: networkx in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (3.3)\n", - "Requirement already satisfied: jinja2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (3.1.4)\n", - "Requirement already satisfied: fsspec in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (2024.10.0)\n", - "Requirement already satisfied: six>=1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas==2.2.2->-r requirements.txt (line 4)) (1.16.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jinja2->torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (2.1.5)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->transformers==4.38.2->-r requirements.txt (line 2)) (3.4.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->transformers==4.38.2->-r requirements.txt (line 2)) (3.10)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->transformers==4.38.2->-r requirements.txt (line 2)) (2.2.3)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->transformers==4.38.2->-r requirements.txt (line 2)) (2024.8.30)\n", - "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from sympy->torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (1.3.0)\n" - ] - } - ], + "outputs": [], "source": [ "! pip install data-prep-connector\n", "! pip install 'data-prep-toolkit[ray]==0.2.2.dev1'\n", @@ -531,7 +82,7 @@ }, { "cell_type": "code", - "execution_count": 500, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -552,7 +103,7 @@ }, { "cell_type": "code", - "execution_count": 501, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -563,19 +114,9 @@ }, { "cell_type": "code", - "execution_count": 502, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Script Path: ./transforms/universal/hap/python/src/hap_local_python.py\n", - "Input Folder: ./input\n", - "Output Folder: ./output\n" - ] - } - ], + "outputs": [], "source": [ "# Ensure the necessary folders exist.\n", "os.makedirs(input_folder, exist_ok=True)\n", @@ -600,18 +141,9 @@ }, { "cell_type": "code", - "execution_count": 503, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found CSV file(s): ['customer_feedback_file.csv']\n", - "Using CSV file: ./input/customer_feedback_file.csv\n" - ] - } - ], + "outputs": [], "source": [ "csv_files = [f for f in os.listdir(input_folder) if f.endswith(\".csv\")]\n", "\n", @@ -636,17 +168,9 @@ }, { "cell_type": "code", - "execution_count": 504, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CSV file converted to Parquet format at: ./input/data.parquet\n" - ] - } - ], + "outputs": [], "source": [ "parquet_file_path = os.path.join(input_folder, \"data.parquet\")\n", "df = pd.read_csv(csv_file_path)\n", @@ -663,17 +187,9 @@ }, { "cell_type": "code", - "execution_count": 505, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Command-line arguments: ['hap_local_python.py', '--input_folder', './input', '--output_folder', './output', '--model_name_or_path', 'ibm-granite/granite-guardian-hap-38m', '--annotation_column', 'hap_score', '--doc_text_column', 'Customer Feedback', '--inference_engine', 'CPU', '--max_length', '512', '--batch_size', '128']\n" - ] - } - ], + "outputs": [], "source": [ "\n", "# Clear sys.argv first to avoid conflicts (reset arguments list)\n", @@ -711,19 +227,9 @@ }, { "cell_type": "code", - "execution_count": 506, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error occurred during transform execution.\n", - "python: can't open file '/Users/aisha/Documents/GitHub/Personal/DPK/examples/notebooks/hap/./transforms/universal/hap/python/src/hap_local_python.py': [Errno 2] No such file or directory\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "# Copy the current environment variables\n", "env = os.environ.copy()\n", @@ -777,134 +283,9 @@ }, { "cell_type": "code", - "execution_count": 507, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Complete Output Parquet File Path: ./output/data.parquet\n", - "Old complete CSV file removed: ./output/hap_complete_output.csv\n", - "Old filtered CSV file removed: ./output/hap_filtered_output.csv\n", - "Filtered Output (only HAP score and document text):\n" - ] - }, - { - "data": { - "text/html": [ - "

\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Customer Feedbackhap_score
0Rating: 4 Comments: \"Service was prompt, but ...0.000195
1Rating: 5 Comments: \"Great help from Peter! H...0.000153
2Rating: 3 Comments: \"The service was quick, b...0.000169
3Rating: 5 Comments: \"Excellent service and ad...0.000158
4Rating: 2 Comments: \"I’m really frustrated. T...0.000875
.........
60Rating: 3 Comments: \"This is not what I expec...0.000150
61Rating: 1 Comments: \"This is insane. I’ve onl...0.579251
62Rating: 4 Comments: \"I need this fixed. I can’...0.000384
63Rating: 2 Comments: \"I’m so done with this ma...0.000285
64Rating: 3 Comments: \"I’ve leveled the machine...0.052517
\n", - "

65 rows × 2 columns

\n", - "
" - ], - "text/plain": [ - " Customer Feedback hap_score\n", - "0 Rating: 4 Comments: \"Service was prompt, but ... 0.000195\n", - "1 Rating: 5 Comments: \"Great help from Peter! H... 0.000153\n", - "2 Rating: 3 Comments: \"The service was quick, b... 0.000169\n", - "3 Rating: 5 Comments: \"Excellent service and ad... 0.000158\n", - "4 Rating: 2 Comments: \"I’m really frustrated. T... 0.000875\n", - ".. ... ...\n", - "60 Rating: 3 Comments: \"This is not what I expec... 0.000150\n", - "61 Rating: 1 Comments: \"This is insane. I’ve onl... 0.579251\n", - "62 Rating: 4 Comments: \"I need this fixed. I can’... 0.000384\n", - "63 Rating: 2 Comments: \"I’m so done with this ma... 0.000285\n", - "64 Rating: 3 Comments: \"I’ve leveled the machine... 0.052517\n", - "\n", - "[65 rows x 2 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Complete output saved to: ./output/hap_complete_output.csv\n", - "Filtered output saved to: ./output/hap_filtered_output.csv\n" - ] - } - ], + "outputs": [], "source": [ "import os\n", "import pandas as pd\n", From 27c6e9a722f6d992d51acf2caea0d770fb6bce75 Mon Sep 17 00:00:00 2001 From: Aisha Mohammed Farooq Darga Date: Thu, 5 Dec 2024 09:26:48 +0530 Subject: [PATCH 16/31] Addressed review comments Signed-off-by: Aisha Mohammed Farooq Darga --- .../hap/generate_hap_score_csv.ipynb | 600 +++++++++++++++++- examples/notebooks/hap/output/metadata.json | 12 +- 2 files changed, 571 insertions(+), 41 deletions(-) diff --git a/examples/notebooks/hap/generate_hap_score_csv.ipynb b/examples/notebooks/hap/generate_hap_score_csv.ipynb index 58c44b94d..09621666d 100644 --- a/examples/notebooks/hap/generate_hap_score_csv.ipynb +++ b/examples/notebooks/hap/generate_hap_score_csv.ipynb @@ -6,8 +6,10 @@ "source": [ "HAP Transform Example Notebook\n", "=====================================\n", - "This notebook picks a CSV file from the `input` folder, converts it to Parquet format,\n", - "runs the `hap_local_python.py` transform, and displays the results." + "\n", + "This notebook processes a CSV file containing text data to analyze for Hate, Abuse, and Profanity (HAP) scores.\n", + "It converts the CSV file into Parquet format, uses the `hap_local_python.py` script to calculate HAP scores, \n", + "and generates outputs for further analysis." ] }, { @@ -63,9 +65,364 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: data-prep-connector in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (0.2.3)\n", + "Requirement already satisfied: scrapy>=2.11.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-connector) (2.12.0)\n", + "Requirement already satisfied: pydantic>=2.8.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-connector) (2.10.2)\n", + "Requirement already satisfied: tldextract>=5.1.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-connector) (5.1.3)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic>=2.8.1->data-prep-connector) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.27.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic>=2.8.1->data-prep-connector) (2.27.1)\n", + "Requirement already satisfied: typing-extensions>=4.12.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic>=2.8.1->data-prep-connector) (4.12.2)\n", + "Requirement already satisfied: Twisted>=21.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.11.0)\n", + "Requirement already satisfied: cryptography>=37.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (44.0.0)\n", + "Requirement already satisfied: cssselect>=0.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.2.0)\n", + "Requirement already satisfied: itemloaders>=1.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.3.2)\n", + "Requirement already satisfied: parsel>=1.5.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.9.1)\n", + "Requirement already satisfied: pyOpenSSL>=22.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.3.0)\n", + "Requirement already satisfied: queuelib>=1.4.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.7.0)\n", + "Requirement already satisfied: service-identity>=18.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.2.0)\n", + "Requirement already satisfied: w3lib>=1.17.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (2.2.1)\n", + "Requirement already satisfied: zope.interface>=5.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (7.2)\n", + "Requirement already satisfied: protego>=0.1.15 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (0.3.1)\n", + "Requirement already satisfied: itemadapter>=0.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (0.10.0)\n", + "Requirement already satisfied: packaging in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.2)\n", + "Requirement already satisfied: lxml>=4.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (4.9.4)\n", + "Requirement already satisfied: defusedxml>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (0.7.1)\n", + "Requirement already satisfied: PyDispatcher>=2.0.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (2.0.7)\n", + "Requirement already satisfied: idna in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (3.10)\n", + "Requirement already satisfied: requests>=2.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (2.32.3)\n", + "Requirement already satisfied: requests-file>=1.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (2.1.0)\n", + "Requirement already satisfied: filelock>=3.0.8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (3.16.1)\n", + "Requirement already satisfied: cffi>=1.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from cryptography>=37.0.0->scrapy>=2.11.2->data-prep-connector) (1.17.1)\n", + "Requirement already satisfied: jmespath>=0.9.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from itemloaders>=1.0.1->scrapy>=2.11.2->data-prep-connector) (1.0.1)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.1.0->tldextract>=5.1.2->data-prep-connector) (3.4.0)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.1.0->tldextract>=5.1.2->data-prep-connector) (2.2.3)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.1.0->tldextract>=5.1.2->data-prep-connector) (2024.8.30)\n", + "Requirement already satisfied: attrs>=19.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from service-identity>=18.1.0->scrapy>=2.11.2->data-prep-connector) (24.2.0)\n", + "Requirement already satisfied: pyasn1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from service-identity>=18.1.0->scrapy>=2.11.2->data-prep-connector) (0.6.1)\n", + "Requirement already satisfied: pyasn1-modules in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from service-identity>=18.1.0->scrapy>=2.11.2->data-prep-connector) (0.4.1)\n", + "Requirement already satisfied: automat>=24.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (24.8.1)\n", + "Requirement already satisfied: constantly>=15.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (23.10.4)\n", + "Requirement already satisfied: hyperlink>=17.1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (21.0.0)\n", + "Requirement already satisfied: incremental>=24.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (24.7.2)\n", + "Requirement already satisfied: setuptools in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from zope.interface>=5.1.0->scrapy>=2.11.2->data-prep-connector) (75.1.0)\n", + "Requirement already satisfied: pycparser in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from cffi>=1.12->cryptography>=37.0.0->scrapy>=2.11.2->data-prep-connector) (2.22)\n", + "Requirement already satisfied: data-prep-toolkit==0.2.2.dev1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]==0.2.2.dev1) (0.2.2.dev1)\n", + "Requirement already satisfied: numpy<1.29.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.26.4)\n", + "Requirement already satisfied: pyarrow==16.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (16.1.0)\n", + "Requirement already satisfied: boto3==1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.34.69)\n", + "Collecting argparse (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1)\n", + " Using cached argparse-1.4.0-py2.py3-none-any.whl.metadata (2.8 kB)\n", + "Requirement already satisfied: mmh3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (4.1.0)\n", + "Requirement already satisfied: psutil in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (5.9.0)\n", + "Requirement already satisfied: ray==2.36.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.36.1)\n", + "Requirement already satisfied: fastapi>=0.110.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]==0.2.2.dev1) (0.115.5)\n", + "Requirement already satisfied: pillow>=10.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]==0.2.2.dev1) (10.4.0)\n", + "Requirement already satisfied: botocore<1.35.0,>=1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.34.162)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (0.10.4)\n", + "Requirement already satisfied: click>=7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (8.1.7)\n", + "Requirement already satisfied: filelock in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.16.1)\n", + "Requirement already satisfied: jsonschema in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (4.23.0)\n", + "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.1.0)\n", + "Requirement already satisfied: packaging in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (24.2)\n", + "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (5.29.0)\n", + "Requirement already satisfied: pyyaml in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (6.0.2)\n", + "Requirement already satisfied: aiosignal in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.3.1)\n", + "Requirement already satisfied: frozenlist in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.5.0)\n", + "Requirement already satisfied: requests in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.32.3)\n", + "Requirement already satisfied: aiohttp>=3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.11.8)\n", + "Requirement already satisfied: aiohttp-cors in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.7.0)\n", + "Requirement already satisfied: colorful in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.5.6)\n", + "Requirement already satisfied: py-spy>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.4.0)\n", + "Requirement already satisfied: opencensus in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.11.4)\n", + "Requirement already satisfied: pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.10.2)\n", + "Requirement already satisfied: prometheus-client>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.21.0)\n", + "Requirement already satisfied: smart-open in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (7.0.5)\n", + "Requirement already satisfied: virtualenv!=20.21.1,>=20.0.24 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (20.28.0)\n", + "Requirement already satisfied: grpcio>=1.42.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.68.0)\n", + "Requirement already satisfied: memray in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.14.0)\n", + "Requirement already satisfied: starlette<0.42.0,>=0.40.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (0.41.3)\n", + "Requirement already satisfied: typing-extensions>=4.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (4.12.2)\n", + "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.4.3)\n", + "Requirement already satisfied: attrs>=17.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (24.2.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (6.1.0)\n", + "Requirement already satisfied: propcache>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.2.0)\n", + "Requirement already satisfied: yarl<2.0,>=1.17.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.18.0)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from botocore<1.35.0,>=1.34.69->boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (2.9.0.post0)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from botocore<1.35.0,>=1.34.69->boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (2.2.3)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.27.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.27.1)\n", + "Requirement already satisfied: anyio<5,>=3.4.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from starlette<0.42.0,>=0.40.0->fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (4.6.2.post1)\n", + "Requirement already satisfied: distlib<1,>=0.3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.3.9)\n", + "Requirement already satisfied: platformdirs<5,>=3.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (4.3.6)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2024.10.1)\n", + "Requirement already satisfied: referencing>=0.28.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.35.1)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.21.0)\n", + "Requirement already satisfied: jinja2>=2.9 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.1.4)\n", + "Requirement already satisfied: rich>=11.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (13.9.4)\n", + "Requirement already satisfied: textual>=0.41.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.87.1)\n", + "Requirement already satisfied: opencensus-context>=0.1.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.1.3)\n", + "Requirement already satisfied: six~=1.16 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.16.0)\n", + "Requirement already satisfied: google-api-core<3.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.23.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.4.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.10)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2024.8.30)\n", + "Requirement already satisfied: wrapt in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from smart-open->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.17.0)\n", + "Requirement already satisfied: sniffio>=1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from anyio<5,>=3.4.0->starlette<0.42.0,>=0.40.0->fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (1.3.1)\n", + "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.66.0)\n", + "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.25.0)\n", + "Requirement already satisfied: google-auth<3.0.dev0,>=2.14.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.36.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jinja2>=2.9->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.1.5)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from rich>=11.2.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.0.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from rich>=11.2.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.18.0)\n", + "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (5.5.0)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.4.1)\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (4.9)\n", + "Requirement already satisfied: mdurl~=0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich>=11.2.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.1.2)\n", + "Requirement already satisfied: linkify-it-py<3,>=1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.0.3)\n", + "Requirement already satisfied: mdit-py-plugins in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.4.2)\n", + "Requirement already satisfied: uc-micro-py in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.0.3)\n", + "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.6.1)\n", + "Using cached argparse-1.4.0-py2.py3-none-any.whl (23 kB)\n", + "Installing collected packages: argparse\n", + "Successfully installed argparse-1.4.0\n", + "Requirement already satisfied: data-prep-toolkit-transforms==0.2.2.dev1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.2.dev1)\n", + "Requirement already satisfied: data-prep-toolkit>=0.2.2.dev1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.2.dev1)\n", + "Requirement already satisfied: bs4==0.0.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.0.2)\n", + "Requirement already satisfied: transformers==4.38.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.38.2)\n", + "Requirement already satisfied: parameterized in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.0)\n", + "Requirement already satisfied: pandas in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.2.2)\n", + "Requirement already satisfied: docling-core==1.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0)\n", + "Requirement already satisfied: llama-index-core<0.12.0,>=0.11.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.11.23)\n", + "Requirement already satisfied: fasttext==0.9.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.2)\n", + "Requirement already satisfied: langcodes==3.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3.0)\n", + "Requirement already satisfied: huggingface-hub<1.0.0,>=0.21.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.26.3)\n", + "Requirement already satisfied: numpy==1.26.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.26.4)\n", + "Requirement already satisfied: sentence-transformers==3.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.0.1)\n", + "Requirement already satisfied: docling-ibm-models==1.1.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.7)\n", + "Requirement already satisfied: deepsearch-glm==0.21.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.0)\n", + "Requirement already satisfied: docling==1.11.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.11.0)\n", + "Requirement already satisfied: filetype<2.0.0,>=1.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.2.0)\n", + "Requirement already satisfied: nltk==3.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.9.1)\n", + "Requirement already satisfied: torch==2.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.4.1)\n", + "Requirement already satisfied: mmh3>=4.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.1.0)\n", + "Requirement already satisfied: xxhash==3.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.4.1)\n", + "Requirement already satisfied: duckdb>=0.10.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.3)\n", + "Requirement already satisfied: networkx==3.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3)\n", + "Requirement already satisfied: colorlog==6.8.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (6.8.2)\n", + "Requirement already satisfied: func-timeout==4.3.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.3.5)\n", + "Requirement already satisfied: emerge-viz==2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.0)\n", + "Requirement already satisfied: beautifulsoup4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from bs4==0.0.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.12.3)\n", + "Requirement already satisfied: pyarrow==16.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (16.1.0)\n", + "Requirement already satisfied: boto3==1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.34.69)\n", + "Collecting argparse (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1)\n", + " Using cached argparse-1.4.0-py2.py3-none-any.whl.metadata (2.8 kB)\n", + "Requirement already satisfied: psutil in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.9.0)\n", + "Requirement already satisfied: docutils!=0.21 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.2)\n", + "Requirement already satisfied: matplotlib<4.0.0,>=3.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.9.2)\n", + "Requirement already satisfied: netwulf<0.2.0,>=0.1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.5)\n", + "Requirement already satisfied: numerize<0.13,>=0.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.12)\n", + "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.1)\n", + "Requirement already satisfied: rich<14.0.0,>=13.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (13.9.4)\n", + "Requirement already satisfied: tabulate>=0.8.9 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.0)\n", + "Requirement already satisfied: tqdm<5.0.0,>=4.64.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.67.1)\n", + "Requirement already satisfied: certifi>=2024.7.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.8.30)\n", + "Requirement already satisfied: docling-parse<2.0.0,>=1.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.6.2)\n", + "Requirement already satisfied: easyocr<2.0,>=1.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.7.2)\n", + "Requirement already satisfied: pydantic<3.0.0,>=2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.10.2)\n", + "Requirement already satisfied: pydantic-settings<3.0.0,>=2.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.6.1)\n", + "Requirement already satisfied: pypdfium2<5.0.0,>=4.30.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.30.0)\n", + "Requirement already satisfied: requests<3.0.0,>=2.32.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.32.3)\n", + "Requirement already satisfied: rtree<2.0.0,>=1.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0)\n", + "Requirement already satisfied: scipy<2.0.0,>=1.14.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.14.1)\n", + "Requirement already satisfied: json-schema-for-humans<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.4)\n", + "Requirement already satisfied: jsonref<2.0.0,>=1.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.0)\n", + "Requirement already satisfied: jsonschema<5.0.0,>=4.16.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.23.0)\n", + "Requirement already satisfied: pyproject-toml<0.0.11,>=0.0.10 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.0.10)\n", + "Requirement already satisfied: Pillow<11.0.0,>=10.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (10.4.0)\n", + "Requirement already satisfied: jsonlines<4.0.0,>=3.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.0)\n", + "Requirement already satisfied: lxml<5.0.0,>=4.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.9.4)\n", + "Requirement already satisfied: mean_average_precision<2022.0.0.0,>=2021.4.26.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2021.4.26.0)\n", + "Requirement already satisfied: onnxruntime<2.0.0,>=1.16.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.20.1)\n", + "Requirement already satisfied: opencv-python-headless<5.0.0.0,>=4.9.0.80 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.10.0.84)\n", + "Requirement already satisfied: torchvision<1,>=0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.19.1)\n", + "Requirement already satisfied: wheel in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.44.0)\n", + "Requirement already satisfied: autopep8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.3.1)\n", + "Requirement already satisfied: coloredlogs in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (15.0.1)\n", + "Requirement already satisfied: interrogate in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.7.0)\n", + "Requirement already satisfied: scikit-learn in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.5.2)\n", + "Requirement already satisfied: prettytable in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.12.0)\n", + "Requirement already satisfied: py in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.11.0)\n", + "Requirement already satisfied: pycodestyle in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.12.1)\n", + "Requirement already satisfied: pylint in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3.1)\n", + "Requirement already satisfied: pyparsing in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.2.0)\n", + "Requirement already satisfied: python-louvain in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.16)\n", + "Requirement already satisfied: PyYAML in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (6.0.2)\n", + "Requirement already satisfied: PyDriller in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.7)\n", + "Requirement already satisfied: pyperclip in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.9.0)\n", + "Requirement already satisfied: pybind11>=2.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fasttext==0.9.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.13.6)\n", + "Requirement already satisfied: setuptools>=0.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fasttext==0.9.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (75.1.0)\n", + "Requirement already satisfied: click in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (8.1.7)\n", + "Requirement already satisfied: joblib in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.4.2)\n", + "Requirement already satisfied: regex>=2021.8.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.11.6)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.2)\n", + "Requirement already satisfied: filelock in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.16.1)\n", + "Requirement already satisfied: typing-extensions>=4.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.12.2)\n", + "Requirement already satisfied: sympy in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.13.3)\n", + "Requirement already satisfied: jinja2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.4)\n", + "Requirement already satisfied: fsspec in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.10.0)\n", + "Requirement already satisfied: packaging>=20.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (24.2)\n", + "Requirement already satisfied: tokenizers<0.19,>=0.14 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.15.2)\n", + "Requirement already satisfied: safetensors>=0.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.5)\n", + "Requirement already satisfied: botocore<1.35.0,>=1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.34.162)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.10.4)\n", + "Requirement already satisfied: ray==2.36.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.36.1)\n", + "Requirement already satisfied: fastapi>=0.110.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.115.5)\n", + "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.0)\n", + "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.29.0)\n", + "Requirement already satisfied: aiosignal in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.1)\n", + "Requirement already satisfied: frozenlist in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.5.0)\n", + "Requirement already satisfied: aiohttp>=3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.11.8)\n", + "Requirement already satisfied: aiohttp-cors in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.7.0)\n", + "Requirement already satisfied: colorful in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.5.6)\n", + "Requirement already satisfied: py-spy>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.0)\n", + "Requirement already satisfied: opencensus in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.11.4)\n", + "Requirement already satisfied: prometheus-client>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.0)\n", + "Requirement already satisfied: smart-open in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (7.0.5)\n", + "Requirement already satisfied: virtualenv!=20.21.1,>=20.0.24 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (20.28.0)\n", + "Requirement already satisfied: grpcio>=1.42.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.68.0)\n", + "Requirement already satisfied: memray in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.14.0)\n", + "Requirement already satisfied: SQLAlchemy>=1.4.49 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.36)\n", + "Requirement already satisfied: dataclasses-json in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.6.7)\n", + "Requirement already satisfied: deprecated>=1.2.9.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.2.15)\n", + "Requirement already satisfied: dirtyjson<2.0.0,>=1.0.8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.8)\n", + "Requirement already satisfied: httpx in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.28.0)\n", + "Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.6.0)\n", + "Requirement already satisfied: tenacity!=8.4.0,<9.0.0,>=8.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (8.5.0)\n", + "Requirement already satisfied: tiktoken>=0.3.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.8.0)\n", + "Requirement already satisfied: typing-inspect>=0.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.0)\n", + "Requirement already satisfied: wrapt in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.17.0)\n", + "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.4.3)\n", + "Requirement already satisfied: attrs>=17.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (24.2.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (6.1.0)\n", + "Requirement already satisfied: propcache>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.0)\n", + "Requirement already satisfied: yarl<2.0,>=1.17.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.18.0)\n", + "Requirement already satisfied: scikit-image in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.24.0)\n", + "Requirement already satisfied: python-bidi in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.6.3)\n", + "Requirement already satisfied: Shapely in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.6)\n", + "Requirement already satisfied: pyclipper in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0.post6)\n", + "Requirement already satisfied: ninja in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.11.1.2)\n", + "Requirement already satisfied: starlette<0.42.0,>=0.40.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fastapi>=0.110.2->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.41.3)\n", + "Requirement already satisfied: MarkupSafe<3.0.0,>=2.1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.1.5)\n", + "Requirement already satisfied: Pygments<3.0.0,>=2.18.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.18.0)\n", + "Requirement already satisfied: htmlmin2<0.2.0,>=0.1.13 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.13)\n", + "Requirement already satisfied: markdown2<3.0.0,>=2.5.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.5.1)\n", + "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from dataclasses-json->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.23.1)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.16.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.10.1)\n", + "Requirement already satisfied: referencing>=0.28.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.16.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.35.1)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.16.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.0)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.1)\n", + "Requirement already satisfied: cycler>=0.10 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.55.0)\n", + "Requirement already satisfied: kiwisolver>=1.3.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.4.7)\n", + "Requirement already satisfied: simplejson>=3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from netwulf<0.2.0,>=0.1.5->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.19.3)\n", + "Requirement already satisfied: flatbuffers in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from onnxruntime<2.0.0,>=1.16.2->docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (24.3.25)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.27.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.27.1)\n", + "Requirement already satisfied: toml in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pyproject-toml<0.0.11,>=0.0.10->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.10.2)\n", + "Requirement already satisfied: six>=1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.16.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests<3.0.0,>=2.32.3->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.4.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests<3.0.0,>=2.32.3->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.10)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests<3.0.0,>=2.32.3->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.2.3)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from rich<14.0.0,>=13.7.0->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.0.0)\n", + "Requirement already satisfied: greenlet!=0.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.1)\n", + "Requirement already satisfied: mypy-extensions>=0.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from typing-inspect>=0.8.0->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.0)\n", + "Requirement already satisfied: soupsieve>1.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from beautifulsoup4->bs4==0.0.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.6)\n", + "Requirement already satisfied: humanfriendly>=9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from coloredlogs->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (10.0)\n", + "Requirement already satisfied: anyio in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.6.2.post1)\n", + "Requirement already satisfied: httpcore==1.* in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.7)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpcore==1.*->httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.14.0)\n", + "Requirement already satisfied: colorama in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from interrogate->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.6)\n", + "Requirement already satisfied: wcwidth in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from prettytable->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.13)\n", + "Requirement already satisfied: gitpython in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.43)\n", + "Requirement already satisfied: types-pytz in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.2.0.20241003)\n", + "Requirement already satisfied: lizard in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.17.13)\n", + "Requirement already satisfied: platformdirs>=2.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.3.6)\n", + "Requirement already satisfied: astroid<=3.4.0-dev0,>=3.3.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3.5)\n", + "Requirement already satisfied: isort!=5.13.0,<6,>=4.2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.13.2)\n", + "Requirement already satisfied: mccabe<0.8,>=0.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.7.0)\n", + "Requirement already satisfied: tomlkit>=0.10.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.13.2)\n", + "Requirement already satisfied: dill>=0.3.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.3.9)\n", + "Requirement already satisfied: threadpoolctl>=3.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-learn->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.5.0)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from sympy->torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0)\n", + "Requirement already satisfied: mdurl~=0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich<14.0.0,>=13.7.0->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.2)\n", + "Requirement already satisfied: sniffio>=1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from anyio->httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.1)\n", + "Requirement already satisfied: distlib<1,>=0.3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.3.9)\n", + "Requirement already satisfied: gitdb<5,>=4.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from gitpython->PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.0.11)\n", + "Requirement already satisfied: textual>=0.41.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.87.1)\n", + "Requirement already satisfied: opencensus-context>=0.1.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.3)\n", + "Requirement already satisfied: google-api-core<3.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.23.0)\n", + "Requirement already satisfied: imageio>=2.33 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-image->easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.36.1)\n", + "Requirement already satisfied: tifffile>=2022.8.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-image->easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.9.20)\n", + "Requirement already satisfied: lazy-loader>=0.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-image->easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4)\n", + "Requirement already satisfied: smmap<6,>=3.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from gitdb<5,>=4.0.1->gitpython->PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.0.1)\n", + "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.66.0)\n", + "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.25.0)\n", + "Requirement already satisfied: google-auth<3.0.dev0,>=2.14.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.36.0)\n", + "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.5.0)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.1)\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.9)\n", + "Requirement already satisfied: linkify-it-py<3,>=1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.3)\n", + "Requirement already satisfied: mdit-py-plugins in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.2)\n", + "Requirement already satisfied: uc-micro-py in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.3)\n", + "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.6.1)\n", + "Using cached argparse-1.4.0-py2.py3-none-any.whl (23 kB)\n", + "Installing collected packages: argparse\n", + "Successfully installed argparse-1.4.0\n", + "Requirement already satisfied: nltk==3.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from -r requirements.txt (line 1)) (3.9.1)\n", + "Requirement already satisfied: transformers==4.38.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from -r requirements.txt (line 2)) (4.38.2)\n", + "Requirement already satisfied: torch<=2.4.1,>=2.2.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from -r requirements.txt (line 3)) (2.4.1)\n", + "Requirement already satisfied: pandas==2.2.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from -r requirements.txt (line 4)) (2.2.2)\n", + "Requirement already satisfied: click in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->-r requirements.txt (line 1)) (8.1.7)\n", + "Requirement already satisfied: joblib in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->-r requirements.txt (line 1)) (1.4.2)\n", + "Requirement already satisfied: regex>=2021.8.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->-r requirements.txt (line 1)) (2024.11.6)\n", + "Requirement already satisfied: tqdm in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->-r requirements.txt (line 1)) (4.67.1)\n", + "Requirement already satisfied: filelock in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (3.16.1)\n", + "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (0.26.3)\n", + "Requirement already satisfied: numpy>=1.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (1.26.4)\n", + "Requirement already satisfied: packaging>=20.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (24.2)\n", + "Requirement already satisfied: pyyaml>=5.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (6.0.2)\n", + "Requirement already satisfied: requests in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (2.32.3)\n", + "Requirement already satisfied: tokenizers<0.19,>=0.14 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (0.15.2)\n", + "Requirement already satisfied: safetensors>=0.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (0.4.5)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas==2.2.2->-r requirements.txt (line 4)) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas==2.2.2->-r requirements.txt (line 4)) (2024.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas==2.2.2->-r requirements.txt (line 4)) (2024.2)\n", + "Requirement already satisfied: typing-extensions>=4.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (4.12.2)\n", + "Requirement already satisfied: sympy in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (1.13.3)\n", + "Requirement already satisfied: networkx in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (3.3)\n", + "Requirement already satisfied: jinja2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (3.1.4)\n", + "Requirement already satisfied: fsspec in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (2024.10.0)\n", + "Requirement already satisfied: six>=1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas==2.2.2->-r requirements.txt (line 4)) (1.16.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jinja2->torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (2.1.5)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->transformers==4.38.2->-r requirements.txt (line 2)) (3.4.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->transformers==4.38.2->-r requirements.txt (line 2)) (3.10)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->transformers==4.38.2->-r requirements.txt (line 2)) (2.2.3)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->transformers==4.38.2->-r requirements.txt (line 2)) (2024.8.30)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from sympy->torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (1.3.0)\n" + ] + } + ], "source": [ "! pip install data-prep-connector\n", "! pip install 'data-prep-toolkit[ray]==0.2.2.dev1'\n", @@ -82,7 +439,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -103,20 +460,35 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ - "hap_script_path = \"./transforms/universal/hap/python/src/hap_local_python.py\"\n", + "from pathlib import Path\n", + "\n", + "notebook_dir = Path().resolve()\n", + "relative_script_dir = '../../../transforms/universal/hap/python/src/hap_local_python.py'\n", + "hap_script_path = (notebook_dir / relative_script_dir).resolve()\n", + "\n", "input_folder = \"./input\"\n", "output_folder = \"./output\"" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Script Path: /Users/aisha/Documents/GitHub/Personal/DPK/transforms/universal/hap/python/src/hap_local_python.py\n", + "Input Folder: ./input\n", + "Output Folder: ./output\n" + ] + } + ], "source": [ "# Ensure the necessary folders exist.\n", "os.makedirs(input_folder, exist_ok=True)\n", @@ -141,9 +513,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found CSV file(s): ['customer_feedback_file.csv']\n", + "Using CSV file: ./input/customer_feedback_file.csv\n" + ] + } + ], "source": [ "csv_files = [f for f in os.listdir(input_folder) if f.endswith(\".csv\")]\n", "\n", @@ -168,9 +549,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CSV file converted to Parquet format at: ./input/data.parquet\n" + ] + } + ], "source": [ "parquet_file_path = os.path.join(input_folder, \"data.parquet\")\n", "df = pd.read_csv(csv_file_path)\n", @@ -182,37 +571,30 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Step 4: Simulate Command-Line Arguments for HAP Parameters" + "### Step 4: Define HAP Parameters" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ - "\n", - "# Clear sys.argv first to avoid conflicts (reset arguments list)\n", - "sys.argv = [\n", - " \"hap_local_python.py\", # Script name\n", - " \"--input_folder\", input_folder, # Correct input folder\n", - " \"--output_folder\", output_folder, # Correct output folder\n", - " \"--model_name_or_path\", hap_params[\"model_name_or_path\"],\n", - " \"--annotation_column\", hap_params[\"annotation_column\"],\n", - " \"--doc_text_column\", hap_params[\"doc_text_column\"],\n", - " \"--inference_engine\", hap_params[\"inference_engine\"],\n", - " \"--max_length\", str(hap_params[\"max_length\"]),\n", - " \"--batch_size\", str(hap_params[\"batch_size\"]),\n", - "]\n", - "\n", - "print(f\"Command-line arguments: {sys.argv}\")" + "hap_params = {\n", + " \"model_name_or_path\": \"ibm-granite/granite-guardian-hap-38m\", # Default model name\n", + " \"annotation_column\": \"hap_score\", # Output column for HAP scores\n", + " \"doc_text_column\": \"Customer Feedback\", # Input column containing document text\n", + " \"inference_engine\": \"CPU\", # Inference engine (CPU or GPU)\n", + " \"max_length\": 512, # Maximum token length\n", + " \"batch_size\": 128, # Batch size\n", + "}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Step 5: Run the Transform with Simulated Arguments\n", + "### Step 5: Run the Transform with defined HAP Paramters\n", "\n", "This cell executes the HAP transformation script:\n", "- `--input_file`: Path to your input CSV/Parquet file.\n", @@ -227,9 +609,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Transform completed successfully.\n", + "Processing batch: 0/1\n", + "Processing batch: 1/1\n", + " Customer Name Date ... Unnamed: 13 hap_score\n", + "0 Ethan Johnson 11/7/24 ... NaN 0.000195\n", + "1 Sophia Wilson 11/8/24 ... NaN 0.000153\n", + "2 Liam Smith 11/6/24 ... NaN 0.000169\n", + "3 Isabella Martinez 11/9/24 ... NaN 0.000158\n", + "4 Noah White 11/10/24 ... NaN 0.000875\n", + ".. ... ... ... ... ...\n", + "60 Karen Brooks 1/2/25 ... NaN 0.000150\n", + "61 David King 1/3/25 ... NaN 0.579251\n", + "62 Angela Hill 1/4/25 ... NaN 0.000384\n", + "63 Richard Young 1/5/25 ... NaN 0.000285\n", + "64 Lisa Green 1/6/25 ... NaN 0.052517\n", + "\n", + "[65 rows x 15 columns]\n", + "\n" + ] + } + ], "source": [ "# Copy the current environment variables\n", "env = os.environ.copy()\n", @@ -283,9 +690,132 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Complete Output Parquet File Path: ./output/data.parquet\n", + "Filtered Output (only HAP score and document text):\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Customer Feedbackhap_score
0Rating: 4 Comments: \"Service was prompt, but ...0.000195
1Rating: 5 Comments: \"Great help from Peter! H...0.000153
2Rating: 3 Comments: \"The service was quick, b...0.000169
3Rating: 5 Comments: \"Excellent service and ad...0.000158
4Rating: 2 Comments: \"I’m really frustrated. T...0.000875
.........
60Rating: 3 Comments: \"This is not what I expec...0.000150
61Rating: 1 Comments: \"This is insane. I’ve onl...0.579251
62Rating: 4 Comments: \"I need this fixed. I can’...0.000384
63Rating: 2 Comments: \"I’m so done with this ma...0.000285
64Rating: 3 Comments: \"I’ve leveled the machine...0.052517
\n", + "

65 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " Customer Feedback hap_score\n", + "0 Rating: 4 Comments: \"Service was prompt, but ... 0.000195\n", + "1 Rating: 5 Comments: \"Great help from Peter! H... 0.000153\n", + "2 Rating: 3 Comments: \"The service was quick, b... 0.000169\n", + "3 Rating: 5 Comments: \"Excellent service and ad... 0.000158\n", + "4 Rating: 2 Comments: \"I’m really frustrated. T... 0.000875\n", + ".. ... ...\n", + "60 Rating: 3 Comments: \"This is not what I expec... 0.000150\n", + "61 Rating: 1 Comments: \"This is insane. I’ve onl... 0.579251\n", + "62 Rating: 4 Comments: \"I need this fixed. I can’... 0.000384\n", + "63 Rating: 2 Comments: \"I’m so done with this ma... 0.000285\n", + "64 Rating: 3 Comments: \"I’ve leveled the machine... 0.052517\n", + "\n", + "[65 rows x 2 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Complete output saved to: ./output/hap_complete_output.csv\n", + "Filtered output saved to: ./output/hap_filtered_output.csv\n" + ] + } + ], "source": [ "import os\n", "import pandas as pd\n", diff --git a/examples/notebooks/hap/output/metadata.json b/examples/notebooks/hap/output/metadata.json index b9f7c11a3..f5df68315 100644 --- a/examples/notebooks/hap/output/metadata.json +++ b/examples/notebooks/hap/output/metadata.json @@ -5,8 +5,8 @@ "job name": "hap", "job type": "pure python", "job id": "job_id", - "start_time": "2024-11-29 13:41:19", - "end_time": "2024-11-29 13:41:21", + "start_time": "2024-12-05 09:25:45", + "end_time": "2024-12-05 09:25:48", "status": "success" }, "code": { @@ -30,18 +30,18 @@ "num_processors": 0 }, "execution_stats": { - "cpus": 35.7, + "cpus": 190.1, "gpus": 0, - "memory": 9.43, + "memory": 10.46, "object_store": 0, - "execution time, min": 0.037 + "execution time, min": 0.035 }, "job_output_stats": { "source_files": 1, "source_size": 47309, "result_files": 1, "result_size": 40282, - "processing_time": 0.419, + "processing_time": 0.439, "source_doc_count": 65, "result_doc_count": 65 }, From f2f9c9da6b732b278325a1e23b97894bc7790c20 Mon Sep 17 00:00:00 2001 From: Aisha Mohammed Farooq Darga Date: Thu, 5 Dec 2024 09:29:25 +0530 Subject: [PATCH 17/31] Cleared output cells Signed-off-by: Aisha Mohammed Farooq Darga --- .../hap/generate_hap_score_csv.ipynb | 560 +----------------- 1 file changed, 15 insertions(+), 545 deletions(-) diff --git a/examples/notebooks/hap/generate_hap_score_csv.ipynb b/examples/notebooks/hap/generate_hap_score_csv.ipynb index 09621666d..a44935a90 100644 --- a/examples/notebooks/hap/generate_hap_score_csv.ipynb +++ b/examples/notebooks/hap/generate_hap_score_csv.ipynb @@ -65,364 +65,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: data-prep-connector in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (0.2.3)\n", - "Requirement already satisfied: scrapy>=2.11.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-connector) (2.12.0)\n", - "Requirement already satisfied: pydantic>=2.8.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-connector) (2.10.2)\n", - "Requirement already satisfied: tldextract>=5.1.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-connector) (5.1.3)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic>=2.8.1->data-prep-connector) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.27.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic>=2.8.1->data-prep-connector) (2.27.1)\n", - "Requirement already satisfied: typing-extensions>=4.12.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic>=2.8.1->data-prep-connector) (4.12.2)\n", - "Requirement already satisfied: Twisted>=21.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.11.0)\n", - "Requirement already satisfied: cryptography>=37.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (44.0.0)\n", - "Requirement already satisfied: cssselect>=0.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.2.0)\n", - "Requirement already satisfied: itemloaders>=1.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.3.2)\n", - "Requirement already satisfied: parsel>=1.5.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.9.1)\n", - "Requirement already satisfied: pyOpenSSL>=22.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.3.0)\n", - "Requirement already satisfied: queuelib>=1.4.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.7.0)\n", - "Requirement already satisfied: service-identity>=18.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.2.0)\n", - "Requirement already satisfied: w3lib>=1.17.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (2.2.1)\n", - "Requirement already satisfied: zope.interface>=5.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (7.2)\n", - "Requirement already satisfied: protego>=0.1.15 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (0.3.1)\n", - "Requirement already satisfied: itemadapter>=0.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (0.10.0)\n", - "Requirement already satisfied: packaging in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.2)\n", - "Requirement already satisfied: lxml>=4.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (4.9.4)\n", - "Requirement already satisfied: defusedxml>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (0.7.1)\n", - "Requirement already satisfied: PyDispatcher>=2.0.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (2.0.7)\n", - "Requirement already satisfied: idna in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (3.10)\n", - "Requirement already satisfied: requests>=2.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (2.32.3)\n", - "Requirement already satisfied: requests-file>=1.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (2.1.0)\n", - "Requirement already satisfied: filelock>=3.0.8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (3.16.1)\n", - "Requirement already satisfied: cffi>=1.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from cryptography>=37.0.0->scrapy>=2.11.2->data-prep-connector) (1.17.1)\n", - "Requirement already satisfied: jmespath>=0.9.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from itemloaders>=1.0.1->scrapy>=2.11.2->data-prep-connector) (1.0.1)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.1.0->tldextract>=5.1.2->data-prep-connector) (3.4.0)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.1.0->tldextract>=5.1.2->data-prep-connector) (2.2.3)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.1.0->tldextract>=5.1.2->data-prep-connector) (2024.8.30)\n", - "Requirement already satisfied: attrs>=19.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from service-identity>=18.1.0->scrapy>=2.11.2->data-prep-connector) (24.2.0)\n", - "Requirement already satisfied: pyasn1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from service-identity>=18.1.0->scrapy>=2.11.2->data-prep-connector) (0.6.1)\n", - "Requirement already satisfied: pyasn1-modules in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from service-identity>=18.1.0->scrapy>=2.11.2->data-prep-connector) (0.4.1)\n", - "Requirement already satisfied: automat>=24.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (24.8.1)\n", - "Requirement already satisfied: constantly>=15.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (23.10.4)\n", - "Requirement already satisfied: hyperlink>=17.1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (21.0.0)\n", - "Requirement already satisfied: incremental>=24.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (24.7.2)\n", - "Requirement already satisfied: setuptools in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from zope.interface>=5.1.0->scrapy>=2.11.2->data-prep-connector) (75.1.0)\n", - "Requirement already satisfied: pycparser in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from cffi>=1.12->cryptography>=37.0.0->scrapy>=2.11.2->data-prep-connector) (2.22)\n", - "Requirement already satisfied: data-prep-toolkit==0.2.2.dev1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]==0.2.2.dev1) (0.2.2.dev1)\n", - "Requirement already satisfied: numpy<1.29.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.26.4)\n", - "Requirement already satisfied: pyarrow==16.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (16.1.0)\n", - "Requirement already satisfied: boto3==1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.34.69)\n", - "Collecting argparse (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1)\n", - " Using cached argparse-1.4.0-py2.py3-none-any.whl.metadata (2.8 kB)\n", - "Requirement already satisfied: mmh3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (4.1.0)\n", - "Requirement already satisfied: psutil in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (5.9.0)\n", - "Requirement already satisfied: ray==2.36.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.36.1)\n", - "Requirement already satisfied: fastapi>=0.110.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]==0.2.2.dev1) (0.115.5)\n", - "Requirement already satisfied: pillow>=10.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]==0.2.2.dev1) (10.4.0)\n", - "Requirement already satisfied: botocore<1.35.0,>=1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.34.162)\n", - "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.0.1)\n", - "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (0.10.4)\n", - "Requirement already satisfied: click>=7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (8.1.7)\n", - "Requirement already satisfied: filelock in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.16.1)\n", - "Requirement already satisfied: jsonschema in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (4.23.0)\n", - "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.1.0)\n", - "Requirement already satisfied: packaging in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (24.2)\n", - "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (5.29.0)\n", - "Requirement already satisfied: pyyaml in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (6.0.2)\n", - "Requirement already satisfied: aiosignal in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.3.1)\n", - "Requirement already satisfied: frozenlist in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.5.0)\n", - "Requirement already satisfied: requests in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.32.3)\n", - "Requirement already satisfied: aiohttp>=3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.11.8)\n", - "Requirement already satisfied: aiohttp-cors in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.7.0)\n", - "Requirement already satisfied: colorful in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.5.6)\n", - "Requirement already satisfied: py-spy>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.4.0)\n", - "Requirement already satisfied: opencensus in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.11.4)\n", - "Requirement already satisfied: pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.10.2)\n", - "Requirement already satisfied: prometheus-client>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.21.0)\n", - "Requirement already satisfied: smart-open in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (7.0.5)\n", - "Requirement already satisfied: virtualenv!=20.21.1,>=20.0.24 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (20.28.0)\n", - "Requirement already satisfied: grpcio>=1.42.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.68.0)\n", - "Requirement already satisfied: memray in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.14.0)\n", - "Requirement already satisfied: starlette<0.42.0,>=0.40.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (0.41.3)\n", - "Requirement already satisfied: typing-extensions>=4.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (4.12.2)\n", - "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.4.3)\n", - "Requirement already satisfied: attrs>=17.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (24.2.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (6.1.0)\n", - "Requirement already satisfied: propcache>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.2.0)\n", - "Requirement already satisfied: yarl<2.0,>=1.17.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.18.0)\n", - "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from botocore<1.35.0,>=1.34.69->boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (2.9.0.post0)\n", - "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from botocore<1.35.0,>=1.34.69->boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (2.2.3)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.27.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.27.1)\n", - "Requirement already satisfied: anyio<5,>=3.4.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from starlette<0.42.0,>=0.40.0->fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (4.6.2.post1)\n", - "Requirement already satisfied: distlib<1,>=0.3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.3.9)\n", - "Requirement already satisfied: platformdirs<5,>=3.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (4.3.6)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2024.10.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.35.1)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.21.0)\n", - "Requirement already satisfied: jinja2>=2.9 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.1.4)\n", - "Requirement already satisfied: rich>=11.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (13.9.4)\n", - "Requirement already satisfied: textual>=0.41.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.87.1)\n", - "Requirement already satisfied: opencensus-context>=0.1.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.1.3)\n", - "Requirement already satisfied: six~=1.16 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.16.0)\n", - "Requirement already satisfied: google-api-core<3.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.23.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.4.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.10)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2024.8.30)\n", - "Requirement already satisfied: wrapt in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from smart-open->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.17.0)\n", - "Requirement already satisfied: sniffio>=1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from anyio<5,>=3.4.0->starlette<0.42.0,>=0.40.0->fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (1.3.1)\n", - "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.66.0)\n", - "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.25.0)\n", - "Requirement already satisfied: google-auth<3.0.dev0,>=2.14.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.36.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jinja2>=2.9->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.1.5)\n", - "Requirement already satisfied: markdown-it-py>=2.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from rich>=11.2.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.0.0)\n", - "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from rich>=11.2.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.18.0)\n", - "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (5.5.0)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.4.1)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (4.9)\n", - "Requirement already satisfied: mdurl~=0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich>=11.2.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.1.2)\n", - "Requirement already satisfied: linkify-it-py<3,>=1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.0.3)\n", - "Requirement already satisfied: mdit-py-plugins in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.4.2)\n", - "Requirement already satisfied: uc-micro-py in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.0.3)\n", - "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.6.1)\n", - "Using cached argparse-1.4.0-py2.py3-none-any.whl (23 kB)\n", - "Installing collected packages: argparse\n", - "Successfully installed argparse-1.4.0\n", - "Requirement already satisfied: data-prep-toolkit-transforms==0.2.2.dev1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.2.dev1)\n", - "Requirement already satisfied: data-prep-toolkit>=0.2.2.dev1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.2.dev1)\n", - "Requirement already satisfied: bs4==0.0.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.0.2)\n", - "Requirement already satisfied: transformers==4.38.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.38.2)\n", - "Requirement already satisfied: parameterized in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.0)\n", - "Requirement already satisfied: pandas in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.2.2)\n", - "Requirement already satisfied: docling-core==1.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0)\n", - "Requirement already satisfied: llama-index-core<0.12.0,>=0.11.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.11.23)\n", - "Requirement already satisfied: fasttext==0.9.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.2)\n", - "Requirement already satisfied: langcodes==3.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3.0)\n", - "Requirement already satisfied: huggingface-hub<1.0.0,>=0.21.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.26.3)\n", - "Requirement already satisfied: numpy==1.26.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.26.4)\n", - "Requirement already satisfied: sentence-transformers==3.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.0.1)\n", - "Requirement already satisfied: docling-ibm-models==1.1.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.7)\n", - "Requirement already satisfied: deepsearch-glm==0.21.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.0)\n", - "Requirement already satisfied: docling==1.11.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.11.0)\n", - "Requirement already satisfied: filetype<2.0.0,>=1.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.2.0)\n", - "Requirement already satisfied: nltk==3.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.9.1)\n", - "Requirement already satisfied: torch==2.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.4.1)\n", - "Requirement already satisfied: mmh3>=4.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.1.0)\n", - "Requirement already satisfied: xxhash==3.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.4.1)\n", - "Requirement already satisfied: duckdb>=0.10.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.3)\n", - "Requirement already satisfied: networkx==3.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3)\n", - "Requirement already satisfied: colorlog==6.8.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (6.8.2)\n", - "Requirement already satisfied: func-timeout==4.3.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.3.5)\n", - "Requirement already satisfied: emerge-viz==2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.0)\n", - "Requirement already satisfied: beautifulsoup4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from bs4==0.0.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.12.3)\n", - "Requirement already satisfied: pyarrow==16.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (16.1.0)\n", - "Requirement already satisfied: boto3==1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.34.69)\n", - "Collecting argparse (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1)\n", - " Using cached argparse-1.4.0-py2.py3-none-any.whl.metadata (2.8 kB)\n", - "Requirement already satisfied: psutil in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.9.0)\n", - "Requirement already satisfied: docutils!=0.21 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.2)\n", - "Requirement already satisfied: matplotlib<4.0.0,>=3.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.9.2)\n", - "Requirement already satisfied: netwulf<0.2.0,>=0.1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.5)\n", - "Requirement already satisfied: numerize<0.13,>=0.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.12)\n", - "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.1)\n", - "Requirement already satisfied: rich<14.0.0,>=13.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (13.9.4)\n", - "Requirement already satisfied: tabulate>=0.8.9 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.0)\n", - "Requirement already satisfied: tqdm<5.0.0,>=4.64.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.67.1)\n", - "Requirement already satisfied: certifi>=2024.7.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.8.30)\n", - "Requirement already satisfied: docling-parse<2.0.0,>=1.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.6.2)\n", - "Requirement already satisfied: easyocr<2.0,>=1.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.7.2)\n", - "Requirement already satisfied: pydantic<3.0.0,>=2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.10.2)\n", - "Requirement already satisfied: pydantic-settings<3.0.0,>=2.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.6.1)\n", - "Requirement already satisfied: pypdfium2<5.0.0,>=4.30.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.30.0)\n", - "Requirement already satisfied: requests<3.0.0,>=2.32.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.32.3)\n", - "Requirement already satisfied: rtree<2.0.0,>=1.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0)\n", - "Requirement already satisfied: scipy<2.0.0,>=1.14.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.14.1)\n", - "Requirement already satisfied: json-schema-for-humans<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.4)\n", - "Requirement already satisfied: jsonref<2.0.0,>=1.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.0)\n", - "Requirement already satisfied: jsonschema<5.0.0,>=4.16.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.23.0)\n", - "Requirement already satisfied: pyproject-toml<0.0.11,>=0.0.10 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.0.10)\n", - "Requirement already satisfied: Pillow<11.0.0,>=10.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (10.4.0)\n", - "Requirement already satisfied: jsonlines<4.0.0,>=3.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.0)\n", - "Requirement already satisfied: lxml<5.0.0,>=4.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.9.4)\n", - "Requirement already satisfied: mean_average_precision<2022.0.0.0,>=2021.4.26.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2021.4.26.0)\n", - "Requirement already satisfied: onnxruntime<2.0.0,>=1.16.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.20.1)\n", - "Requirement already satisfied: opencv-python-headless<5.0.0.0,>=4.9.0.80 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.10.0.84)\n", - "Requirement already satisfied: torchvision<1,>=0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.19.1)\n", - "Requirement already satisfied: wheel in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.44.0)\n", - "Requirement already satisfied: autopep8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.3.1)\n", - "Requirement already satisfied: coloredlogs in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (15.0.1)\n", - "Requirement already satisfied: interrogate in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.7.0)\n", - "Requirement already satisfied: scikit-learn in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.5.2)\n", - "Requirement already satisfied: prettytable in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.12.0)\n", - "Requirement already satisfied: py in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.11.0)\n", - "Requirement already satisfied: pycodestyle in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.12.1)\n", - "Requirement already satisfied: pylint in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3.1)\n", - "Requirement already satisfied: pyparsing in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.2.0)\n", - "Requirement already satisfied: python-louvain in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.16)\n", - "Requirement already satisfied: PyYAML in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (6.0.2)\n", - "Requirement already satisfied: PyDriller in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.7)\n", - "Requirement already satisfied: pyperclip in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.9.0)\n", - "Requirement already satisfied: pybind11>=2.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fasttext==0.9.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.13.6)\n", - "Requirement already satisfied: setuptools>=0.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fasttext==0.9.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (75.1.0)\n", - "Requirement already satisfied: click in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (8.1.7)\n", - "Requirement already satisfied: joblib in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.4.2)\n", - "Requirement already satisfied: regex>=2021.8.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.11.6)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.9.0.post0)\n", - "Requirement already satisfied: pytz>=2020.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.2)\n", - "Requirement already satisfied: tzdata>=2022.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.2)\n", - "Requirement already satisfied: filelock in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.16.1)\n", - "Requirement already satisfied: typing-extensions>=4.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.12.2)\n", - "Requirement already satisfied: sympy in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.13.3)\n", - "Requirement already satisfied: jinja2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.4)\n", - "Requirement already satisfied: fsspec in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.10.0)\n", - "Requirement already satisfied: packaging>=20.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (24.2)\n", - "Requirement already satisfied: tokenizers<0.19,>=0.14 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.15.2)\n", - "Requirement already satisfied: safetensors>=0.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.5)\n", - "Requirement already satisfied: botocore<1.35.0,>=1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.34.162)\n", - "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.1)\n", - "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.10.4)\n", - "Requirement already satisfied: ray==2.36.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.36.1)\n", - "Requirement already satisfied: fastapi>=0.110.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.115.5)\n", - "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.0)\n", - "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.29.0)\n", - "Requirement already satisfied: aiosignal in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.1)\n", - "Requirement already satisfied: frozenlist in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.5.0)\n", - "Requirement already satisfied: aiohttp>=3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.11.8)\n", - "Requirement already satisfied: aiohttp-cors in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.7.0)\n", - "Requirement already satisfied: colorful in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.5.6)\n", - "Requirement already satisfied: py-spy>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.0)\n", - "Requirement already satisfied: opencensus in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.11.4)\n", - "Requirement already satisfied: prometheus-client>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.0)\n", - "Requirement already satisfied: smart-open in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (7.0.5)\n", - "Requirement already satisfied: virtualenv!=20.21.1,>=20.0.24 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (20.28.0)\n", - "Requirement already satisfied: grpcio>=1.42.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.68.0)\n", - "Requirement already satisfied: memray in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.14.0)\n", - "Requirement already satisfied: SQLAlchemy>=1.4.49 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.36)\n", - "Requirement already satisfied: dataclasses-json in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.6.7)\n", - "Requirement already satisfied: deprecated>=1.2.9.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.2.15)\n", - "Requirement already satisfied: dirtyjson<2.0.0,>=1.0.8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.8)\n", - "Requirement already satisfied: httpx in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.28.0)\n", - "Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.6.0)\n", - "Requirement already satisfied: tenacity!=8.4.0,<9.0.0,>=8.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (8.5.0)\n", - "Requirement already satisfied: tiktoken>=0.3.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.8.0)\n", - "Requirement already satisfied: typing-inspect>=0.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.0)\n", - "Requirement already satisfied: wrapt in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.17.0)\n", - "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.4.3)\n", - "Requirement already satisfied: attrs>=17.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (24.2.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (6.1.0)\n", - "Requirement already satisfied: propcache>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.0)\n", - "Requirement already satisfied: yarl<2.0,>=1.17.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.18.0)\n", - "Requirement already satisfied: scikit-image in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.24.0)\n", - "Requirement already satisfied: python-bidi in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.6.3)\n", - "Requirement already satisfied: Shapely in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.6)\n", - "Requirement already satisfied: pyclipper in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0.post6)\n", - "Requirement already satisfied: ninja in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.11.1.2)\n", - "Requirement already satisfied: starlette<0.42.0,>=0.40.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fastapi>=0.110.2->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.41.3)\n", - "Requirement already satisfied: MarkupSafe<3.0.0,>=2.1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.1.5)\n", - "Requirement already satisfied: Pygments<3.0.0,>=2.18.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.18.0)\n", - "Requirement already satisfied: htmlmin2<0.2.0,>=0.1.13 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.13)\n", - "Requirement already satisfied: markdown2<3.0.0,>=2.5.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.5.1)\n", - "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from dataclasses-json->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.23.1)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.16.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.10.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.16.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.35.1)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.16.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.0)\n", - "Requirement already satisfied: contourpy>=1.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.1)\n", - "Requirement already satisfied: cycler>=0.10 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.12.1)\n", - "Requirement already satisfied: fonttools>=4.22.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.55.0)\n", - "Requirement already satisfied: kiwisolver>=1.3.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.4.7)\n", - "Requirement already satisfied: simplejson>=3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from netwulf<0.2.0,>=0.1.5->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.19.3)\n", - "Requirement already satisfied: flatbuffers in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from onnxruntime<2.0.0,>=1.16.2->docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (24.3.25)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.27.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.27.1)\n", - "Requirement already satisfied: toml in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pyproject-toml<0.0.11,>=0.0.10->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.10.2)\n", - "Requirement already satisfied: six>=1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.16.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests<3.0.0,>=2.32.3->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.4.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests<3.0.0,>=2.32.3->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.10)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests<3.0.0,>=2.32.3->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.2.3)\n", - "Requirement already satisfied: markdown-it-py>=2.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from rich<14.0.0,>=13.7.0->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.0.0)\n", - "Requirement already satisfied: greenlet!=0.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.1)\n", - "Requirement already satisfied: mypy-extensions>=0.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from typing-inspect>=0.8.0->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.0)\n", - "Requirement already satisfied: soupsieve>1.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from beautifulsoup4->bs4==0.0.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.6)\n", - "Requirement already satisfied: humanfriendly>=9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from coloredlogs->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (10.0)\n", - "Requirement already satisfied: anyio in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.6.2.post1)\n", - "Requirement already satisfied: httpcore==1.* in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.7)\n", - "Requirement already satisfied: h11<0.15,>=0.13 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpcore==1.*->httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.14.0)\n", - "Requirement already satisfied: colorama in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from interrogate->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.6)\n", - "Requirement already satisfied: wcwidth in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from prettytable->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.13)\n", - "Requirement already satisfied: gitpython in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.43)\n", - "Requirement already satisfied: types-pytz in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.2.0.20241003)\n", - "Requirement already satisfied: lizard in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.17.13)\n", - "Requirement already satisfied: platformdirs>=2.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.3.6)\n", - "Requirement already satisfied: astroid<=3.4.0-dev0,>=3.3.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3.5)\n", - "Requirement already satisfied: isort!=5.13.0,<6,>=4.2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.13.2)\n", - "Requirement already satisfied: mccabe<0.8,>=0.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.7.0)\n", - "Requirement already satisfied: tomlkit>=0.10.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.13.2)\n", - "Requirement already satisfied: dill>=0.3.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.3.9)\n", - "Requirement already satisfied: threadpoolctl>=3.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-learn->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.5.0)\n", - "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from sympy->torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0)\n", - "Requirement already satisfied: mdurl~=0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich<14.0.0,>=13.7.0->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.2)\n", - "Requirement already satisfied: sniffio>=1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from anyio->httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.1)\n", - "Requirement already satisfied: distlib<1,>=0.3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.3.9)\n", - "Requirement already satisfied: gitdb<5,>=4.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from gitpython->PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.0.11)\n", - "Requirement already satisfied: textual>=0.41.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.87.1)\n", - "Requirement already satisfied: opencensus-context>=0.1.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.3)\n", - "Requirement already satisfied: google-api-core<3.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.23.0)\n", - "Requirement already satisfied: imageio>=2.33 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-image->easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.36.1)\n", - "Requirement already satisfied: tifffile>=2022.8.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-image->easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.9.20)\n", - "Requirement already satisfied: lazy-loader>=0.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-image->easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4)\n", - "Requirement already satisfied: smmap<6,>=3.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from gitdb<5,>=4.0.1->gitpython->PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.0.1)\n", - "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.66.0)\n", - "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.25.0)\n", - "Requirement already satisfied: google-auth<3.0.dev0,>=2.14.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.36.0)\n", - "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.5.0)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.1)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.9)\n", - "Requirement already satisfied: linkify-it-py<3,>=1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.3)\n", - "Requirement already satisfied: mdit-py-plugins in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.2)\n", - "Requirement already satisfied: uc-micro-py in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.3)\n", - "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.6.1)\n", - "Using cached argparse-1.4.0-py2.py3-none-any.whl (23 kB)\n", - "Installing collected packages: argparse\n", - "Successfully installed argparse-1.4.0\n", - "Requirement already satisfied: nltk==3.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from -r requirements.txt (line 1)) (3.9.1)\n", - "Requirement already satisfied: transformers==4.38.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from -r requirements.txt (line 2)) (4.38.2)\n", - "Requirement already satisfied: torch<=2.4.1,>=2.2.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from -r requirements.txt (line 3)) (2.4.1)\n", - "Requirement already satisfied: pandas==2.2.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from -r requirements.txt (line 4)) (2.2.2)\n", - "Requirement already satisfied: click in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->-r requirements.txt (line 1)) (8.1.7)\n", - "Requirement already satisfied: joblib in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->-r requirements.txt (line 1)) (1.4.2)\n", - "Requirement already satisfied: regex>=2021.8.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->-r requirements.txt (line 1)) (2024.11.6)\n", - "Requirement already satisfied: tqdm in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->-r requirements.txt (line 1)) (4.67.1)\n", - "Requirement already satisfied: filelock in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (3.16.1)\n", - "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (0.26.3)\n", - "Requirement already satisfied: numpy>=1.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (1.26.4)\n", - "Requirement already satisfied: packaging>=20.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (24.2)\n", - "Requirement already satisfied: pyyaml>=5.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (6.0.2)\n", - "Requirement already satisfied: requests in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (2.32.3)\n", - "Requirement already satisfied: tokenizers<0.19,>=0.14 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (0.15.2)\n", - "Requirement already satisfied: safetensors>=0.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->-r requirements.txt (line 2)) (0.4.5)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas==2.2.2->-r requirements.txt (line 4)) (2.9.0.post0)\n", - "Requirement already satisfied: pytz>=2020.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas==2.2.2->-r requirements.txt (line 4)) (2024.2)\n", - "Requirement already satisfied: tzdata>=2022.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas==2.2.2->-r requirements.txt (line 4)) (2024.2)\n", - "Requirement already satisfied: typing-extensions>=4.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (4.12.2)\n", - "Requirement already satisfied: sympy in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (1.13.3)\n", - "Requirement already satisfied: networkx in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (3.3)\n", - "Requirement already satisfied: jinja2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (3.1.4)\n", - "Requirement already satisfied: fsspec in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (2024.10.0)\n", - "Requirement already satisfied: six>=1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas==2.2.2->-r requirements.txt (line 4)) (1.16.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jinja2->torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (2.1.5)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->transformers==4.38.2->-r requirements.txt (line 2)) (3.4.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->transformers==4.38.2->-r requirements.txt (line 2)) (3.10)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->transformers==4.38.2->-r requirements.txt (line 2)) (2.2.3)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->transformers==4.38.2->-r requirements.txt (line 2)) (2024.8.30)\n", - "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from sympy->torch<=2.4.1,>=2.2.2->-r requirements.txt (line 3)) (1.3.0)\n" - ] - } - ], + "outputs": [], "source": [ "! pip install data-prep-connector\n", "! pip install 'data-prep-toolkit[ray]==0.2.2.dev1'\n", @@ -439,7 +84,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -460,7 +105,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -476,19 +121,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Script Path: /Users/aisha/Documents/GitHub/Personal/DPK/transforms/universal/hap/python/src/hap_local_python.py\n", - "Input Folder: ./input\n", - "Output Folder: ./output\n" - ] - } - ], + "outputs": [], "source": [ "# Ensure the necessary folders exist.\n", "os.makedirs(input_folder, exist_ok=True)\n", @@ -513,18 +148,9 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found CSV file(s): ['customer_feedback_file.csv']\n", - "Using CSV file: ./input/customer_feedback_file.csv\n" - ] - } - ], + "outputs": [], "source": [ "csv_files = [f for f in os.listdir(input_folder) if f.endswith(\".csv\")]\n", "\n", @@ -549,17 +175,9 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CSV file converted to Parquet format at: ./input/data.parquet\n" - ] - } - ], + "outputs": [], "source": [ "parquet_file_path = os.path.join(input_folder, \"data.parquet\")\n", "df = pd.read_csv(csv_file_path)\n", @@ -576,7 +194,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -609,34 +227,9 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Transform completed successfully.\n", - "Processing batch: 0/1\n", - "Processing batch: 1/1\n", - " Customer Name Date ... Unnamed: 13 hap_score\n", - "0 Ethan Johnson 11/7/24 ... NaN 0.000195\n", - "1 Sophia Wilson 11/8/24 ... NaN 0.000153\n", - "2 Liam Smith 11/6/24 ... NaN 0.000169\n", - "3 Isabella Martinez 11/9/24 ... NaN 0.000158\n", - "4 Noah White 11/10/24 ... NaN 0.000875\n", - ".. ... ... ... ... ...\n", - "60 Karen Brooks 1/2/25 ... NaN 0.000150\n", - "61 David King 1/3/25 ... NaN 0.579251\n", - "62 Angela Hill 1/4/25 ... NaN 0.000384\n", - "63 Richard Young 1/5/25 ... NaN 0.000285\n", - "64 Lisa Green 1/6/25 ... NaN 0.052517\n", - "\n", - "[65 rows x 15 columns]\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "# Copy the current environment variables\n", "env = os.environ.copy()\n", @@ -690,132 +283,9 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Complete Output Parquet File Path: ./output/data.parquet\n", - "Filtered Output (only HAP score and document text):\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Customer Feedbackhap_score
0Rating: 4 Comments: \"Service was prompt, but ...0.000195
1Rating: 5 Comments: \"Great help from Peter! H...0.000153
2Rating: 3 Comments: \"The service was quick, b...0.000169
3Rating: 5 Comments: \"Excellent service and ad...0.000158
4Rating: 2 Comments: \"I’m really frustrated. T...0.000875
.........
60Rating: 3 Comments: \"This is not what I expec...0.000150
61Rating: 1 Comments: \"This is insane. I’ve onl...0.579251
62Rating: 4 Comments: \"I need this fixed. I can’...0.000384
63Rating: 2 Comments: \"I’m so done with this ma...0.000285
64Rating: 3 Comments: \"I’ve leveled the machine...0.052517
\n", - "

65 rows × 2 columns

\n", - "
" - ], - "text/plain": [ - " Customer Feedback hap_score\n", - "0 Rating: 4 Comments: \"Service was prompt, but ... 0.000195\n", - "1 Rating: 5 Comments: \"Great help from Peter! H... 0.000153\n", - "2 Rating: 3 Comments: \"The service was quick, b... 0.000169\n", - "3 Rating: 5 Comments: \"Excellent service and ad... 0.000158\n", - "4 Rating: 2 Comments: \"I’m really frustrated. T... 0.000875\n", - ".. ... ...\n", - "60 Rating: 3 Comments: \"This is not what I expec... 0.000150\n", - "61 Rating: 1 Comments: \"This is insane. I’ve onl... 0.579251\n", - "62 Rating: 4 Comments: \"I need this fixed. I can’... 0.000384\n", - "63 Rating: 2 Comments: \"I’m so done with this ma... 0.000285\n", - "64 Rating: 3 Comments: \"I’ve leveled the machine... 0.052517\n", - "\n", - "[65 rows x 2 columns]" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Complete output saved to: ./output/hap_complete_output.csv\n", - "Filtered output saved to: ./output/hap_filtered_output.csv\n" - ] - } - ], + "outputs": [], "source": [ "import os\n", "import pandas as pd\n", From d790bb727e303e3636beb5cceb85081d76f1e5ad Mon Sep 17 00:00:00 2001 From: Sujee Maniyam Date: Thu, 5 Dec 2024 11:08:50 -0800 Subject: [PATCH 18/31] Removed HTML input files that were checkedin Signed-off-by: Sujee Maniyam --- examples/notebooks/html-processing/.gitignore | 4 +-- .../input2/thealliance.ai/about-aia.html | 25 ------------------- .../input2/thealliance.ai/aia-members.html | 25 ------------------- .../thealliance.ai/become-a-collaborator.html | 25 ------------------- .../input2/thealliance.ai/blog.html | 25 ------------------- .../input2/thealliance.ai/community.html | 25 ------------------- .../input2/thealliance.ai/contact.html | 25 ------------------- .../input2/thealliance.ai/contribute.html | 25 ------------------- .../core-projects/ntia_request.html | 25 ------------------- .../thealliance.ai/core-projects/sb1047.html | 25 ------------------- .../core-projects/trusted-evals.html | 25 ------------------- .../thealliance.ai/focus-areas/advocacy.html | 25 ------------------- .../focus-areas/applications-and-tools.html | 25 ------------------- .../foundation-models-datasets.html | 25 ------------------- .../focus-areas/hardware-enablement.html | 25 ------------------- .../focus-areas/skills-education.html | 25 ------------------- .../focus-areas/trust-and-safety.html | 25 ------------------- .../input2/thealliance.ai/index.html | 25 ------------------- .../input2/thealliance.ai/our-work.html | 25 ------------------- .../input2/thealliance.ai/robots.txt | 3 --- 20 files changed, 2 insertions(+), 455 deletions(-) delete mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/about-aia.html delete mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/aia-members.html delete mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/become-a-collaborator.html delete mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/blog.html delete mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/community.html delete mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/contact.html delete mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/contribute.html delete mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/core-projects/ntia_request.html delete mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/core-projects/sb1047.html delete mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/core-projects/trusted-evals.html delete mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/advocacy.html delete mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/applications-and-tools.html delete mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/foundation-models-datasets.html delete mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/hardware-enablement.html delete mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/skills-education.html delete mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/trust-and-safety.html delete mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/index.html delete mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/our-work.html delete mode 100644 examples/notebooks/html-processing/input2/thealliance.ai/robots.txt diff --git a/examples/notebooks/html-processing/.gitignore b/examples/notebooks/html-processing/.gitignore index ebbfa428f..947edf418 100644 --- a/examples/notebooks/html-processing/.gitignore +++ b/examples/notebooks/html-processing/.gitignore @@ -1,2 +1,2 @@ -input/* -output*/ \ No newline at end of file +input*/ +output*/ diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/about-aia.html b/examples/notebooks/html-processing/input2/thealliance.ai/about-aia.html deleted file mode 100644 index 4249f5693..000000000 --- a/examples/notebooks/html-processing/input2/thealliance.ai/about-aia.html +++ /dev/null @@ -1,25 +0,0 @@ -AI Alliance

About AI Alliance

Developing AI collaboratively, transparently, and with a focus on safety, ethics, and the greater good

The AI Alliance is focused on fostering an open community and enabling developers and researchers to accelerate responsible innovation in AI while ensuring scientific rigor, trust, safety, security, diversity and economic competitiveness. 

We bring together a critical mass of compute, data, tools, and talent to accelerate and advocate for open innovation in AI.

Meet our members

Frequently asked questions

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/aia-members.html b/examples/notebooks/html-processing/input2/thealliance.ai/aia-members.html deleted file mode 100644 index c61ef9b27..000000000 --- a/examples/notebooks/html-processing/input2/thealliance.ai/aia-members.html +++ /dev/null @@ -1,25 +0,0 @@ -About | AI Alliance

Meet the AI Alliance

The AI Alliance is a collaborative network of companies, startups, universities, research institutions, government organizations, and non-profit foundations that are working at the forefront of AI technology, applications, and governance.

We unite top developers, scientists, academic institutions, companies, and policymakers to create the future of open, safe, and responsible AI.

Become a contributor
IBM logo
Meta logo
AMD logo
anyoneai
Core 42 logo
Databricks logo
Esade logo
FPT Software
Intel logo
KissanAI logo
MediaTek
University of Notre Dame logo
People+ai logo
Red Hat logo
Sony logo
St. Johns University
Tokyo Electron Ltd. logo
Uber logo
Wadhwani AI
Zilliz logo
\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/become-a-collaborator.html b/examples/notebooks/html-processing/input2/thealliance.ai/become-a-collaborator.html deleted file mode 100644 index 7d0e60bcd..000000000 --- a/examples/notebooks/html-processing/input2/thealliance.ai/become-a-collaborator.html +++ /dev/null @@ -1,25 +0,0 @@ -AI Alliance

Become a collaborator

Want to get involved? Apply to join one of our working groups or major initiatives.

Our team will review your request and get back to you as soon as possible (within 30 days or fewer).

Working group application

Select which working groups or major initiatives you are interested in (select one ore more)*
Interests and contributions*

The AI Alliance is committed to protecting and respecting your privacy, and we’ll only use your personal information to contact you upon submission of this form.

By submitting the preceding form, you consent to allow IBM to store and process the personal information submitted to keep you informed about AI Alliance.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/blog.html b/examples/notebooks/html-processing/input2/thealliance.ai/blog.html deleted file mode 100644 index 5e883cd71..000000000 --- a/examples/notebooks/html-processing/input2/thealliance.ai/blog.html +++ /dev/null @@ -1,25 +0,0 @@ -AI Alliance

Blog & Articles

Perspectives, news, and technical reports from our community.

Articles

Spotlight on Raphaël Vienne of datacraft

In this AI Alliance member spotlight we meet Raphaël Vienne, Head of AI at datacraft.

Spotlight on Roberto Trotta of SISSA

In this AI Alliance member spotlight we meet Roberto Trotta, Professor of Theoretical Physics and Head of Data Science at SISSA, Trieste

UNESCO Language Translator, powered by Meta and Hugging Face

News

UNESCO, Meta, and Hugging Face launch an online Language Translator as part of United Nations General Assembly week, when leaders from around the world meet to discuss the world’s most pressing challenges.

Screenshot BU website reg page

AI for Drug Discovery Open Innovation Forum

News

Please join us on Wednesday, October 30, 2024 at Boston University’s Center for Computing & Data Sciences for the AI for Drug Discovery Open Innovation Forum.

Public Statement from the AI Alliance on the Possible Veto of SB 1047

Public Statement from the AI Alliance on the Possible Veto of SB 1047

News

While we support thoughtful AI regulation, SB 1047 does not strike the right balance. A
veto would allow for a more balanced approach that safeguards innovation while
addressing safety.

abstract gradient

AI Alliance Expands with Seven New Members from India

News

The AI Alliance, a global community dedicated to advancing safe and responsible AI through open innovation, has expanded its membership to include seven leading Indian organizations: AI4Bharat (IIT Madras), IIT Jodhpur, Infosys, KissanAI, People+AI, Sarvam AI, and, Wadhwani AI. This move reinforces India's growing contribution to the global AI landscape and marks a significant expansion of the Alliance. These companies are joined by IIT Bombay as one of the founding members of the Alliance.

Open Source AI Demo Night

News

The AI Alliance, in collaboration with Cerebral Valley and Ollama, hosted Open Source AI Demo Night in San Francisco, bringing together more than 200+ developers and innovators to showcase and celebrate the latest advances in open-source AI.

Spotlight on Sean Hughes of ServiceNow

In this AI Alliance member spotlight we meet Sean Hughes, AI Ecosystem Director at ServiceNow.

Spotlight on Zhuo Li of HydroX AI

Member spotlight

In this AI Alliance member spotlight we meet Zhuo Li, Founder & CEO of Hydrox AI.

Statement from AI Alliance Regarding Commerce's Final Report on Open-Weights Models

News

Commerce Department has taken a crucial step towards ensuring that the U.S. remains at the forefront of AI development by concluding that "the government should not restrict the wide availability of model weights for dual-use foundation models."

abstract office

What we're doing and why: an introduction to The AI Alliance skills and education working group

Education and skill-building is a key priority of the AI Alliance. Already, leaders are sounding alarm about an AI skills crisis, with too few people having the skills needed to work in the field, and too few opportunities for diverse populations to gain the training and experiences they need at scale, and economies standing to lose trillions.

Screen shot of the User Guide website.

Getting started with AI trust and safety

Technical Report

The AI trust and safety guide is a “living” document that provides an introduction to current trends in research and development for ensuring AI models and applications meet requirements for trustworthy results, and in particular, results that satisfy various safety criteria. Aimed at developers and leaders who are relatively new to this topic, the guide defines some common terms, provides an overview of several leading trust and safety education and technology projects, and offers recommendations for how to build-in trust and safety into your AI-based applications.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/community.html b/examples/notebooks/html-processing/input2/thealliance.ai/community.html deleted file mode 100644 index 347e728bf..000000000 --- a/examples/notebooks/html-processing/input2/thealliance.ai/community.html +++ /dev/null @@ -1,25 +0,0 @@ -Community | AI Alliance

Join the Community

Join leading AI innovators, researchers, developers, engineers, and early enterprise adopters in our goal to advance open, safe, and responsible AI. Follow the links on this page to subscribe to our community newsletter, contribute to a working group, join The Alliance, follow us on social media, or attend one of our upcoming events.

Subscribe to our community newsletter

Get updates about everything that’s happening in the community and be the first to know about new projects, new resources, upcoming events, opportunities to collaborate and contribute, etc.

To subscribe, please fill out the newsletter sign-up form in the footer below.

Contribute code

We strive to provide a clear and simple path for anyone to join our community and contribute to AI Alliance projects, whether they are part of an existing member organization, or not. Learn more...

Become a collaborator in one of our working groups

The heartbeat of the AI Alliance is our working groups and major initiatives! Since launch, they have rolled out core projects and affiliated projects – with more to come:

  • Skills, Education, and Exploratory Research WG
  • AI Trust and Safety WG
  • Application and Tools WG
  • Diverse Hardware Enablement WG
  • Foundation Models WG
  • Policy WG

Our major initiatives currently include:

  • Open Trusted Data Initiative
  • Industry Open FMs Initiative
  • Time Series Initiative (data, benchmarks, benchmarking methods)
  • Domain Specific Trust and Safety Initiative

To join one or more, please request an invite. Thank you!

Join The Alliance

If your organization would like to join the AI Alliance as an official member, please use the contact form, and someone on our membership team will get back to you. Thanks!

Attend an AI Alliance event

Follow us on social media

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/contact.html b/examples/notebooks/html-processing/input2/thealliance.ai/contact.html deleted file mode 100644 index 4909ec128..000000000 --- a/examples/notebooks/html-processing/input2/thealliance.ai/contact.html +++ /dev/null @@ -1,25 +0,0 @@ -AI Alliance

Contact us

We’re here to help and answer any questions you may have regarding the AI Alliance. We look forward to hearing from you!

Please ensure to include enough detail so we can route your inquiry to the right team member. Thanks!

Keep in touch

The AI Alliance is committed to protecting and respecting your privacy, and we’ll only use your personal information to contact you upon submission of this form.

By submitting the preceding form, you consent to allow IBM to store and process the personal information submitted to keep you informed about AI Alliance.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/contribute.html b/examples/notebooks/html-processing/input2/thealliance.ai/contribute.html deleted file mode 100644 index 4208d2d2c..000000000 --- a/examples/notebooks/html-processing/input2/thealliance.ai/contribute.html +++ /dev/null @@ -1,25 +0,0 @@ -AI Alliance

Join us in building the open future of AI

We strive to provide a clear and simple path for anyone to join our community and contribute to AI Alliance projects, whether they are part of an existing member organization, or not.

We are excited to welcome contributions to AI Alliance Core and Affiliated projects.

Core Projects
Core projects address substantial cross-community challenges and are an opportunity for individual contributors and members to collaborate, build, and make an impact on the future of AI. Core Projects are managed directly by the AI Alliance and governed as described below. You can find a list of AI Alliance Core Projects here on the AI Alliance website. 

Learn about Core Projects

Affiliated Projects
We encourage individuals to contribute to Affiliated projects run by an AI Alliance member.  Although Affiliated Projects are not managed by the AI Alliance, we highlight them as contributing to open, safe, and responsible AI. You can find a list of AI Alliance Affiliated Projects here on the AI Alliance website. Contribution to Affiliated Projects is governed as set out by the Affiliate Project, not the AI Alliance contribution process set out below.

Learn about Affiliated Projects

Making a contribution

The AI Alliance utilizes the Linux Foundation’s Developer Certificate of Origin (DCO) on a per-commit basis to enable contributions to Core Projects. The DCO does not require the committer to give away ownership of the contribution - your IP remains yours and others' IP remains theirs.

Core Project Licenses

AI Alliance Core Projects’ utilize the following licenses unless otherwise noted in the project repository:

The AI Alliance leaves open the possibility of additional terms concerning safe and responsible use for certain elements in special Core Projects, for example weights associated with certain AI models. Any decision to use any such additional terms for a Core Project must be made by the AI Alliance Steering Committee and will be clearly identified in the Core Project repository.

Anyone participating in and/or contributing to the AI Alliance community must agree to the following agreements and conditions: 

  1. AI Alliance Code of Conduct - We strive for a vibrant, diverse, and collegial community.
  2. This Developer Certificate of Origin applies to all contributions to AI Alliance Core Projects. 
  3. AI Alliance Competition Law Guidelines - The AI Alliance and its members are committed to compliance with applicable antitrust and competition laws.
Check out the AI Alliance Github for more information and ways to contribute
\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/core-projects/ntia_request.html b/examples/notebooks/html-processing/input2/thealliance.ai/core-projects/ntia_request.html deleted file mode 100644 index 77a5985dd..000000000 --- a/examples/notebooks/html-processing/input2/thealliance.ai/core-projects/ntia_request.html +++ /dev/null @@ -1,25 +0,0 @@ -Responding to the U.S. NTIA request for comment on Dual Use Foundation Artificial Intelligence Models with Widely Available Model Weights | AI Alliance

Responding to the U.S. NTIA request for comment on Dual Use Foundation Artificial Intelligence Models with Widely Available Model Weights

Project
abstract gradient

The AI Alliance recently drafted a response to the NTIA’s request for comment on “Dual Use Foundation Artificial Intelligence Models with Widely Available Model Weights.” The request seeks public input on the potential risks, benefits, and policy approaches for AI foundation models whose weights are broadly accessible.

This initiative follows an Executive Order from President Biden on AI development and use. The public comment period is significant as it allows stakeholders to influence the policy framework, ensuring that these models are developed and used safely and ethically.  

Download our response
\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/core-projects/sb1047.html b/examples/notebooks/html-processing/input2/thealliance.ai/core-projects/sb1047.html deleted file mode 100644 index 89c80b88b..000000000 --- a/examples/notebooks/html-processing/input2/thealliance.ai/core-projects/sb1047.html +++ /dev/null @@ -1,25 +0,0 @@ -AI Alliance Statement in Opposition to California SB 1047 | AI Alliance

A statement in opposition to California SB 1047

Project
abstract gradient

The AI Alliance, a community of technology creators, developers, and adopters collaborating to advance safe, responsible AI rooted in open innovation, respectfully opposes SB 1047, the proposed Safe and Secure Innovation for Frontier Artificial Intelligence Models Act. It would slow innovation, thwart advancements in safety and security, and undermine California’s economic growth. The bill’s technically infeasible requirements will chill innovation in the field of AI and lower access to the field’s cutting edge, thereby directly contradicting the bill’s stated purpose “...to ensure that artificial intelligence innovation . . . is accessible to academic researchers and startups, in addition to large companies.” Sec. 2(d).


The AI Alliance is a community of researchers, developers, and organizational leaders committed to supporting and enhancing open innovation across the AI technology landscape. Many members of the AI Alliance are based in California or have a nexus to the state or its residents⏤and we are deeply concerned about the potential establishment of an anti-open-source precedent. The Alliance enables developers and researchers to accelerate responsible open innovation in AI while ensuring scientific rigor, trust, safety, security, diversity, and economic competitiveness. By bringing together this community, the AI Alliance pools resources, knowledge, and best practices to address safety concerns and provides a platform for sharing and developing solutions that benefit the whole AI ecosystem.

Examples of AI Alliance contributions. AI Alliance members contribute to and rely on open AI models, with uses demonstrating the ways open models enhance innovation, economic productivity, safety, and security, while broadening access to AI. IBM, in partnership with NASA, has democratized the largest geospatial AI foundation model on Hugging Face. Stability AI is developing innovative, resource-efficient, open-access AI models. Snowflake is empowering businesses to build potent AI and machine learning applications from its enterprise data. SeedAI is crafting frameworks for co-creation, evaluation, and decision-making regarding AI technology and policies. Meta launched Llama 3, which has already been downloaded over 1 million times; over 600 derivative models have been uploaded to Hugging Face. Databricks is democratizing AI within the enterprise space, by helping thousands of organizations customize and deploy DBRX, Llama 3, and other opensource AI models for enterprise applications. ServiceNow, in collaboration with Hugging Face and the BigCode community of more than 1,200 members from 60+ countries, released its StarCoder family of models with the aim of introducing more responsibility and transparency in the development of large language models for code. AI Alliance member universities globally are leveraging open-source AI for scientific discovery, research, and innovation.

Open-source approach is safe and secure. If the bill does aim to promote “Safe and Secure Innovation for Frontier Artificial Intelligence Models,” it should facilitate an AI ecosystem that provides for the full range of options regarding form of release. Historically, open source has been a more secure approach to technology development. This remains true. The availability of model weights, for example, enables the identification and examination of model vulnerabilities, driving security enhancements for both closed and open models. Open models facilitate cross-examination and the detection of potential improvements or bugs, fostering a collaborative development environment. This perspective is shared by twenty-five leading AI experts from industry, academia, and civil society. And this sentiment is shared by over 1,200 individuals across the AI ecosystem, who, in a Joint Statement on AI Safety and Openness, declared that openness and transparency are fundamental to achieving safety, security, and accountability in AI.

The bill severely penalizes responsible open-source development. While SB 1047 is not targeting open-source development specifically, it will affect the open-source community dramatically. The bill requires developers of AI models of 10^26 FLOPS or similar performance (as determined by undefined benchmarks) to implement a full shutdown control that would halt operation of the model and all derivative models. Once a model is open sourced and subsequently downloaded by a third party, by design developers no longer have control over a model. Before such a “shutdown switch” provision is passed, we need to understand how it can be done in the context of open source; the bill does not answer that question. No models at 10^26 FLOPS are openly available today but technology is rapidly advancing and the open ecosystem could evolve alongside it. However, this legislation seems intended to freeze open-source AI development at the 2024 level.

Current exemptions are not practicable. The bill does attempt to provide some reasonable requirements for models subject to a “limited duty exemption,” including an exception to the shutdown requirement. However, few⏤if any⏤models would ever qualify for this exemption because it includes technically impracticable obligations, such as requiring developers to determine whether the exemption applies prior to training. Further, the exception is only available if the model does not enable a “hazardous capability,” but a model is considered to have a hazardous capability “even if the hazardous capability [at issue] would not manifest but for fine tuning and post training modifications performed by third-party experts intending to demonstrate those abilities.”4 As fine tuning and post training modifications could introduce almost anything to the model, this requirement would be nearly impossible to meet.

The bill needs a number of changes to make it effective and practicable. This bill will also affect smaller companies, which often rely on open-source models created by larger companies. Compliance costs may discourage open sourcing, depriving smaller companies of the models they need to build their businesses. Therefore, we recommend refocusing regulatory attention from model development to model applications (i.e., how the model is used), particularly those applications that pose high risks to public safety and security.

However, assuming the bill retains the current structure, we recommend the following changes:

  1. Amending the requirements for the limited duty exemption to make it feasible for responsible open-source developers;
  2. When a limited duty exemption does not apply, amending the obligations on developers to limit scope to instances where developers have control. Without this limitation, developers will be liable for all kinds of downstream uses that will make open sourcing models unviable;
  3. Incorporating discrete open-source exceptions throughout the bill to ensure that responsible developers can open-source models with adequate mitigations.

Download the statement
\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/core-projects/trusted-evals.html b/examples/notebooks/html-processing/input2/thealliance.ai/core-projects/trusted-evals.html deleted file mode 100644 index 5c0d9b8a0..000000000 --- a/examples/notebooks/html-processing/input2/thealliance.ai/core-projects/trusted-evals.html +++ /dev/null @@ -1,25 +0,0 @@ -Trusted evals request for proposals | AI Alliance

Trusted evals request for proposals

Project
abstract gradient

Context

Model evaluation, or ‘evals’, in the generative AI era is simultaneously one of the most important areas of investment but also one of the highest areas of entropy.

Our goals for the Trusted Evals working group are to:

  1. Raise awareness for community efforts around trust and safety – including, but not limited to, work happening globally in various languages and in various domains (e.g. cyber security, CBRNE, etc.). Foster and grow the academic and technical communities in Trustworthy AI. And by extension create a center of mass for domain experts that can help us push beyond where evals stand today and into areas where we don’t have good visibility; and
  2. Drive the development of comprehensive, reliable, and stable tools for model evaluation, where these tools provide repeatable, reproducible, and diverse results, and where these tools are regularly refreshed and ever evolving as we learn about new risks for generative AI and other evaluation concerns. This means, as a community, creating new benchmarks and metrics to address quality, safety, robustness and performance aspects of generative AI models. The approach is to be as broad and diverse as possible with the goal of uncovering new evals in as many domains as possible so we can learn as a community. The goal is NOT to create a standard for model evaluations, but to work closely with MLCommons to help shepherd a subset of these evals into their standardization effort.

About

Generally speaking, evals shine a light on model capabilities (we love to evaluate how well our models can reason) but they also can expose where models could present certain risks of harm. One of the major challenges as evals become more esoteric is that access is limited for experts in various fields that also intersect in GenAI. For example, at Meta there is a team of cyber security experts who also build models for coding and productivity and, in parallel, are building safeguards for things like malicious code generation. For many of CBRNE risks, these experts don’t have a central place to aggregate nor are they working with generative AI experts. Further to this, there isn’t a de facto place today for the open community to evaluate your models across a growing number of potential harms.

Request for proposal

We are seeking new perspectives in the AI evaluation domain. To foster further innovation in this area, we are pleased to invite the community to participate in the AI Alliance’s Trusted Evals working group by submitting a response to our Request for Proposal to be included in the working group’s efforts to:

  1. Raise awareness - selected proposals will be showcased through AI Alliance communications, including our newsletter, blog, whitepapers, and website; and
  2. Drive the development of comprehensive, reliable, and stable tools - The AI Alliance intends to support select project proposals with resources to help teams accelerate progress in building the foundations of safe, trusted AI.

For this RFP, we are excited to work with those in academia, big industry, startups and anyone excited to collaborate in the open and build an ecosystem around their work.

Areas of interest

  1. Cybersecurity threats
  2. Sensitive data detection including areas such as toxic content (e.g. hate speech), personally identifiable information (PII), bias, etc.
  3. Model performance including helpfulness, quality, alignment, robustness, etc., (as opposed to operational concerns like throughput, latency, scalability, etc.)
  4. Knowledge and factuality
  5. Multilingual evaluation
  6. Mediation
  7. Balancing harms and helpfulness
  8. Personal data memorization / data governance
  9. Vertical domains such as legal, financial, medical
  10. Areas related to CBRNE (chemical, biological, radiological, nuclear, and high yield explosives)
  11. Weapons acquisition specifically
  12. Measuring data set integrity when data is created by AI: label fairness, prompt generation for RLHF
  13. Effectiveness of tool use that exacerbates malicious intent
  14. Demographic representation across different countries
  15. Distributionbias
  16. Politicalbias
  17. Capability fairness
  18. Undesirable use cases
  19. Regional bias, discrimination
  20. Violence and hate
  21. Terrorism and sanctioned individuals
  22. Defamation
  23. Misinformation
  24. Guns,illegal weapons, controlled substances

Requirements

Proposals should include a link to a summary of your project, in English, explaining the area of focus, dataset description and any relevant prior work. This includes:

  • Name of project and abstract
  • Core team member bios & affiliations
  • Link to website (if applicable)
  • Link to GitHub repository
  • License
  • Links to whitepapers or publications
Submit your proposal
\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/advocacy.html b/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/advocacy.html deleted file mode 100644 index 7da87a218..000000000 --- a/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/advocacy.html +++ /dev/null @@ -1,25 +0,0 @@ -Advocacy | AI Alliance

Advocacy

Supporting regulatory policies that create healthy, sustainable, and open ecosystems for AI.

A thriving and competitive open innovation ecosystem for AI must be a priority for industry, civil society, and academia — policymakers should take note. Though these ecosystems are largely decentralized and self-directed, there's room for policy to aid their growth. 
    

Our work will help policymakers and governments recognize and support open innovation ecosystems for AI. We’re also bridging gaps between policy and industry, fostering responsible and ethical AI practices to benefit societies around the world.   

Working Groups

Advocacy

Open Source AI Demo Night

The Advocacy Working Group supports regulatory policies that create healthy, sustainable, and open ecosystems for AI. It works on projects like Open Source AI Demo Night, which highlighted entrepreneurs using open source AI to innovate.

Our projects

Statement from AI Alliance Regarding Commerce's Final Report on Open-Weights Models

News

Commerce Department has taken a crucial step towards ensuring that the U.S. remains at the forefront of AI development by concluding that "the government should not restrict the wide availability of model weights for dual-use foundation models."

abstract gradient

A statement in opposition to California SB 1047

Advocacy

Our perspectives and recommendations in opposition to California SB 1047, the proposed Safe and Secure Innovation for Frontier Artificial Intelligence Models Act.

abstract gradient

Responding to the U.S. NTIA request for comment on Dual Use Foundation Artificial Intelligence Models with Widely Available Model Weights

Advocacy

The request seeks public input on the potential risks, benefits, and policy approaches for AI foundation models whose weights are broadly accessible.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/applications-and-tools.html b/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/applications-and-tools.html deleted file mode 100644 index eadb27fed..000000000 --- a/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/applications-and-tools.html +++ /dev/null @@ -1,25 +0,0 @@ -Tools | AI Alliance

Applications and Tools

Building the most capable tools for AI model builders and GenAI application developers.

We build and promote open-source tools for model training, tuning, and inference. We are also collaborating to simplify, automate, and optimize the deployment and execution of large-scale AI workloads on Kubernetes. 

Our work

abstract gradient

Advancing Domain-Specific Q&A: The AI Alliance's Guide to Best Practices

Technical Report

The AI Alliance application and tools working group has conducted a comprehensive study on best practices for advancing domain-specific Q&A using retrieval-augmented generation (RAG) techniques. The findings of this research, provide insights and recommendations for maximizing the capabilities of Q&A AI in specialized domains.

The Living Guide to Applying AI

Applications and Tools

A "living guide" for building AI-enabled applications, this guide provides an introduction to several established design patterns for building AI systems and products, with contributions from different experts in the AI Alliance. Several common patterns, like RAG (retrieval-augmented generation), are explored from different angles, and emerging patterns, like GraphRAG (using a graph network as a source of RAG content) and agents.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/foundation-models-datasets.html b/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/foundation-models-datasets.html deleted file mode 100644 index 8f107d89b..000000000 --- a/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/foundation-models-datasets.html +++ /dev/null @@ -1,25 +0,0 @@ -Foundation Models | AI Alliance

Open Foundation Models and Datasets

Enabling an ecosystem of open foundation models, including those with multilingual and multi-modal capabilities, and open datasets.

We are responsibly enhancing the ecosystem of open foundation models and datasets. We are embracing multilingual and multimodal models, as well as science models tackling broad societal issues like climate change and education.

To aid AI model builders and application developers, we’re collaborating to develop and promote open-source tools for model training, tuning, and inference. We are also launching programs to foster the open development of AI in safe and beneficial ways, and hosting events to explore AI use cases.

Without good datasets, model training and tuning would be impossible. We are promoting the development of open datasets with clear governance and provenance controls so they can be used without concerns for legal and other risks.

Working groups

AI for Drug Discovery

This working group aims to create a world-class research community that harnesses the potential of AI foundation models, transforms the field of drug discovery, and accelerates scientific progress by driving interdisciplinary collaboration on AI-powered drug discovery projects in the open.

Foundation Models

abstract gradient

Materials and Chemistry

We aim to curate datasets, tasks and benchmarks for materials science, build out foundation models in chemistry for prediction of properties, experimental outcomes or generation of new candidates and create a framework to foster collaboration between human experts and AI agents that will ultimately help solve global urgent challenges in sustainability and safety of materials.

Our work

These are the projects we are currently working on.

Home page for the Open Trusted Data Initiative

Open Trusted Data Initiative

Foundation Models and Datasets

Cataloging and managing trustworthy datasets.

abstract gradient

Time Series Data and Model Initiative

Foundation Models and Datasets

Time-series applications are an important target for AI. In addition to gathering high-quality and fully-governed time series datasets as part of the Open Trusted Data Initiative, Alliance members are collaborating on new and improved time series models (as part of the Industry Open FMs Initiative and benchmarks, both general-purpose and application-specific.

Please join us. We need time series and domain experts, including especially subject matter experts and use case and product owners who would like to apply emerging time series foundation models to new applications. There is an acute shortage of good, open datasets for time series and data specially benchmarks and evaluation methods for various use cases. Contributions are especially welcome here.

abstract gradient

Industry Open FMs Initiative

Foundation Models and Datasets

We have seen rapid progress in building and releasing highly-capable and open foundation models for general language, coding, scientific discovery, and multi-modal scenarios.

A key development in model strategies is a focus on building smaller, more specialized models.

More details are coming soon, but we would love for you to join us. We need both model-building and domain experts, including those outside the target domains listed above.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/hardware-enablement.html b/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/hardware-enablement.html deleted file mode 100644 index 93d4e8801..000000000 --- a/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/hardware-enablement.html +++ /dev/null @@ -1,25 +0,0 @@ -Hardware | AI Alliance

Hardware Enablement

Fostering a vibrant AI hardware accelerator ecosystem through enabling software technology.

We collaborate on the benchmarking, optimization, and adaptation of AI workloads to accelerate innovation in a diverse set of hardware. Our work focuses on scalability, platform adaptation, and advanced power, energy, and carbon modeling. Benchmarks and metrics developed for model inference, fine-tuning, and energy consumption of large-scale AI workloads will be contributed to the open-source community. 

Our Work

These are the projects we are currently working on.

Screen shot of the AI Accelerator Software Ecosystem Guide

AI Accelerator Software Ecosystem Guide

Hardware Enablement

The AI Accelerator Software Ecosystem Guide provides an introduction to the broad topic of software tools that support seamless execution of AI workloads on different hardware accelerators. Industry experts provide guidance on the state of the art and how to ensure success.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/skills-education.html b/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/skills-education.html deleted file mode 100644 index 8fdb0109e..000000000 --- a/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/skills-education.html +++ /dev/null @@ -1,25 +0,0 @@ -Skills & Education | AI Alliance

Skills & Education

Supporting global AI skill-building, education, and exploratory research.

Our members partner with academics, industry leaders, researchers, and students, empowering them to develop and leverage resources to build an AI-ready workforce and engage with vital research in AI models, algorithms, and platforms.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/trust-and-safety.html b/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/trust-and-safety.html deleted file mode 100644 index 849aeec58..000000000 --- a/examples/notebooks/html-processing/input2/thealliance.ai/focus-areas/trust-and-safety.html +++ /dev/null @@ -1,25 +0,0 @@ -Trust | AI Alliance

Trust and Safety

Ensuring safe and trusted generative AI with benchmarks, tools, and methodologies

Our members work together to understand the landscape of AI trust and safety risks, as well as other uses of AI system evaluation. We identify use cases specific to various important domains (finance, healthcare, education, and others). We work together and with external collaborators to build tools, refine methods, and create benchmarks for detecting and mitigating those risks and performing other kinds of evaluation. We also help educate the public about responsible AI and the developer community about responsible model and application development. 

Our work

These are the projects we are currently working on.

Understanding AI Trust and Safety: A Living Guide

A major challenge for the successful use of AI is the importance of understanding potential trust and safety issues, along with their mitigation strategies. Failure to consider these issues could impact an organization's operations and the experience of its users. Concerns about safety are also a driver for current regulatory initiatives. Hence, applications built with AI must be designed and implemented with AI trust and safety in mind. This guide provides an introduction to trust and safety concerns, and offers guidance for AI projects.

Ranking AI Safety Priorities by Domain

Trust & Safety

A challenge for software development teams adopting generative AI is making sense of the safety issues that their applications must address. The AI safety ecosystem is broad and growing quickly, making it difficult for these development teams to know where they should focus their efforts. What safety concerns are most important for them to work on first?

Trust and Safety Evaluations

Trust & Safety

Much like other software, generative AI (“GenAI”) models and the AI systems that use them need to be trusted and useful to their users. The Trust and Safety Evaluations project fills gaps in the current landscape of the taxonomy of different kinds of evaluation, the tools for creating and running evaluations, and leaderboards to address particular categories of user needs.

abstract gradient

Trusted evals request for proposals

Trust & Safety

The AI Alliance Trusted Evals request for proposals is aimed at seeking new perspectives in the AI evaluation domain. We are excited to work with those in academia, industry, startups and anyone excited to collaborate in the open and build an ecosystem around their work.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/index.html b/examples/notebooks/html-processing/input2/thealliance.ai/index.html deleted file mode 100644 index fd6a8217d..000000000 --- a/examples/notebooks/html-processing/input2/thealliance.ai/index.html +++ /dev/null @@ -1,25 +0,0 @@ -Home | AI Alliance

Building the open future of AI

We are technology developers, researchers, industry leaders and advocates who collaborate to advance safe, responsible AI rooted in open innovation.

Advocacy

abstract gradient

Supporting regulatory policies that create healthy, sustainable, and open ecosystems for AI.

Trust & Safety

abstract image

Creating benchmarks, tools, and methodologies to evaluate and ensure safe, trusted generative AI.

Applications and Tools

abstract gradient

Building the most capable tools for AI model builders and GenAI application developers.

Foundation Models and Datasets

Foundation Models and Datasets

Enabling an ecosystem of open foundation models, including those with multilingual, multi-modal, science, and time series capabilities. Similarly, promote the development of datasets in these areas.

Hardware Enablement

abstract gradient

Fostering a vibrant AI hardware accelerator ecosystem through enabling software technology.

Skills & Education

abstract gradient

Supporting global AI skill-building, education, and exploratory research.

News & Projects

Open Source AI Demo Night

News

The AI Alliance, in collaboration with Cerebral Valley and Ollama, hosted Open Source AI Demo Night in San Francisco, bringing together more than 200+ developers and innovators to showcase and celebrate the latest advances in open-source AI.

abstract gradient

A statement in opposition to California SB 1047

Advocacy

Our perspectives and recommendations in opposition to California SB 1047, the proposed Safe and Secure Innovation for Frontier Artificial Intelligence Models Act.

abstract gradient

Trusted evals request for proposals

Trust & Safety

The AI Alliance Trusted Evals request for proposals is aimed at seeking new perspectives in the AI evaluation domain. We are excited to work with those in academia, industry, startups and anyone excited to collaborate in the open and build an ecosystem around their work.

abstract gradient

Responding to the U.S. NTIA request for comment on Dual Use Foundation Artificial Intelligence Models with Widely Available Model Weights

Advocacy

The request seeks public input on the potential risks, benefits, and policy approaches for AI foundation models whose weights are broadly accessible.

  • IBM logo
  • Meta logo
  • AMD logo
  • Core 42 logo
  • Databricks logo
  • Esade logo
  • FPT Software
  • Intel logo
  • Red Hat logo
  • Sony logo
  • Uber logo
  • University of Notre Dame logo
  • Zilliz logo

We build, enable, and advocate for open innovation across the AI technology landscape, including software, data and models, safety, security and trust, tooling, evaluation, hardware, education, open science, and advocacy.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/our-work.html b/examples/notebooks/html-processing/input2/thealliance.ai/our-work.html deleted file mode 100644 index fd06a0f8a..000000000 --- a/examples/notebooks/html-processing/input2/thealliance.ai/our-work.html +++ /dev/null @@ -1,25 +0,0 @@ -Our Work | AI Alliance

Accelerating responsible innovation in AI

Open and transparent innovation is essential for equipping AI researchers, developers, and users with the knowledge and tools to leverage AI advancements safely and inclusively, prioritizing diversity and economic opportunity for all. 

Through member-driven working groups, we bring together builders and experts from various fields to collaboratively and transparently address the challenges of generative AI and democratize its benefits.

Contribute to an AI Alliance project

Core projects

View all

Core projects address substantial cross-community challenges and are an opportunity for individual contributors and members to collaborate, build, and make an impact on the future of AI. Core Projects are managed directly by the AI Alliance and governed as described below.

abstract gradient

A statement in opposition to California SB 1047

Advocacy

Our perspectives and recommendations in opposition to California SB 1047, the proposed Safe and Secure Innovation for Frontier Artificial Intelligence Models Act.

abstract gradient

Trusted evals request for proposals

Trust & Safety

The AI Alliance Trusted Evals request for proposals is aimed at seeking new perspectives in the AI evaluation domain. We are excited to work with those in academia, industry, startups and anyone excited to collaborate in the open and build an ecosystem around their work.

abstract gradient

Responding to the U.S. NTIA request for comment on Dual Use Foundation Artificial Intelligence Models with Widely Available Model Weights

Advocacy

The request seeks public input on the potential risks, benefits, and policy approaches for AI foundation models whose weights are broadly accessible.

Ranking AI Safety Priorities by Domain

Trust & Safety

A challenge for software development teams adopting generative AI is making sense of the safety issues that their applications must address. The AI safety ecosystem is broad and growing quickly, making it difficult for these development teams to know where they should focus their efforts. What safety concerns are most important for them to work on first?

Home page for the Open Trusted Data Initiative

Open Trusted Data Initiative

Foundation Models and Datasets

Cataloging and managing trustworthy datasets.

abstract gradient

Time Series Data and Model Initiative

Foundation Models and Datasets

Time-series applications are an important target for AI. In addition to gathering high-quality and fully-governed time series datasets as part of the Open Trusted Data Initiative, Alliance members are collaborating on new and improved time series models (as part of the Industry Open FMs Initiative and benchmarks, both general-purpose and application-specific.

Please join us. We need time series and domain experts, including especially subject matter experts and use case and product owners who would like to apply emerging time series foundation models to new applications. There is an acute shortage of good, open datasets for time series and data specially benchmarks and evaluation methods for various use cases. Contributions are especially welcome here.

Screen shot of the AI Accelerator Software Ecosystem Guide

AI Accelerator Software Ecosystem Guide

Hardware Enablement

The AI Accelerator Software Ecosystem Guide provides an introduction to the broad topic of software tools that support seamless execution of AI workloads on different hardware accelerators. Industry experts provide guidance on the state of the art and how to ensure success.

abstract gradient

Industry Open FMs Initiative

Foundation Models and Datasets

We have seen rapid progress in building and releasing highly-capable and open foundation models for general language, coding, scientific discovery, and multi-modal scenarios.

A key development in model strategies is a focus on building smaller, more specialized models.

More details are coming soon, but we would love for you to join us. We need both model-building and domain experts, including those outside the target domains listed above.

Understanding AI Trust and Safety: A Living Guide

A major challenge for the successful use of AI is the importance of understanding potential trust and safety issues, along with their mitigation strategies. Failure to consider these issues could impact an organization's operations and the experience of its users. Concerns about safety are also a driver for current regulatory initiatives. Hence, applications built with AI must be designed and implemented with AI trust and safety in mind. This guide provides an introduction to trust and safety concerns, and offers guidance for AI projects.

The Living Guide to Applying AI

Applications and Tools

A "living guide" for building AI-enabled applications, this guide provides an introduction to several established design patterns for building AI systems and products, with contributions from different experts in the AI Alliance. Several common patterns, like RAG (retrieval-augmented generation), are explored from different angles, and emerging patterns, like GraphRAG (using a graph network as a source of RAG content) and agents.

Trust and Safety Evaluations

Trust & Safety

Much like other software, generative AI (“GenAI”) models and the AI systems that use them need to be trusted and useful to their users. The Trust and Safety Evaluations project fills gaps in the current landscape of the taxonomy of different kinds of evaluation, the tools for creating and running evaluations, and leaderboards to address particular categories of user needs.

Affiliated projects

View all

Affiliated projects are led and managed by members and identified as being aligned to the AI Alliance mission. Although Affiliated Projects are not managed by the AI Alliance, we highlight them as contributing to open, safe, and responsible AI.

IBM Granite Code Models

IBM logo

A series of code models trained by IBM licensed under Apache 2.0 license. Includes base pretrained and instruct models.

AIConfig & Editor

AIConfig is a config-based framework for building generative AI applications.

GenAI in Education: Usage Guidance

A report evaluating the feasibility, benefits, and limitations of using generative AI technologies in an educational setting and its impact on learning outcomes.

LLM 360: Amber

The Amber project includes a 7B English language model with the LLaMA architecture, an instruction-following model finetuned from LLM360-Amber, and a safety-finetuned instruction model using LLM360-AmberChat as the base.

TrustyAI

Red Hat logo

TrustyAI is an open-source toolkit designed for responsible AI.

Ragna

Ragna is an open-source RAG-based (Retrieval-Augmented Generation) AI orchestration framework designed to bridge the gap between AI research and production deployment.

\ No newline at end of file diff --git a/examples/notebooks/html-processing/input2/thealliance.ai/robots.txt b/examples/notebooks/html-processing/input2/thealliance.ai/robots.txt deleted file mode 100644 index 2333b4e75..000000000 --- a/examples/notebooks/html-processing/input2/thealliance.ai/robots.txt +++ /dev/null @@ -1,3 +0,0 @@ -User-Agent: * -Allow: / -Sitemap: https://thealliance.ai/sitemap.xml \ No newline at end of file From 90844833767b63f8d60be41e43cd4e84c1c9d741 Mon Sep 17 00:00:00 2001 From: SHAHROKH DAIJAVAD Date: Thu, 5 Dec 2024 12:35:40 -0800 Subject: [PATCH 19/31] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b63ac2d5d..1fa739fdd 100644 --- a/README.md +++ b/README.md @@ -79,8 +79,8 @@ conda install gxx_linux-64 Next, install the data prep toolkit library. This library installs both the python and ray versions of the transforms. For better management of dependencies, it is recommended to install the same tagged version of both the library and the transform. ```bash -pip3 install 'data-prep-toolkit[ray]==0.2.2.dev1' -pip3 install 'data-prep-toolkit-transforms[ray,all]==0.2.2.dev1' +pip3 install 'data-prep-toolkit[ray]==0.2.3.dev0' +pip3 install 'data-prep-toolkit-transforms[ray,all]==0.2.3.dev1' pip3 install jupyterlab ipykernel ipywidgets ## install custom kernel From b4390e4ccfce2c68295b16c99213abe7dffa8b99 Mon Sep 17 00:00:00 2001 From: Constantin M Adam Date: Thu, 5 Dec 2024 16:12:26 -0500 Subject: [PATCH 20/31] Documented the limit choice Signed-off-by: Constantin M Adam --- .../ray/src/data_processing_ray/runtime/ray/ray_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/data-processing-lib/ray/src/data_processing_ray/runtime/ray/ray_utils.py b/data-processing-lib/ray/src/data_processing_ray/runtime/ray/ray_utils.py index d5e6754ac..ff6e53892 100644 --- a/data-processing-lib/ray/src/data_processing_ray/runtime/ray/ray_utils.py +++ b/data-processing-lib/ray/src/data_processing_ray/runtime/ray/ray_utils.py @@ -22,6 +22,9 @@ from ray.util.actor_pool import ActorPool +# This value matches the constant `RAY_MAX_LIMIT_FROM_API_SERVER` defined in the ray source code here: +# https://github.com/ray-project/ray/blob/569f7df9067c5654fb57ba7bc4792b3ba5aaa846/python/ray/util/state/common.py#L50-L53 + RAY_MAX_ACTOR_LIMIT = 10000 From 0ad00153af948ce22db483f91f9bff360952c094 Mon Sep 17 00:00:00 2001 From: Constantin M Adam Date: Thu, 5 Dec 2024 22:52:45 -0500 Subject: [PATCH 21/31] Added links to example notebooks - issue #848 fix Signed-off-by: Constantin M Adam --- transforms/universal/fdedup/python/README.md | 2 +- transforms/universal/fdedup/ray/README.md | 4 ++++ transforms/universal/fdedup/spark/README.md | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/transforms/universal/fdedup/python/README.md b/transforms/universal/fdedup/python/README.md index 295862221..4c531476f 100644 --- a/transforms/universal/fdedup/python/README.md +++ b/transforms/universal/fdedup/python/README.md @@ -213,7 +213,7 @@ To see results of the transform. ### Code example -TBD (link to the notebook will be provided) +This is a [sample notebook](../fdedup_python.ipynb) that shows how to invoke the python fuzzy dedup transform. ### Transforming data using the transform image diff --git a/transforms/universal/fdedup/ray/README.md b/transforms/universal/fdedup/ray/README.md index d93be3a4a..298ac39ba 100644 --- a/transforms/universal/fdedup/ray/README.md +++ b/transforms/universal/fdedup/ray/README.md @@ -51,6 +51,10 @@ To use the transform image to transform your data, please refer to the [running images quickstart](../../../../doc/quick-start/run-transform-image.md), substituting the name of this transform image and runtime as appropriate. +## Code Example + +This is a [sample notebook](../fdedup_ray.ipynb) that shows how to invoke the ray fuzzy dedup transform. + ## Testing For testing fuzzy deduplication in a ray runtime, use the following `make` targets. To launch integration tests diff --git a/transforms/universal/fdedup/spark/README.md b/transforms/universal/fdedup/spark/README.md index dd0294aed..1b02ddd00 100644 --- a/transforms/universal/fdedup/spark/README.md +++ b/transforms/universal/fdedup/spark/README.md @@ -51,6 +51,10 @@ To use the transform image to transform your data, please refer to the [running images quickstart](../../../../doc/quick-start/run-transform-image.md), substituting the name of this transform image and runtime as appropriate. +## Code Example + +This is a [sample notebook](../fdedup_spark.ipynb) that shows how to invoke the spark fuzzy dedup transform. + ## Testing For testing fuzzy deduplication in a spark runtime, use the following `make` targets. To launch integration tests From eca9faa9201271607dbee070fdbbd161b9de9e74 Mon Sep 17 00:00:00 2001 From: Aisha Mohammed Farooq Darga Date: Thu, 5 Dec 2024 09:32:21 +0530 Subject: [PATCH 22/31] Cleared output cells Signed-off-by: Aisha Mohammed Farooq Darga --- examples/notebooks/hap/generate_hap_score_csv.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/notebooks/hap/generate_hap_score_csv.ipynb b/examples/notebooks/hap/generate_hap_score_csv.ipynb index a44935a90..f36e538d0 100644 --- a/examples/notebooks/hap/generate_hap_score_csv.ipynb +++ b/examples/notebooks/hap/generate_hap_score_csv.ipynb @@ -53,7 +53,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Click link to open notebook in google colab: [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/IBM/data-prep-kit/blob/ed990c3445d0187062dbc3ca3913a46674c7a7e3/examples/notebooks/hap/generate_hap_score_csv.ipynb)\n" + "Click link to open notebook in google colab: [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/IBM/data-prep-kit/blob/8381d353c3aa90d334e81ac3029ab774c753cc4b/examples/notebooks/hap/generate_hap_score_csv.ipynb)\n" ] }, { From b2193f5f195e5ef372886ecc862809b8ccc6a686 Mon Sep 17 00:00:00 2001 From: Aisha Darga <37473748+AishaDarga@users.noreply.github.com> Date: Thu, 5 Dec 2024 09:49:18 +0530 Subject: [PATCH 23/31] Update pip install Signed-off-by: Aisha Mohammed Farooq Darga --- examples/notebooks/hap/generate_hap_score_csv.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/notebooks/hap/generate_hap_score_csv.ipynb b/examples/notebooks/hap/generate_hap_score_csv.ipynb index f36e538d0..763621f5d 100644 --- a/examples/notebooks/hap/generate_hap_score_csv.ipynb +++ b/examples/notebooks/hap/generate_hap_score_csv.ipynb @@ -72,7 +72,7 @@ "! pip install data-prep-connector\n", "! pip install 'data-prep-toolkit[ray]==0.2.2.dev1'\n", "! pip install 'data-prep-toolkit-transforms[ray,all]==0.2.2.dev1'\n", - "! pip install -r requirements.txt" + "! pip install nltk==3.9.1 transformers==4.38.2 torch>=2.2.2,<=2.4.1 pandas==2.2.2" ] }, { From c66bc1002565019873293d4acf46bc0c74130ed6 Mon Sep 17 00:00:00 2001 From: Aisha Mohammed Farooq Darga Date: Thu, 5 Dec 2024 10:57:15 +0530 Subject: [PATCH 24/31] Remove Google Colab Compatibility Signed-off-by: Aisha Mohammed Farooq Darga --- .../hap/generate_hap_score_csv.ipynb | 20 +++---------------- examples/notebooks/hap/output/metadata.json | 12 +++++------ 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/examples/notebooks/hap/generate_hap_score_csv.ipynb b/examples/notebooks/hap/generate_hap_score_csv.ipynb index 763621f5d..7603bec2e 100644 --- a/examples/notebooks/hap/generate_hap_score_csv.ipynb +++ b/examples/notebooks/hap/generate_hap_score_csv.ipynb @@ -46,21 +46,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Open this notebook in Google Colab" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Click link to open notebook in google colab: [![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/IBM/data-prep-kit/blob/8381d353c3aa90d334e81ac3029ab774c753cc4b/examples/notebooks/hap/generate_hap_score_csv.ipynb)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Install dependencies for Google Colab environment" + "### Install dependencies" ] }, { @@ -162,7 +148,7 @@ "\n", "# Pick the first CSV file in the folder\n", "csv_file_path = os.path.join(input_folder, csv_files[0])\n", - "print(f\"Using CSV file: {csv_file_path}\")\n" + "print(f\"Using CSV file: {csv_file_path}\")" ] }, { @@ -331,7 +317,7 @@ " print(f\"Filtered output saved to: {filtered_output_csv}\")\n", "\n", "else:\n", - " print(\"No output files found. Please check the script or configuration.\")\n" + " print(\"No output files found. Please check the script or configuration.\")" ] } ], diff --git a/examples/notebooks/hap/output/metadata.json b/examples/notebooks/hap/output/metadata.json index f5df68315..ee3fed7b9 100644 --- a/examples/notebooks/hap/output/metadata.json +++ b/examples/notebooks/hap/output/metadata.json @@ -5,8 +5,8 @@ "job name": "hap", "job type": "pure python", "job id": "job_id", - "start_time": "2024-12-05 09:25:45", - "end_time": "2024-12-05 09:25:48", + "start_time": "2024-12-05 10:56:36", + "end_time": "2024-12-05 10:56:38", "status": "success" }, "code": { @@ -30,18 +30,18 @@ "num_processors": 0 }, "execution_stats": { - "cpus": 190.1, + "cpus": 59.3, "gpus": 0, - "memory": 10.46, + "memory": 12.22, "object_store": 0, - "execution time, min": 0.035 + "execution time, min": 0.031 }, "job_output_stats": { "source_files": 1, "source_size": 47309, "result_files": 1, "result_size": 40282, - "processing_time": 0.439, + "processing_time": 0.518, "source_doc_count": 65, "result_doc_count": 65 }, From 8d29d3a4884385127882b56cfa57d8a0329bf054 Mon Sep 17 00:00:00 2001 From: Aisha Mohammed Farooq Darga Date: Fri, 6 Dec 2024 11:00:15 +0530 Subject: [PATCH 25/31] Remove the changes made to the transform and reframe the example notebook accordingly Signed-off-by: Aisha Mohammed Farooq Darga --- .../hap/generate_hap_score_csv.ipynb | 862 ++++++++++++++---- .../customer_feedback_file.csv | 0 .../customer-feedback.parquet} | Bin examples/notebooks/hap/output/data.parquet | Bin 40282 -> 0 bytes .../hap/output/hap_complete_output.csv | 66 -- .../hap/output/hap_filtered_output.csv | 66 -- examples/notebooks/hap/output/metadata.json | 56 -- .../hap/python/src/hap_local_python.py | 18 +- 8 files changed, 716 insertions(+), 352 deletions(-) rename examples/notebooks/hap/{input => input-csv}/customer_feedback_file.csv (100%) rename examples/notebooks/hap/{input/data.parquet => input-parquet/customer-feedback.parquet} (100%) delete mode 100644 examples/notebooks/hap/output/data.parquet delete mode 100644 examples/notebooks/hap/output/hap_complete_output.csv delete mode 100644 examples/notebooks/hap/output/hap_filtered_output.csv delete mode 100644 examples/notebooks/hap/output/metadata.json diff --git a/examples/notebooks/hap/generate_hap_score_csv.ipynb b/examples/notebooks/hap/generate_hap_score_csv.ipynb index 7603bec2e..241dd8b24 100644 --- a/examples/notebooks/hap/generate_hap_score_csv.ipynb +++ b/examples/notebooks/hap/generate_hap_score_csv.ipynb @@ -49,11 +49,348 @@ "### Install dependencies" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "These pip installs need to be adapted to use the appropriate release level. Alternatively, The venv running the jupyter lab could be pre-configured with a requirement file that includes the right release. Example for transform developers working from git clone:\n", + "```\n", + "make venv \n", + "source venv/bin/activate \n", + "pip install jupyterlab\n", + "```" + ] + }, { "cell_type": "code", - "execution_count": null, + "execution_count": 39, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: data-prep-connector in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (0.2.3)\n", + "Requirement already satisfied: scrapy>=2.11.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-connector) (2.12.0)\n", + "Requirement already satisfied: pydantic>=2.8.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-connector) (2.10.2)\n", + "Requirement already satisfied: tldextract>=5.1.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-connector) (5.1.3)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic>=2.8.1->data-prep-connector) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.27.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic>=2.8.1->data-prep-connector) (2.27.1)\n", + "Requirement already satisfied: typing-extensions>=4.12.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic>=2.8.1->data-prep-connector) (4.12.2)\n", + "Requirement already satisfied: Twisted>=21.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.11.0)\n", + "Requirement already satisfied: cryptography>=37.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (44.0.0)\n", + "Requirement already satisfied: cssselect>=0.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.2.0)\n", + "Requirement already satisfied: itemloaders>=1.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.3.2)\n", + "Requirement already satisfied: parsel>=1.5.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.9.1)\n", + "Requirement already satisfied: pyOpenSSL>=22.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.3.0)\n", + "Requirement already satisfied: queuelib>=1.4.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.7.0)\n", + "Requirement already satisfied: service-identity>=18.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.2.0)\n", + "Requirement already satisfied: w3lib>=1.17.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (2.2.1)\n", + "Requirement already satisfied: zope.interface>=5.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (7.2)\n", + "Requirement already satisfied: protego>=0.1.15 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (0.3.1)\n", + "Requirement already satisfied: itemadapter>=0.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (0.10.0)\n", + "Requirement already satisfied: packaging in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.2)\n", + "Requirement already satisfied: lxml>=4.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (4.9.4)\n", + "Requirement already satisfied: defusedxml>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (0.7.1)\n", + "Requirement already satisfied: PyDispatcher>=2.0.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (2.0.7)\n", + "Requirement already satisfied: idna in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (3.10)\n", + "Requirement already satisfied: requests>=2.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (2.32.3)\n", + "Requirement already satisfied: requests-file>=1.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (2.1.0)\n", + "Requirement already satisfied: filelock>=3.0.8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (3.16.1)\n", + "Requirement already satisfied: cffi>=1.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from cryptography>=37.0.0->scrapy>=2.11.2->data-prep-connector) (1.17.1)\n", + "Requirement already satisfied: jmespath>=0.9.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from itemloaders>=1.0.1->scrapy>=2.11.2->data-prep-connector) (1.0.1)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.1.0->tldextract>=5.1.2->data-prep-connector) (3.4.0)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.1.0->tldextract>=5.1.2->data-prep-connector) (2.2.3)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.1.0->tldextract>=5.1.2->data-prep-connector) (2024.8.30)\n", + "Requirement already satisfied: attrs>=19.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from service-identity>=18.1.0->scrapy>=2.11.2->data-prep-connector) (24.2.0)\n", + "Requirement already satisfied: pyasn1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from service-identity>=18.1.0->scrapy>=2.11.2->data-prep-connector) (0.6.1)\n", + "Requirement already satisfied: pyasn1-modules in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from service-identity>=18.1.0->scrapy>=2.11.2->data-prep-connector) (0.4.1)\n", + "Requirement already satisfied: automat>=24.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (24.8.1)\n", + "Requirement already satisfied: constantly>=15.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (23.10.4)\n", + "Requirement already satisfied: hyperlink>=17.1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (21.0.0)\n", + "Requirement already satisfied: incremental>=24.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (24.7.2)\n", + "Requirement already satisfied: setuptools in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from zope.interface>=5.1.0->scrapy>=2.11.2->data-prep-connector) (75.1.0)\n", + "Requirement already satisfied: pycparser in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from cffi>=1.12->cryptography>=37.0.0->scrapy>=2.11.2->data-prep-connector) (2.22)\n", + "Requirement already satisfied: data-prep-toolkit==0.2.2.dev1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]==0.2.2.dev1) (0.2.2.dev1)\n", + "Requirement already satisfied: numpy<1.29.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.26.4)\n", + "Requirement already satisfied: pyarrow==16.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (16.1.0)\n", + "Requirement already satisfied: boto3==1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.34.69)\n", + "Collecting argparse (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1)\n", + " Using cached argparse-1.4.0-py2.py3-none-any.whl.metadata (2.8 kB)\n", + "Requirement already satisfied: mmh3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (4.1.0)\n", + "Requirement already satisfied: psutil in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (5.9.0)\n", + "Requirement already satisfied: ray==2.36.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.36.1)\n", + "Requirement already satisfied: fastapi>=0.110.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]==0.2.2.dev1) (0.115.5)\n", + "Requirement already satisfied: pillow>=10.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]==0.2.2.dev1) (10.4.0)\n", + "Requirement already satisfied: botocore<1.35.0,>=1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.34.162)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (0.10.4)\n", + "Requirement already satisfied: click>=7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (8.1.7)\n", + "Requirement already satisfied: filelock in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.16.1)\n", + "Requirement already satisfied: jsonschema in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (4.23.0)\n", + "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.1.0)\n", + "Requirement already satisfied: packaging in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (24.2)\n", + "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (5.29.0)\n", + "Requirement already satisfied: pyyaml in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (6.0.2)\n", + "Requirement already satisfied: aiosignal in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.3.1)\n", + "Requirement already satisfied: frozenlist in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.5.0)\n", + "Requirement already satisfied: requests in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.32.3)\n", + "Requirement already satisfied: aiohttp>=3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.11.8)\n", + "Requirement already satisfied: aiohttp-cors in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.7.0)\n", + "Requirement already satisfied: colorful in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.5.6)\n", + "Requirement already satisfied: py-spy>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.4.0)\n", + "Requirement already satisfied: opencensus in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.11.4)\n", + "Requirement already satisfied: pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.10.2)\n", + "Requirement already satisfied: prometheus-client>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.21.0)\n", + "Requirement already satisfied: smart-open in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (7.0.5)\n", + "Requirement already satisfied: virtualenv!=20.21.1,>=20.0.24 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (20.28.0)\n", + "Requirement already satisfied: grpcio>=1.42.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.68.0)\n", + "Requirement already satisfied: memray in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.14.0)\n", + "Requirement already satisfied: starlette<0.42.0,>=0.40.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (0.41.3)\n", + "Requirement already satisfied: typing-extensions>=4.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (4.12.2)\n", + "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.4.3)\n", + "Requirement already satisfied: attrs>=17.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (24.2.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (6.1.0)\n", + "Requirement already satisfied: propcache>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.2.0)\n", + "Requirement already satisfied: yarl<2.0,>=1.17.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.18.0)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from botocore<1.35.0,>=1.34.69->boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (2.9.0.post0)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from botocore<1.35.0,>=1.34.69->boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (2.2.3)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.27.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.27.1)\n", + "Requirement already satisfied: anyio<5,>=3.4.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from starlette<0.42.0,>=0.40.0->fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (4.6.2.post1)\n", + "Requirement already satisfied: distlib<1,>=0.3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.3.9)\n", + "Requirement already satisfied: platformdirs<5,>=3.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (4.3.6)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2024.10.1)\n", + "Requirement already satisfied: referencing>=0.28.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.35.1)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.21.0)\n", + "Requirement already satisfied: jinja2>=2.9 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.1.4)\n", + "Requirement already satisfied: rich>=11.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (13.9.4)\n", + "Requirement already satisfied: textual>=0.41.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.87.1)\n", + "Requirement already satisfied: opencensus-context>=0.1.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.1.3)\n", + "Requirement already satisfied: six~=1.16 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.16.0)\n", + "Requirement already satisfied: google-api-core<3.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.23.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.4.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.10)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2024.8.30)\n", + "Requirement already satisfied: wrapt in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from smart-open->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.17.0)\n", + "Requirement already satisfied: sniffio>=1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from anyio<5,>=3.4.0->starlette<0.42.0,>=0.40.0->fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (1.3.1)\n", + "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.66.0)\n", + "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.25.0)\n", + "Requirement already satisfied: google-auth<3.0.dev0,>=2.14.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.36.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jinja2>=2.9->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.1.5)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from rich>=11.2.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.0.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from rich>=11.2.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.18.0)\n", + "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (5.5.0)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.4.1)\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (4.9)\n", + "Requirement already satisfied: mdurl~=0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich>=11.2.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.1.2)\n", + "Requirement already satisfied: linkify-it-py<3,>=1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.0.3)\n", + "Requirement already satisfied: mdit-py-plugins in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.4.2)\n", + "Requirement already satisfied: uc-micro-py in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.0.3)\n", + "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.6.1)\n", + "Using cached argparse-1.4.0-py2.py3-none-any.whl (23 kB)\n", + "Installing collected packages: argparse\n", + "Successfully installed argparse-1.4.0\n", + "Requirement already satisfied: data-prep-toolkit-transforms==0.2.2.dev1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.2.dev1)\n", + "Requirement already satisfied: data-prep-toolkit>=0.2.2.dev1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.2.dev1)\n", + "Requirement already satisfied: bs4==0.0.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.0.2)\n", + "Requirement already satisfied: transformers==4.38.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.38.2)\n", + "Requirement already satisfied: parameterized in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.0)\n", + "Requirement already satisfied: pandas in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.2.2)\n", + "Requirement already satisfied: docling-core==1.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0)\n", + "Requirement already satisfied: llama-index-core<0.12.0,>=0.11.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.11.23)\n", + "Requirement already satisfied: fasttext==0.9.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.2)\n", + "Requirement already satisfied: langcodes==3.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3.0)\n", + "Requirement already satisfied: huggingface-hub<1.0.0,>=0.21.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.26.3)\n", + "Requirement already satisfied: numpy==1.26.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.26.4)\n", + "Requirement already satisfied: sentence-transformers==3.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.0.1)\n", + "Requirement already satisfied: docling-ibm-models==1.1.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.7)\n", + "Requirement already satisfied: deepsearch-glm==0.21.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.0)\n", + "Requirement already satisfied: docling==1.11.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.11.0)\n", + "Requirement already satisfied: filetype<2.0.0,>=1.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.2.0)\n", + "Requirement already satisfied: nltk==3.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.9.1)\n", + "Requirement already satisfied: torch==2.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.4.1)\n", + "Requirement already satisfied: mmh3>=4.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.1.0)\n", + "Requirement already satisfied: xxhash==3.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.4.1)\n", + "Requirement already satisfied: duckdb>=0.10.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.3)\n", + "Requirement already satisfied: networkx==3.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3)\n", + "Requirement already satisfied: colorlog==6.8.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (6.8.2)\n", + "Requirement already satisfied: func-timeout==4.3.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.3.5)\n", + "Requirement already satisfied: emerge-viz==2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.0)\n", + "Requirement already satisfied: beautifulsoup4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from bs4==0.0.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.12.3)\n", + "Requirement already satisfied: pyarrow==16.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (16.1.0)\n", + "Requirement already satisfied: boto3==1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.34.69)\n", + "Collecting argparse (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1)\n", + " Using cached argparse-1.4.0-py2.py3-none-any.whl.metadata (2.8 kB)\n", + "Requirement already satisfied: psutil in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.9.0)\n", + "Requirement already satisfied: docutils!=0.21 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.2)\n", + "Requirement already satisfied: matplotlib<4.0.0,>=3.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.9.2)\n", + "Requirement already satisfied: netwulf<0.2.0,>=0.1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.5)\n", + "Requirement already satisfied: numerize<0.13,>=0.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.12)\n", + "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.1)\n", + "Requirement already satisfied: rich<14.0.0,>=13.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (13.9.4)\n", + "Requirement already satisfied: tabulate>=0.8.9 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.0)\n", + "Requirement already satisfied: tqdm<5.0.0,>=4.64.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.67.1)\n", + "Requirement already satisfied: certifi>=2024.7.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.8.30)\n", + "Requirement already satisfied: docling-parse<2.0.0,>=1.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.6.2)\n", + "Requirement already satisfied: easyocr<2.0,>=1.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.7.2)\n", + "Requirement already satisfied: pydantic<3.0.0,>=2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.10.2)\n", + "Requirement already satisfied: pydantic-settings<3.0.0,>=2.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.6.1)\n", + "Requirement already satisfied: pypdfium2<5.0.0,>=4.30.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.30.0)\n", + "Requirement already satisfied: requests<3.0.0,>=2.32.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.32.3)\n", + "Requirement already satisfied: rtree<2.0.0,>=1.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0)\n", + "Requirement already satisfied: scipy<2.0.0,>=1.14.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.14.1)\n", + "Requirement already satisfied: json-schema-for-humans<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.4)\n", + "Requirement already satisfied: jsonref<2.0.0,>=1.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.0)\n", + "Requirement already satisfied: jsonschema<5.0.0,>=4.16.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.23.0)\n", + "Requirement already satisfied: pyproject-toml<0.0.11,>=0.0.10 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.0.10)\n", + "Requirement already satisfied: Pillow<11.0.0,>=10.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (10.4.0)\n", + "Requirement already satisfied: jsonlines<4.0.0,>=3.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.0)\n", + "Requirement already satisfied: lxml<5.0.0,>=4.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.9.4)\n", + "Requirement already satisfied: mean_average_precision<2022.0.0.0,>=2021.4.26.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2021.4.26.0)\n", + "Requirement already satisfied: onnxruntime<2.0.0,>=1.16.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.20.1)\n", + "Requirement already satisfied: opencv-python-headless<5.0.0.0,>=4.9.0.80 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.10.0.84)\n", + "Requirement already satisfied: torchvision<1,>=0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.19.1)\n", + "Requirement already satisfied: wheel in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.44.0)\n", + "Requirement already satisfied: autopep8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.3.1)\n", + "Requirement already satisfied: coloredlogs in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (15.0.1)\n", + "Requirement already satisfied: interrogate in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.7.0)\n", + "Requirement already satisfied: scikit-learn in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.5.2)\n", + "Requirement already satisfied: prettytable in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.12.0)\n", + "Requirement already satisfied: py in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.11.0)\n", + "Requirement already satisfied: pycodestyle in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.12.1)\n", + "Requirement already satisfied: pylint in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3.1)\n", + "Requirement already satisfied: pyparsing in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.2.0)\n", + "Requirement already satisfied: python-louvain in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.16)\n", + "Requirement already satisfied: PyYAML in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (6.0.2)\n", + "Requirement already satisfied: PyDriller in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.7)\n", + "Requirement already satisfied: pyperclip in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.9.0)\n", + "Requirement already satisfied: pybind11>=2.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fasttext==0.9.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.13.6)\n", + "Requirement already satisfied: setuptools>=0.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fasttext==0.9.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (75.1.0)\n", + "Requirement already satisfied: click in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (8.1.7)\n", + "Requirement already satisfied: joblib in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.4.2)\n", + "Requirement already satisfied: regex>=2021.8.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.11.6)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.2)\n", + "Requirement already satisfied: filelock in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.16.1)\n", + "Requirement already satisfied: typing-extensions>=4.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.12.2)\n", + "Requirement already satisfied: sympy in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.13.3)\n", + "Requirement already satisfied: jinja2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.4)\n", + "Requirement already satisfied: fsspec in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.10.0)\n", + "Requirement already satisfied: packaging>=20.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (24.2)\n", + "Requirement already satisfied: tokenizers<0.19,>=0.14 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.15.2)\n", + "Requirement already satisfied: safetensors>=0.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.5)\n", + "Requirement already satisfied: botocore<1.35.0,>=1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.34.162)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.10.4)\n", + "Requirement already satisfied: ray==2.36.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.36.1)\n", + "Requirement already satisfied: fastapi>=0.110.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.115.5)\n", + "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.0)\n", + "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.29.0)\n", + "Requirement already satisfied: aiosignal in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.1)\n", + "Requirement already satisfied: frozenlist in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.5.0)\n", + "Requirement already satisfied: aiohttp>=3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.11.8)\n", + "Requirement already satisfied: aiohttp-cors in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.7.0)\n", + "Requirement already satisfied: colorful in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.5.6)\n", + "Requirement already satisfied: py-spy>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.0)\n", + "Requirement already satisfied: opencensus in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.11.4)\n", + "Requirement already satisfied: prometheus-client>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.0)\n", + "Requirement already satisfied: smart-open in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (7.0.5)\n", + "Requirement already satisfied: virtualenv!=20.21.1,>=20.0.24 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (20.28.0)\n", + "Requirement already satisfied: grpcio>=1.42.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.68.0)\n", + "Requirement already satisfied: memray in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.14.0)\n", + "Requirement already satisfied: SQLAlchemy>=1.4.49 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.36)\n", + "Requirement already satisfied: dataclasses-json in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.6.7)\n", + "Requirement already satisfied: deprecated>=1.2.9.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.2.15)\n", + "Requirement already satisfied: dirtyjson<2.0.0,>=1.0.8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.8)\n", + "Requirement already satisfied: httpx in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.28.0)\n", + "Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.6.0)\n", + "Requirement already satisfied: tenacity!=8.4.0,<9.0.0,>=8.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (8.5.0)\n", + "Requirement already satisfied: tiktoken>=0.3.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.8.0)\n", + "Requirement already satisfied: typing-inspect>=0.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.0)\n", + "Requirement already satisfied: wrapt in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.17.0)\n", + "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.4.3)\n", + "Requirement already satisfied: attrs>=17.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (24.2.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (6.1.0)\n", + "Requirement already satisfied: propcache>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.0)\n", + "Requirement already satisfied: yarl<2.0,>=1.17.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.18.0)\n", + "Requirement already satisfied: scikit-image in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.24.0)\n", + "Requirement already satisfied: python-bidi in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.6.3)\n", + "Requirement already satisfied: Shapely in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.6)\n", + "Requirement already satisfied: pyclipper in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0.post6)\n", + "Requirement already satisfied: ninja in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.11.1.2)\n", + "Requirement already satisfied: starlette<0.42.0,>=0.40.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fastapi>=0.110.2->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.41.3)\n", + "Requirement already satisfied: MarkupSafe<3.0.0,>=2.1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.1.5)\n", + "Requirement already satisfied: Pygments<3.0.0,>=2.18.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.18.0)\n", + "Requirement already satisfied: htmlmin2<0.2.0,>=0.1.13 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.13)\n", + "Requirement already satisfied: markdown2<3.0.0,>=2.5.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.5.1)\n", + "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from dataclasses-json->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.23.1)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.16.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.10.1)\n", + "Requirement already satisfied: referencing>=0.28.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.16.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.35.1)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.16.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.0)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.1)\n", + "Requirement already satisfied: cycler>=0.10 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.55.0)\n", + "Requirement already satisfied: kiwisolver>=1.3.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.4.7)\n", + "Requirement already satisfied: simplejson>=3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from netwulf<0.2.0,>=0.1.5->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.19.3)\n", + "Requirement already satisfied: flatbuffers in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from onnxruntime<2.0.0,>=1.16.2->docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (24.3.25)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.27.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.27.1)\n", + "Requirement already satisfied: toml in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pyproject-toml<0.0.11,>=0.0.10->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.10.2)\n", + "Requirement already satisfied: six>=1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.16.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests<3.0.0,>=2.32.3->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.4.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests<3.0.0,>=2.32.3->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.10)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests<3.0.0,>=2.32.3->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.2.3)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from rich<14.0.0,>=13.7.0->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.0.0)\n", + "Requirement already satisfied: greenlet!=0.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.1)\n", + "Requirement already satisfied: mypy-extensions>=0.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from typing-inspect>=0.8.0->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.0)\n", + "Requirement already satisfied: soupsieve>1.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from beautifulsoup4->bs4==0.0.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.6)\n", + "Requirement already satisfied: humanfriendly>=9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from coloredlogs->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (10.0)\n", + "Requirement already satisfied: anyio in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.6.2.post1)\n", + "Requirement already satisfied: httpcore==1.* in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.7)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpcore==1.*->httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.14.0)\n", + "Requirement already satisfied: colorama in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from interrogate->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.6)\n", + "Requirement already satisfied: wcwidth in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from prettytable->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.13)\n", + "Requirement already satisfied: gitpython in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.43)\n", + "Requirement already satisfied: types-pytz in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.2.0.20241003)\n", + "Requirement already satisfied: lizard in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.17.13)\n", + "Requirement already satisfied: platformdirs>=2.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.3.6)\n", + "Requirement already satisfied: astroid<=3.4.0-dev0,>=3.3.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3.5)\n", + "Requirement already satisfied: isort!=5.13.0,<6,>=4.2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.13.2)\n", + "Requirement already satisfied: mccabe<0.8,>=0.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.7.0)\n", + "Requirement already satisfied: tomlkit>=0.10.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.13.2)\n", + "Requirement already satisfied: dill>=0.3.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.3.9)\n", + "Requirement already satisfied: threadpoolctl>=3.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-learn->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.5.0)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from sympy->torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0)\n", + "Requirement already satisfied: mdurl~=0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich<14.0.0,>=13.7.0->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.2)\n", + "Requirement already satisfied: sniffio>=1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from anyio->httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.1)\n", + "Requirement already satisfied: distlib<1,>=0.3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.3.9)\n", + "Requirement already satisfied: gitdb<5,>=4.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from gitpython->PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.0.11)\n", + "Requirement already satisfied: textual>=0.41.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.87.1)\n", + "Requirement already satisfied: opencensus-context>=0.1.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.3)\n", + "Requirement already satisfied: google-api-core<3.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.23.0)\n", + "Requirement already satisfied: imageio>=2.33 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-image->easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.36.1)\n", + "Requirement already satisfied: tifffile>=2022.8.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-image->easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.9.20)\n", + "Requirement already satisfied: lazy-loader>=0.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-image->easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4)\n", + "Requirement already satisfied: smmap<6,>=3.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from gitdb<5,>=4.0.1->gitpython->PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.0.1)\n", + "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.66.0)\n", + "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.25.0)\n", + "Requirement already satisfied: google-auth<3.0.dev0,>=2.14.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.36.0)\n", + "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.5.0)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.1)\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.9)\n", + "Requirement already satisfied: linkify-it-py<3,>=1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.3)\n", + "Requirement already satisfied: mdit-py-plugins in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.2)\n", + "Requirement already satisfied: uc-micro-py in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.3)\n", + "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.6.1)\n", + "Using cached argparse-1.4.0-py2.py3-none-any.whl (23 kB)\n", + "Installing collected packages: argparse\n", + "Successfully installed argparse-1.4.0\n", + "zsh:1: 2.2.2, not found\n" + ] + } + ], "source": [ "! pip install data-prep-connector\n", "! pip install 'data-prep-toolkit[ray]==0.2.2.dev1'\n", @@ -70,85 +407,151 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 40, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[nltk_data] Downloading package punkt_tab to /Users/aisha/nltk_data...\n", + "[nltk_data] Package punkt_tab is already up-to-date!\n" + ] + } + ], "source": [ "import os\n", + "import sys\n", "import pandas as pd\n", "import subprocess\n", - "import sys" + "\n", + "from data_processing.runtime.pure_python import PythonTransformLauncher\n", + "from data_processing.utils import ParamsUtils\n", + "from hap_transform_python import HAPPythonTransformConfiguration\n", + "from hap_transform import HAPTransform" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Step 1: Define Paths\n", - "---------------------\n", - "Define the paths for the script, input folder, and output folder." + "### Step 1: Setup runtime parameters\n", + "\n", + "- input-folder: Path to the input data to be used by the transform.\n", + "- output-folder: Path where the output file with HAP scores will be saved.\n", + "- doc_text_column: The column containing the text for analysis (For ex.: `Customer Feedback`).\n", + "- annotation_column: The column where HAP scores will be saved (default: `hap_score`).\n", + "\n", + "**Customization**: \n", + "- Ensure the column containing the text matches the `doc_text_column` parameter.\n", + "- If your text column has a different name, update the value of `--doc_text_column` accordingly.\n", + "- You can adjust other parameters like `--batch_size` and `--max_length` if needed." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 47, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "./input-parquet\n" + ] + } + ], "source": [ - "from pathlib import Path\n", "\n", - "notebook_dir = Path().resolve()\n", - "relative_script_dir = '../../../transforms/universal/hap/python/src/hap_local_python.py'\n", - "hap_script_path = (notebook_dir / relative_script_dir).resolve()\n", - "\n", - "input_folder = \"./input\"\n", - "output_folder = \"./output\"" + "# create parameters\n", + "input_csv = './input-csv'\n", + "input_folder = './input-parquet'\n", + "output_folder = './output'\n", + "output_parquet_file = os.path.join(output_folder, \"customer-feedback.parquet\")\n", + "output_csv_file = os.path.join(output_folder, \"customer-feedback.csv\")\n", + "\n", + "local_conf = {\n", + " \"input_folder\": input_folder,\n", + " \"output_folder\": output_folder,\n", + "}\n", + "\n", + "print(input_folder)\n", + "code_location = {\"github\": \"github\", \"commit_hash\": \"12345\", \"path\": \"path\"}\n", + "params = {\n", + " # Data access. Only required parameters are specified\n", + " \"data_local_config\": ParamsUtils.convert_to_ast(local_conf),\n", + " # execution info\n", + " \"runtime_pipeline_id\": \"pipeline_id\",\n", + " \"runtime_job_id\": \"job_id\",\n", + " \"runtime_code_location\": ParamsUtils.convert_to_ast(code_location),\n", + " # hap params\n", + " \"doc_text_column\": \"Customer Feedback\",\n", + "}" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "# Ensure the necessary folders exist.\n", - "os.makedirs(input_folder, exist_ok=True)\n", - "os.makedirs(output_folder, exist_ok=True)\n", + "### Step 2: Setup Input Data\n", "\n", - "print(f\"Script Path: {hap_script_path}\")\n", - "print(f\"Input Folder: {input_folder}\")\n", - "print(f\"Output Folder: {output_folder}\")" + "- Place your CSV file in the `input_csv`.\n", + "- Ensure the column containing the text matches the `doc_text_column` parameter.\n", + "- If your text column has a different name, update the `doc_text_column` parameter in later cells." ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 48, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Deleted file: ./input-csv/customer_feedback_file.csv\n" + ] + } + ], "source": [ - "### Step 2: Check for CSV Files in Input Folder\n", - "\n", - "- Place your CSV file in the `input_folder`.\n", - "- Ensure the column containing the text matches the `doc_text_column` parameter.\n", - "- If your text column has a different name, update the `doc_text_column` parameter in later cells.\n", - "- This cell sets up the file paths for the input file.\n" + "#Clear the existing input-parquet folder\n", + "if os.path.exists(input_csv):\n", + " for file_name in os.listdir(input_csv):\n", + " file_path = os.path.join(input_csv, file_name)\n", + " try:\n", + " os.remove(file_path)\n", + " print(f\"Deleted file: {file_path}\")\n", + " except Exception as e:\n", + " print(f\"Failed to delete {file_path}: {e}\")\n", + "else:\n", + " os.makedirs(input_csv)\n", + " os.makedirs(input_folder)\n", + " print(f\"Created folder: {input_csv}\")" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 50, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using CSV file: ./input-csv/customer_feedback_file.csv\n" + ] + } + ], "source": [ - "csv_files = [f for f in os.listdir(input_folder) if f.endswith(\".csv\")]\n", + "csv_files = [f for f in os.listdir(input_csv) if f.endswith(\".csv\")]\n", "\n", "if not csv_files:\n", - " print(f\"No CSV files found in the input folder: {input_folder}\")\n", - " print(\"Please place a CSV file in the input folder and rerun this notebook.\")\n", + " print(f\"No CSV files found in the input folder: {input_csv}\")\n", + " print(\"Please place a CSV file in the input folder and rerun this script.\")\n", "else:\n", - " print(f\"Found CSV file(s): {csv_files}\")\n", - "\n", - "# Pick the first CSV file in the folder\n", - "csv_file_path = os.path.join(input_folder, csv_files[0])\n", - "print(f\"Using CSV file: {csv_file_path}\")" + " # Pick the first CSV file in the folder\n", + " csv_file_path = os.path.join(input_csv, csv_files[0])\n", + " print(f\"Using CSV file: {csv_file_path}\")" ] }, { @@ -161,11 +564,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 51, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CSV file converted to Parquet format at: ./input-parquet/customer-feedback.parquet\n" + ] + } + ], "source": [ - "parquet_file_path = os.path.join(input_folder, \"data.parquet\")\n", + "parquet_file_path = os.path.join(input_folder, \"customer-feedback.parquet\")\n", "df = pd.read_csv(csv_file_path)\n", "df.to_parquet(parquet_file_path, index=False)\n", "print(f\"CSV file converted to Parquet format at: {parquet_file_path}\")" @@ -175,83 +586,69 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Step 4: Define HAP Parameters" + "### Step 4: Invoke the transform\n", + "Use python runtime to invoke the transform" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 52, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "10:55:04 INFO - hap params are {'model_name_or_path': 'ibm-granite/granite-guardian-hap-38m', 'annotation_column': 'hap_score', 'doc_text_column': 'Customer Feedback', 'inference_engine': 'CPU', 'max_length': 512, 'batch_size': 128} \n", + "10:55:04 INFO - pipeline id pipeline_id\n", + "10:55:04 INFO - code location {'github': 'github', 'commit_hash': '12345', 'path': 'path'}\n", + "10:55:04 INFO - data factory data_ is using local data access: input_folder - ./input-parquet output_folder - ./output\n", + "10:55:04 INFO - data factory data_ max_files -1, n_sample -1\n", + "10:55:04 INFO - data factory data_ Not using data sets, checkpointing False, max files -1, random samples -1, files to use ['.parquet'], files to checkpoint ['.parquet']\n", + "10:55:04 INFO - orchestrator hap started at 2024-12-06 10:55:04\n", + "10:55:04 INFO - Number of files is 1, source profile {'max_file_size': 0.04511737823486328, 'min_file_size': 0.04511737823486328, 'total_file_size': 0.04511737823486328}\n", + "10:55:06 INFO - Completed 1 files (100.0%) in 0.008 min\n", + "10:55:06 INFO - Done processing 1 files, waiting for flush() completion.\n", + "10:55:06 INFO - done flushing in 0.0 sec\n", + "10:55:06 INFO - Completed execution in 0.043 min, execution result 0\n" + ] + } + ], "source": [ - "hap_params = {\n", - " \"model_name_or_path\": \"ibm-granite/granite-guardian-hap-38m\", # Default model name\n", - " \"annotation_column\": \"hap_score\", # Output column for HAP scores\n", - " \"doc_text_column\": \"Customer Feedback\", # Input column containing document text\n", - " \"inference_engine\": \"CPU\", # Inference engine (CPU or GPU)\n", - " \"max_length\": 512, # Maximum token length\n", - " \"batch_size\": 128, # Batch size\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Step 5: Run the Transform with defined HAP Paramters\n", - "\n", - "This cell executes the HAP transformation script:\n", - "- `--input_file`: Path to your input CSV/Parquet file.\n", - "- `--output_file`: Path where the output file with HAP scores will be saved.\n", - "- `--doc_text_column`: The column containing the text for analysis (default: `Customer Feedback`).\n", - "- `--annotation_column`: The column where HAP scores will be saved (default: `hap_score`).\n", - "\n", - "**Customization**: \n", - "- If your text column has a different name, update the value of `--doc_text_column` accordingly.\n", - "- You can adjust other parameters like `--batch_size` and `--max_length` if needed." + "%%capture\n", + "sys.argv = ParamsUtils.dict_to_req(d=params)\n", + "launcher = PythonTransformLauncher(runtime_config=HAPPythonTransformConfiguration())\n", + "launcher.launch()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 53, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 53, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "# Copy the current environment variables\n", - "env = os.environ.copy()\n", - "\n", - "# Set Environment Variables for HAP Parameters\n", - "os.environ[\"MODEL_NAME_OR_PATH\"] = \"ibm-granite/granite-guardian-hap-38m\"\n", - "os.environ[\"ANNOTATION_COLUMN\"] = \"hap_score\"\n", - "os.environ[\"DOC_TEXT_COLUMN\"] = \"Customer Feedback\"\n", - "os.environ[\"INFERENCE_ENGINE\"] = \"CPU\"\n", - "os.environ[\"MAX_LENGTH\"] = \"512\"\n", - "os.environ[\"BATCH_SIZE\"] = \"128\"\n", - "os.environ[\"INPUT_FOLDER\"] = input_folder\n", - "os.environ[\"OUTPUT_FOLDER\"] = output_folder\n", - "try:\n", - " result = subprocess.run(\n", - " [\"python\", hap_script_path],\n", - " check=True,\n", - " text=True,\n", - " capture_output=True\n", - " )\n", + "#The specified folder will include the transformed parquet files.\n", "\n", - " # If successful, print the result of the transform\n", - " print(\"Transform completed successfully.\")\n", - " print(result.stdout)\n", - "\n", - "except subprocess.CalledProcessError as e:\n", - " # If there was an error, print the error message\n", - " print(\"Error occurred during transform execution.\")\n", - " print(e.stderr)" + "import glob\n", + "glob.glob(\"python/output/*\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Step 6: Generate the Output CSV\n", + "### Step 5: Display Output in a Readable Format\n", "\n", "This step checks for any existing CSV files in the output folder and removes them before generating new ones. The following actions are performed:\n", "\n", @@ -260,65 +657,220 @@ "3. **Remove Old CSV Files**: If any previous output files (`hap_complete_output.csv` or `hap_filtered_output.csv`) exist, they are deleted.\n", "4. **Read Parquet File**: The Parquet file is read into a DataFrame.\n", "5. **Filter Data**: The relevant columns, `doc_text_column` (from the environment variable) and `hap_score_column`, are selected from the DataFrame.\n", - "6. **Save New CSV Files**: The filtered data is saved into two new CSV files:\n", - " - `hap_complete_output.csv` (containing the full output)\n", - " - `hap_filtered_output.csv` (containing only the filtered relevant columns).\n", - "\n", - "This ensures that only the latest output is retained, and no old files remain in the output folder." + "6. **CSV Output**: Convert the parquet output to CSV \n", + "7. **Display Output**: Display Output in the notebook for a quick reference \n", + "\n", + "Example Output:\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Customer Feedbackhap_score
0Rating: 4 Comments: \"Service was prompt, but ...0.000195
1Rating: 5 Comments: \"Great help from Peter! H...0.000153
2Rating: 3 Comments: \"The service was quick, b...0.000169
3Rating: 5 Comments: \"Excellent service and ad...0.000158
\n", + "
" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 55, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Deleted old CSV file: ./output/customer-feedback.csv\n", + "Renamed Parquet file to: ./output/customer-feedback.parquet\n", + "Transformed CSV file saved at: ./output/customer-feedback.csv\n", + "\n", + "Selected Columns (Customer Feedback and HAP Score):\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Customer Feedbackhap_score
0Rating: 4 Comments: \"Service was prompt, but ...0.000195
1Rating: 5 Comments: \"Great help from Peter! H...0.000153
2Rating: 3 Comments: \"The service was quick, b...0.000169
3Rating: 5 Comments: \"Excellent service and ad...0.000158
4Rating: 2 Comments: \"I’m really frustrated. T...0.000875
5Rating: 4 Comments: \"The service was helpful,...0.015778
6Rating: 3 Comments: \"The support was helpful,...0.000221
7Rating: 5 Comments: \"The service was very qui...0.000179
8Rating: 5 Comments: \"The support was excellen...0.000236
9Rating: 2 Comments: \"I’m frustrated with this...0.000448
\n", + "
" + ], + "text/plain": [ + " Customer Feedback hap_score\n", + "0 Rating: 4 Comments: \"Service was prompt, but ... 0.000195\n", + "1 Rating: 5 Comments: \"Great help from Peter! H... 0.000153\n", + "2 Rating: 3 Comments: \"The service was quick, b... 0.000169\n", + "3 Rating: 5 Comments: \"Excellent service and ad... 0.000158\n", + "4 Rating: 2 Comments: \"I’m really frustrated. T... 0.000875\n", + "5 Rating: 4 Comments: \"The service was helpful,... 0.015778\n", + "6 Rating: 3 Comments: \"The support was helpful,... 0.000221\n", + "7 Rating: 5 Comments: \"The service was very qui... 0.000179\n", + "8 Rating: 5 Comments: \"The support was excellen... 0.000236\n", + "9 Rating: 2 Comments: \"I’m frustrated with this... 0.000448" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ - "import os\n", - "import pandas as pd\n", - "\n", - "# List all files in the output folder\n", - "output_files = os.listdir(output_folder)\n", - "\n", - "if output_files:\n", - " for file in output_files:\n", - " if file.endswith(\".parquet\"): # Check for Parquet output files\n", - " output_file_path = os.path.join(output_folder, file)\n", - " output_df = pd.read_parquet(output_file_path) # Read the Parquet file\n", - " print(f\"Complete Output Parquet File Path: {output_file_path}\")\n", - "\n", - " # Define the output CSV file paths\n", - " complete_output_csv = os.path.join(output_folder, \"hap_complete_output.csv\")\n", - " filtered_output_csv = os.path.join(output_folder, \"hap_filtered_output.csv\")\n", - "\n", - " # Remove old CSV files if they exist\n", - " if os.path.exists(complete_output_csv):\n", - " os.remove(complete_output_csv)\n", - " print(f\"Old complete CSV file removed: {complete_output_csv}\")\n", - "\n", - " if os.path.exists(filtered_output_csv):\n", - " os.remove(filtered_output_csv)\n", - " print(f\"Old filtered CSV file removed: {filtered_output_csv}\")\n", - "\n", - " # Filter the output DataFrame to only include the relevant columns\n", - " hap_score_column = hap_params[\"annotation_column\"]\n", - " doc_text_column = os.getenv('DOC_TEXT_COLUMN') # Read from environment variable\n", - " filtered_df = output_df[[doc_text_column, hap_score_column]]\n", - "\n", - " # Print the filtered DataFrame (only showing the HAP score and document text)\n", - " print(f\"Filtered Output (only HAP score and document text):\")\n", - " display(filtered_df)\n", - "\n", - " # Save the complete output as a CSV file\n", - " output_df.to_csv(complete_output_csv, index=False) # Convert the Parquet to CSV\n", - " print(f\"Complete output saved to: {complete_output_csv}\")\n", - "\n", - " # Save the filtered output as a CSV file\n", - " filtered_df.to_csv(filtered_output_csv, index=False)\n", - " print(f\"Filtered output saved to: {filtered_output_csv}\")\n", - "\n", + "# Locate transformed Parquet files in the output folder\n", + "output_parquet_files = [f for f in os.listdir(output_folder) if f.endswith(\".parquet\")]\n", + "\n", + "if output_parquet_files:\n", + " # Clear existing CSV files in the output folder\n", + " for file_name in os.listdir(output_folder):\n", + " if file_name.endswith(\".csv\"):\n", + " file_path = os.path.join(output_folder, file_name)\n", + " try:\n", + " os.remove(file_path)\n", + " print(f\"Deleted old CSV file: {file_path}\")\n", + " except Exception as e:\n", + " print(f\"Failed to delete {file_path}: {e}\")\n", + "\n", + " for output_parquet in output_parquet_files:\n", + " original_parquet_path = os.path.join(output_folder, output_parquet)\n", + " try:\n", + " # Rename the Parquet file\n", + " os.rename(original_parquet_path, output_parquet_file)\n", + " print(f\"Renamed Parquet file to: {output_parquet_file}\")\n", + "\n", + " # Convert the renamed Parquet file to CSV\n", + " transformed_df = pd.read_parquet(output_parquet_file)\n", + " transformed_df.to_csv(output_csv_file, index=False)\n", + " print(f\"Transformed CSV file saved at: {output_csv_file}\")\n", + "\n", + " # Display selected columns in tabular format\n", + " if 'Customer Feedback' in transformed_df.columns and 'hap_score' in transformed_df.columns:\n", + " display_df = transformed_df[['Customer Feedback', 'hap_score']]\n", + " print(\"\\nSelected Columns (Customer Feedback and HAP Score):\")\n", + " from IPython.display import display # Ensure pretty display in Jupyter\n", + " display(display_df.head(10)) # Display the first 10 rows\n", + " else:\n", + " print(\"The required columns ('Customer Feedback' and 'hap_score') are not in the transformed data.\")\n", + "\n", + " except Exception as e:\n", + " print(f\"Error processing files: {e}\")\n", "else:\n", - " print(\"No output files found. Please check the script or configuration.\")" + " print(f\"No Parquet files found in the output folder: {output_folder}\")\n" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/examples/notebooks/hap/input/customer_feedback_file.csv b/examples/notebooks/hap/input-csv/customer_feedback_file.csv similarity index 100% rename from examples/notebooks/hap/input/customer_feedback_file.csv rename to examples/notebooks/hap/input-csv/customer_feedback_file.csv diff --git a/examples/notebooks/hap/input/data.parquet b/examples/notebooks/hap/input-parquet/customer-feedback.parquet similarity index 100% rename from examples/notebooks/hap/input/data.parquet rename to examples/notebooks/hap/input-parquet/customer-feedback.parquet diff --git a/examples/notebooks/hap/output/data.parquet b/examples/notebooks/hap/output/data.parquet deleted file mode 100644 index 2aa2e88e829def4f213e82023e7a0ceb54893645..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 40282 zcmeFZWmH_vwl3VbyL;pA?(PI9xYGo84esvlPH+oOAhni%nIJSLVwt{NX;S;hDt6YRHKaGH@uWwU8)e5sxl18j-dV+wS;i3{_53tK(%jm(j z<783&c*g8mrn+<5dnaZ?z-@4&)G#$Wq-pjei{WjEeFRInwU%R>BnHLfk?hYDrIXXUq{< zhH_D;;&ml@<9lZD&tIul9FEMgF~7SUHZ=E!ey=GGIs+Ki?2O5`nCP3RVd+HWD^MTU zAsx9=wvrvGqOZjpR=Gwj>;WN56L7hG!WmaQtZwuH^n0ksA@-!hOZ&J)9P3|<%zg#U zF?tTGzL=~9b*1m;t2hXToXN_9xPtC*XZN_)C`+Q8qPrk$H0E4sJpC};Afww;1t*Ry z#fh6$RE+EdQx!tG`eP*u_j1KP-jVE`qM-7x^zbEB>3Cx^5-M*mq~Ibc)o!>1xX{w4 zL_5XR2K%_=_|xBa4~m$rIS1t^u{5*bkp!>!Grg^$=2yD)2rECg+xtyr4j};)3pPvL zh05G(hZ5Ry%jce?9Hh(f#QmDz`~ZAN()-1$E-JJFKtqv0Cyxc#Z~%CO8oXxTbl4H6 zWNxv%qwRpe97q88fDSGI0PuM}63#fJ5{~abjco>E0~*j2 zj)oRk_65Sy!j^t)7J8W-T8?KX+Ok2{!%|{O{o)1u5&aybCo~<+f~Psk$Jn_eF|0d> zgJP3T$ql!b1pzG#>|Gegmmc>vt)&ShA11IIG#H4@A-3Vo6+TY3`<3 ztJ zJ(O4G{~Ln8r8^NGX9w!vof|xaFdP6Hq3Z|IA8`oJ7C&sRw~mIPk;@Qd0EQ<>SBS{$ zXa^W``2K`Q=n3T3w4WIXS*X~dxt6%!3sQ@$NeIbg$!jTdsX@IJ(hW4IH&uGqY~5g& z?^5O3?mh2w6DS@U6zLN;lsKHso{^A)To_pjQz=q4QTMwkr0sWCVQ+OG+0cX9IL}Pj z{OOmXRpdW7M1fn&P-sFIy^}IT_RVwudYG~*iQ`%Qn?v@$ReNof<59=IUT*lVd9L4z zNFDEQObQ?VFNyf@TSx!k02e^_H&?;1{^<07)x3@fh;g9*!@Q9G)4b%R{$X7Gh`8mj zG{bl_{aD2R2SEQ;^#_DF4T%4)`j0T~*np)Y3zEQ=DF~DeIg8)pb$(xYEbcLNqeyT~ z7Tbn0`yCMV0fqp5z%L8bsrv?!1|PTV6F6x}8qtW_+Gg2dk03+lMHbVOgfmFz1RsVq zzzQaPM9<<~A%J_$XhTC{Kz9{aj2TDpR;d+^KFx<>(5w}yWwPv3KZK#uoIoxqce5ch z0#*NPM)*Xa^X>zES=-5Mm_d1h9&f5wF+ zWrr1?-ymyXj~D5W5dYhM(F9>&DgP{RC_K_HP7y>l(O?ZEj6lA!IojcJ6$bM?KZXsQ)9ZIkA;gpUepf+wC=OOlrey=k42wF z0xFhW40NR)TQs}}LV}XG_~Po94b{CB3ci%C5kJQjB0Ib&%KIAOTh}#MGiKM!<8Xi7 zfek8iuLgSyh05HsV3Fl5i;8V3=SZN1m7L{hGl5ueNFoUt1{<4I=q$gC)*7s8JJ2On z`*K#zagMIFEUpVxwHE2_qG8+M_Y(XaP*`G$E8xigtG zP9J^K#axh;l^oL)L`1!u1FtL9N=68z1R3 z(+a6Deg_pq->+yw+qenR;rpK8ZvLQij#D}B4ff#kouip5CqDi`lIi=dq>^BrfX*39v|Nn9R65b{tAEpyz_s>Qa{oC(+#6Y zzk&dYTUfY&TwVWRs}&@;p(r%bG&Jo*um1S|6!PENYy%9=5;!`({bzL40ieJDxL~i( zUzoqYygPU)+J3=VdC9zed0u}$Y| z<~`vP@A8iwk2|_=NMDu??5>6OLRXA_zW=gy40o6D?O|X1vVPV1_%*t|yf6T_c%lAy z;QxMT2!bQ^+WMEnc5hw{zw==>mS-J5c^7!;{9j`vKK#4qfAotF561cD^Xg{*`_;dc z_g_P&lqs=XF}h>}+Zz);v{0#eDXpb6lYc}{atq*F>UjO#MC$ZRh@w2|&QgXyF@&a+ z^`G7qS=GwX0R&Wbw>Ja1{6l4vaJZo`H0jJVg$!W#`ro1a4*>sGRg-i$pD_O2z0x3* zQ2;Ex%c(C?ulIt|F(}AMVVAot+_GI%of0yQhornHSi>)09ePH$DeU(iRhluMl}bupfMcoOD( zhQmd@92ImlrkC`lruo;vBy5d4)iQ-8te5A@LBml?M|z2LDGa>5)&2dX(nT)+sX|6w ztA!EL+{~}&WV{8Fn;+ZNRnSp*Sgfc$ArXO3{FX8o1va11Pzu7LlRf9rWM(GeQp%+F zCL;?+3X?VW4G4ozvj~bhy#xdEVPa13ip-a-a-plK0sTsXDjFyBt_$GRt&cGyTUi)T#{`|QM?!4nJ zr29<`m-#IF$6g<#o6pz@TRuT@Qq%F}olC_Vl#y9T42Saxu5npNdNe#YSQk(1bUzu6 zC4adRJ;r6>@u+_rgEB7jnh{?p%?=^u#OsY5U*w;g`EKb=%t^*|citi`&1$1ggbAws zSd8ZrK+iipF^eWV%(745bH^Nd%1^{l! zOCZtrG;D;6FlEP;cCeS83OUM`8ScdESahdO$@fG*yd8Puuwk}Oa$SvzTm;Eo>86ZU zJ|-b}*(7DBTldgoVvTO5AViOM~v@qu`#E<$cNRhQ><$1Bv z!~p!|d4SkEClN)Pd7-*Lvr);9srbf86VbS|QRiw7I>3yw)wNLKIEEfQfHPniX`|`E z+<{~(x0rk^J%4CP4I&vUggsEqwA6WwLpzKW&I zX9)vbOz6c!Je&x|?veqbu+`^O;cPxUG^&hOh3+5j^Oh=U#1rXhMh>Km6o=76gvDO? zv#+p(TOPECJASY7QHAf^P8M@#HTc%py7bb?Xj`|`3P;SQQl11b#!sFGj*_F z0;;G3!S~s?@#)#wz|~Z(!M;%31LWWiVghQIIsj!{OdZUvUCkYtfRf@selB)4-oHHY z|LTHwT>7736kWs0+R4e<;oZMS(s@SQMQoZy37R{$*GT%mKd!%Z;NVB2C+L57;P4P- z2mnpgllxtC2rwC;A#1p=KZW|kd5N;EBl!csVo>OQ| zGU!UP`T&IMlVQD30mIvFB0Gk2xf*8n9f9Xb?FR}`?j30-ADhpXSPrd*wztKox_jAb zn9Eo?{Mm;nvUWPEF@?Ra%qJ(@pMM>7BNOj`A|YsG3HK8{@KJS4GL;K+A@>e3vQI5D zEj!g}-IgCQ!COhFIV64^gb?_7i^HLrd{>6(A4P(+#k!T`a2(HQ`&@V|k^N}9X()^&_`f9K!~aF-Ykd9hSHWQZ_n))c zRsLlqpyFcj`Wwc-tOURhTD@>I>2GKrWWg(e|BmT@K>D{6sbU-wr`hh`XHu$K*b1PE zWuHY3;G9wlWo!+?&;9I_PZXxHjc5!65y2p^r zTINKq*ly1!pfn1Qqtqby;@p^B%{G_Z1^ESX1Jc7s;{#iU9#FUmhj@%-$*S!}b}(HNUQWBh2)_Do11f*qf07S(#>(bF12*QmlClYidQ zk1BycEcQSyY`oEJeBwHQJrMa#bzB+T)dlhO_Gsk-Uz~y&!`)b!YOBzB^2f3dA(fBN zu*ih@D`GdRSKB_;vhOnSpD%VF#0D^AmNb#)(#j~V>D{u{ex=%ym7SG+EAu*JcgV8+ zoRoCQC6sw=uo++X_!iDVBP}8R5trP@)+p$3S`maRtex(Y=M~KyUPpD+l(IbTTU?a9 zpLsT+ck_jHd41~x#UcBrhYZ5XvctB_%Z&HurTLr`8txz@8W!(ZAI>XM*R$)|L1keq zpIF=Gfu!kO{e!k;D;6S&!N|qy?@pl^wpxb@U;B-PX-^)RRr7uk!(sX{cvVN&GsHVM zv*a0fUeTtv9A*5BNW`kAD({hgT$NoecL*30q;G#x|D4jqbxWi`z(YeZQsGUlCa>62 zp6=51fY7YDG3|*v5Av@Rbw1&m#v=G#)|1?h)XjW5S7vs|{25i2%>=j|g%V1NsvWU7 zK+D2TqF+Zv(|QnlT-o^O!Y~J_qxwf=+-(YE}X#Nzbw9@pOJ@aU(`hx`v^2VhoB zU?GM*qQ_cNXC4+mMc;fVwZBdq9EQe1@{}>e!6br*twID(Ao$*-X?^89?zU?C1l_t? zan%s^{gP(Nvcv49#S!n=E>u~TZf;pGitD~s9N6hh)a`Jxh*0De<&EOx$n+Mgz7Iva z`w$jJ#A4x089ysOhb200dXF2C{bXmV?WylbZfDy+FB|q@eP(|D;Ps68nVEa@d+UC7 zdQ4C>)Q=U3tr2pF5XwsKv+B6F6WDFx%%a0Zofg(McFK?u#wU!5aVlYz9e!GkcXG`3 zs$`qr;!)Kw@v5g**<-tlQb{Zb{bC5t6n3EDshj}h=a?w3c0t8593+J0PmIZTq~Aba zGkI1Wae|_c>!Cnz*J?XeVo=4WusisXIw&zW!wF z`P-IsU6Fyl!XZW`j%-Q{LPG zY-l0sf+|?f3Run=d*`G#tM2|_4`24fMVe@kDm~eT%ZT;}pL^@@>_CzhOi(07jn65{ z7kTj^(QuZLiD`|-M!Jkj#+whAT_LsJtp{-^?#LEB1O3mdJmSN8oZ%RE^K0rp;Ql{t zjV1*7DLY${hllxalmR3OcKsMGef&t`nNAnKmZ89SkoMcYXqwe#i4+|UH42NS0V#Lm zn(G;`f0`?%+7!EtNtxN^$CH>RU_HC@C9e>IMvS#m6_@>^nQI_a1l{{dKZ@iUT5X3} zCz`^s)pCpq(;Zbu;@@e`UA7L}`k{lN_to(Bj9+t4cV$hkNZJQhkYSvw!}KJ-*&oCXH$} z_i|#D8%-27%5_g?foLLpkagNdq@-C&KeQ+mG>{;wcVLko03LR$_Imu~zfVrv{A?RFY}|g*5LH27%XM#$q7}x(n*-1$MffLV`O-A>M8& z#~D{J(*sdA{=(1cUX0GBAr{XVsbDxYIFvcCJ!OyYP$a%+)}UoO$SR~_;XxT<`*3Qe z?^AG6R()8B@fNvNFGj?(h38Noy-avpf3kx7YfJxqa;+a&{k~i90@7+_^k<9wufpT( z5E>68y|pp1?B^i!M}21pOr4d$C=k$Pb>t0XxSgO~>7!6A+Qz3xX2Y0CLTtb`>MR={ zG*ttVay(_99*sgarq4B@c0NmHwVk7VtB(hQTbS?I|Ac-2aHr^m#;oO7VZL7c4kopI zJUH~D2`RpcuJCL0lrZs&6hFw&bDaxyq*L047byLZqp@Cxk}(Iuhcb1)7gNpbPmqL! zv7tZa;(1zYiLPO=!G>P_bZTO=o6PxLAq<^;N5OP}?@L9eGZ|yl>$zD%QEcve;@(k@ zw}@aKv8;OD$Q#gkS4XotS1ZOgZfDc3x~v_ANm;R-&YvB{Z!@-FYhx}nt_^>rk3v>_ zXJH4Xr+I6Xw8YpA$2(U4AoiWqaOuZp49lp7Wx8wWYz4wMPfM+{dEXvQ}F{oib# zKP{H#-xGos%h;b5f24g%Do5Vz;U>y?oo;=>l$At=`+9hW2Xt|b_FddPi|>9(6*KFl zc7_qv&$=>X1V z;^u$Uhyarw8`4pYwnm)Yt}Ewt+@>^K1`>9GeHfv!7{t0c*h<>N!sf6!y$QQ?eHca_ zEFE_$)alg&7$T*IF~!Z&J~1IcFl({+Tx@lsPkiy%#MVQVoXP0@`H-C2fz#@ZZzN$A z;e?;~P_c;nwAeyTDEM$rW4o7p{20KsiJ4ZsqOgST6=YE)BJnSmlmtJQ$|>@_?aiJ? zWr?7Sn(W(mvYqNYEfXoGVra#eSRPRisMlZUQ2A!!%r4B~5F4o(4$|*<)AXvn<0g@OgvCBA_9_F-)s*?v5dgCm0j~8gKy&%n$vy6A zxE^-d8J1zx%;d&ST${;9>ej&;kw)$)6let{F!WQf=~;M_0Km44$e^#K)y!AbQ)T6> zS!)A@UdtT`Mr8)5~NIXfUODp);GOty1pJ3`X4hF43P?KRT1{1Rk* zd`%bMg5yN(WjwtVC$aQP{Kl{KisIgWHwm_?|0+bqLFTXw4qL>+eDkU~9*CDsxWWn> z{ht?Y$Ei6o_w(~IAigB`CP?-J#;4hb)_*5Y@Y;2hkt3dP#Uu7jkZ_uS7qCDSHSiGN zx8OgR6-+F;g5cU_Ds4YuhAUSLAP7$c$I?gCNKwfrbHS#dtBT2qu~0ADpFx4QS$h+# z-GYc=#fQ<&BEzL#e^Ftw<)l^^mM22t95S!SX1UG6{f=YHElL45oMIA4YaS}YjT5!j zoy}FpTSy)%;{^;Z9V(Z4Ow_9PCeO*l3a2?Ib*Xah^h+d{eA&|znp9w{V7xKhDU?%AkI0wfautqyWNh zakEh>oMlZ_-GaRfU)8TQ*{BuUh*|H6o$sx>dpa_z&FSI8_8N)rmOb?Pxx~EKz7*7n ze~D%O3{}T$_03>UHOJE=QV0GG=h5IpNj6`RPjf5IZYTeO7pL2t5p}oaBvU6tIqfpu zSTk8)Y3q~f^UvV3E>ums1S8QsL)>-LDijew;J(@Bw#4->G;_>d)5?)36w%5c-Ncs&T^GkQ9P>W(-3I-V&a zuKIbj;P4r9tRmbs9SkWJ>B*>!qu1kkc1m8K@Uos3p~oU;P!&ZF*FnN}t@(=Zbioq1 zENGK^NHy$5`ZH@<@?h2?I{gK8jh|$f&$U(bIxdS#--=lzdWe2Fs|s3Rh$R4{Cs8p7 zkmuXN?JvUXv~XWMSuvchD`j1;h1jg--JuJO-b%~VsR85IwCs}M3*QLG--t4iD<3sf zp{zZ3)3$VP)c)G%o?F6XqpO!h3?S{I+#2ydBn!SaPeZ3DFPpH;E3)SzVnnkfeu9L< z7X0Fj#vh4AEV<1?EMI6`gW@?Zk!(~&q%yXST>6BSpTdE=GN_3c7t)Q6p93ZDiDi@j2 z3+x^kT)+WY{BDi*j-JROf|zY<;w}V?KxQHflK2JT9I0XP0Erjzv0-HuzlN9zhwb=3|9!Okrz96Zf#W-x4JFQdU*trpV8u@Mf~!IxJJ2 z?v#rEGL=CPgZ&{LXi60v=aQXe(z)trOgs6pvHMZv2GKYl-qoX<3kKr6yHTI`ShoVC zClOpX%j(wlE2Op-ogs}>AcR_k?TVk z0{f*rN&RCoW{eCrC!54xM#B@)SiS>Y(91_))m|FRzNwQ!`~E7aoXNr$?J9WYgjQCO z8)8>~T-%tI!ek zWYd7d=+SLU%7mqRkI*%g<1XBLE7yU`^JbqKX>;mDlG?( zQ&>Z8#r0U0CFl!MxG-2U9G?5@=xb+bmq}oCM$MpnL4CZ#<8;px;W$HSn3-ar4{seSVt5wC*yMlg>Wr?Zo*%~Lq0n@sJmrf#R959iKzjA*bC_`N<>jc8;{ zwAv~U^YxR|CZJOv&7(5=pnPt@K#W%XIksQu95F=Y2k$wG zn?mY|X1lAz?xi>S(ID)P_}LvlA@iyNe#?XnXS^w{RER>Fd=y!Y-{88?*&H#|JGkt? z(1j@He&Nh-f)YsM>hqc7&(q}W$hj^cLem(}(wI~haLnoWG(?;O2XK&Y3pXi{-XbHi z!?^lk7joI&y*miA*$)xrkzcw|R@Kr-bp;3%J}=8ERiZZ05)J<-yy~67t>5%?DyH`v z5rE$GW=S{-dYTjsb}FNc@80&Ij^(`uUA@$|$2X$xC`w*qAp zZ!P4WN<>f)nlTN-B}tF!YbZjm%Ib0}D2U8u@y!g<)}noKa~;{V7!F}#L@XaQcZ`Yz z(bmGD=DXMdYf!-v%^k|*oDR$*{YRj}I9mkoT$q}bl znV&)BeG+6>TBur~J7g*D9kHfY!c-`?wsR7e>RMnrw;RrmsYphKRP{c}%qQLp-s|49 z#FE+rE?GTQoU6{3gSHi{%LXRd7PA6sRkKg@Y?Ef)T1EqrDtQDmjx9Nc^-Yu-=)Yn{ zdyR||>N9A)DM_^(tFUrd+k&eo!xAGWNHJfkz>7Pzm+=HHL~WWa+lB~HOc!(ax!Sr0 zRev&tf^vVwtLrTOb`0Ja9vLsAJzts$cZ&Rbs*v*Na$K%HcxQf#_UA&Ovxhm{iTDUI zyfwqVc76AVuilh{Pm%JLdF3h8cWf|$1a$Fp)i>-x7344@&2zqtLd_6aZZU312(lPs zi#czC_IosF15j{qAq4_#Ej)EKEFF)fekXqyr{RE6x0ho|+z6&Bvv)%cZ)>3`TH>Gm zHL{cYb?o~WWI>nDvv4abejxJ!YX4*>0hf_@gFEb7Z_XW8?U1zjIosfl=P$TvDFi{{ z)0(7FXu-Rm?!_c8@mb@SU`LnZ zq^G_O(5*(X4u|ryxc?SvlU(eN@%Tho#SQ%_abGtebC+w#g0{=D`MTlUswR;kDK@97 zrKbT8LbPvt6vzCW+5%vO8t(XeLM($`sQ7v1E6d#Hp`8AN+$)?-j(oO}@yn{YvpPwn z29;^SwKbglDvo$g$9x3=?_#Sn+w_R&Mr<#S3pwxJx>k}Hmc86J zMW5oqe`jxL!n?6S!)qV5t22hQ$^Lo8Fp+0fkq*to!aG^E0{ff zW!5M~oil>L&^?WyPI_O~{YVr7WkCcb=8?9g#Cf;*gl9L22YYXTL#JGy6|Y;h>{g_w zc4gUZ=K<$L5SHUU$fq~{hZe-wv`fb+hP|`e;{80L&Mw~aj!O${uhYX7*$`V^=H%~n z2kpf?2K@qPSZ&F&{;3OSt91yjtwjN)khbv%bvq|!Mmx!x3ULg#PP6)*8}yT{u(I^M zJ457~vUML+7Z1LFyFb0N?|#Q1x&7-o@u5P6bD6!?CBM0|@j9>eolsFNsK>qw4TEzq zmav18Tc#Ru@ApKr7+$pAem>i=>31?_42T+LyC6=FIH;#uj-Q>rJzK4U9O0e}ezNi& zDso4C8-YSsJ*IKq_rFlTXR~9z>xVi#{7vU)97;zUFOT4*@xhoF;v#YJcT{Ls1t8^T zE$0G12%X_TT(0zhP2qGc;|dlkR4I}Zx?<)CP0t3sXU^(jfBe%XYbQlMCqXj#30j{< zI-&u()Z1f6b0&mR=YAKvyof!^%Mn_Kp1@_so2k@!w+LZ3n&Q&c1aV{(ztifGtgz}$ zn6>#{TMA)8z>V`l^q08%um+2ht|Ro{5z?zD#nY}ja$(T2gl{4JU~8dS`OD~0uNBGQ z{qusl=0#ni)Wb_wB{54-2fxEFpy$BnJ=-5ivpu3oUiJ$lcR6y-DR@9xDDPt$ z6~k|X$@MuTZws=MY+=y%RrP(Ky3Qw~ z!F?@$;&WM|;4wvcNl{&4E+X@^R3hln{*3U&pzfOmrH?Cq+Lu#s^VZe(orjI@0+Qop zf`5pqvhq^)UZF%+DznL{dP){zUY`ZlC1UfIYVZ}~UX$WQG0(FUONX+X!tAuE^f01B zA%|N*X8WBO#j^$@1tf8`zi$ng1KdqG!}7I;!XH6e*&r2PK}6BL}i)~>GZAfTtUn-vh`We(2Avi1N0J*>@KOx>&<9bADH?k=yb+^j%A*Vn$xz0K{w zo8BPs=J!8Zx!RlB*#YewO)XqmfEwUx_8^DXrXUM&BU86OE&q~f@8|&nOEq_Kbanl6 z|ZsFo?543Ou*MF7f;^+p>b!Pbk@(+vu%AUn5=XQSv3hw7sbwD#yaH4}b zSg~ec39sG#MUfMj1#4F~@R@1&-Li{tM0agCbWK;(^ffFx47XL^y8~g%^yW>f- z(+xort@fJH@c*Xx|C{3fx2E{trX<{;;S9qo;ROE6J54_!r_ceoW`-1OwWTOE+!zhm zsMT!eJg6SOMD^N)S`;m%poy{R>`wMlAT|ydn9FWxEb@f-D)ueA~-0ZL*DPIU|_{(ZmZU^LlvuaAL2ZFe;`d-SN=~)_v!%|+N zW#B9$wn_H_u+X8yJ0ggkw0^^JRh63~wxIF<=A0eZY?MZREa*)eEmob1s8o!_{Nb$% zK_9*BWg2aIQY$E?R3Ko7#tgYW6i#1!NcTeElVK^_h%1^V5@uA|%+MZUS%HnCFjV->8;Pm^7NEi+_=1OT z8DU>;t7BKH_@#Np%y{r-o92b61ux4MN%I?Hn>hv(``^sM$mDV#kL)09fyG6+d$A#BR zG83n$p|NSaQ`>13p`q09;gac)7WmMx z<*~GM@zSk{mc7aFqbYT>X|t;s*E(5=CpZn)c=NvtZvGMl_X2*|CGy+9cvD;rkc)@4 zIS2^$r{F|ua8w3|bdUdd|K$b)cLk9qa~*=FVFBXRtNxGh{?;T?p>ZZ5lyGSNnOcnq z*oOpYLLq`-uyX^!!OGnYe!J1F+Q_ zDA~*I1kcC9$F_y;giH)cjPLaN2agJi3NeDSZt^zW~8ak2f?2RI7a4fqW%0&^3}vq3C)aa3Ro}R0T@oG(1j7I(C}Vt?yHXAEGH`UAGPT@ zTphf3FPo98zg`~g_JY`H#RZj*g7Pwh5amfP^f6N)OCb9oBYb>WC3&qc0Q@6nTCTib5x@)b^!p$_)@d_ zQr}-j#q@KATsQ+I&x%pc$*rzknYB~YcZ%)H$(8l_DvdHW^$h;X@@p-dVU#v!edjIg zbyWGy2e!uYLLSFaGABY@nd|u}=^Xid8MD};;XSKW$jpdezq=_$ba^nej+H%7J5KQ` zkq}75g14Q&;I*Wao!Gt@=yyI-lZ%Ngc5=<0_b0i=l=|j;r#HR+if^NGaY6p&X%;JQ zq-}y8*OZHz6A|zC@sB;v}SmL+fON!2GTlqfd z+1^aY{aNc)-vHR|M+u_^&-OLVNlIRGKAG$!B|V$7*uOYKM4RBV@bI7f5K>d@OSD_< z02Umj;R=6+DQeO1xAjY;>yq7yLAh(7_)R&Vm>(`Y!$gjFhtH*khOQw6(Yb9GJJl_R z5kFV8BmLu@olhekqEeME)hKtpV}5m5?se~j6Qd%2zQcTs>KT=|;Joq1MwASVwc_bBP+w>`aba!53ZBKAI_|R-~9D zzfQSBg1Abs{#n$MyrR1GZAW28cZ^D;&sWB?BYCANl9gJqEr6xyHg7Gd$xTfZr3F4g zZn*tU#q>p0RH7xo8sDE70529<*%EVx%@K$E`^>DzaFW?qg-Z;20EPzniI1V-PVd@L z%Xw_3k>*-vz9ctR8z=i_UUsF-_L}2*@WkhMIX>ys>(&p$ZXAufsY-DfoH-?Ev?SjS zR_l^@Pq)Jx+epY5i+g!rTy&sO}UJ?l=SEr z0<29rab=+4i8gH|^G+=tk>vE{kw<2c9({i=sx75UVvj z;zrGo#XFTIC0E+7>EC2j2>M^2p%p2YS~zwkXGS3zA2ZY~<$oGFn}2{N+O4j|i+AhX zYwHz`LfYL}?~lP!F#B=Rb}+LcFGRF!bJi%;vg|p%(XV3gASdp2c7J(|zDw)Ay(ZXm z_b7mN3pV2kEFoXHF-)7hX4+L!qv#l^O*zT7R2KGDvFrS;ZAlm$VzrFq6w8J)JC$Sn zcRs-yjZ>g1e%{}uw=Qs@gPTLwfJ>T<8O_c9X_iTG(%_c z>TEe7Db^Trw0lwZ!o0OQO~+mw$(A6mgY>y zv$f58@6E4SoETQM|Kx)3Vk^lNT}J5xKd^iPer*2y)6E*G#6$woJ917AzaJwkUqjqz zE%%7E?+vCsn#;}_CQkXUWK_dn;Cu!a|9NZ@ZVn|qBiigv;pNt%Z(m}odRFchV)cA_ zeoC3C*?Jd$trb2emQwNkP-Zq&?yx*vEj!bmqDrk5MTktiZKO|V-NqK0kvk!1RWx;0 zB|hv>?}mdYZ1X06MSt5VNI4dN%OSN=W-dJzf5w5&;%|3A z7lovyjc*}>jFJKwsk3kaoui%_^yP!}DO5c1NK!w>bMX?UX~rpLdDu1|Dn7CNF*dE^ z)8%_x7S6T_B0ogiQriJng`%Q9i*5Ug1gIT=?A_`eW(?)F!cJeAmBf}@c9YZ7mR!9d zP>qD2&kVy_KZu&-)ofshx0_8ql>Io8pS*C5K#iwUowI13nW(WIZ3ZVGIW<&8I7_L< zT94-H33J$E$fQtj!ILoxET%8f6{gedsc zUnV&L*V?XA=9_92D$IRK#SRj1GYscF z)enmA-$-^!WqUKUaxoPbT$yWUpsOn^KdxX|rBby&I@1OBi6EW5RKrQr;CIfFj9fU1K6@Kd*x7;Fql&-Wn!0_;%VBe+=ZTsLbzg}lL|#cnLga=HJ`{$ z16{Qc01@nJbFAtQPA3AO$d?vz#hVeEBN9+ha8iIj$yD+J)f0gZDgn8=Yb6hGYybl` zK|V^|fUjdw-%JY7>H@`YXhL3$kpXKuR(iewjO78+%&+YoAJ7WDU-03HBxM_INL(RM zZoF3rU5RT8vp-V*=uqgnis8fi?h>13|FGdbc2oj48Jp>PH%b9EN+IA>~A&#Z2L!oI|*}A z(lLsOa5nSVSR2zQ^qfc=Xw4s@xptcsW+i74^DW?F@prtNA=@dZoeOipfOne2A%D0_-NfinK1E|Jcq=3;ciYMrn}8?8xftN!OD+x{jQhbV!qG*U-k zW)V998_xb^K}=fL$WOc9y$}b5Xc}^ znU1j63YBA&tNH(S>BXYL`miwytXe>~P0NLr+Zo7kK%xJ}K6cOmV6Z#jk;GNVg(&S= z=+2dw&$UjB*TaAVu})^K6RKT|R5@{fCC%n)*?@WKEw`xOr67~ z{*{!K;gx6bV`=3+R>X_Pm5=3F<0TaUQK{_pl1j4y-9>@3kIS z;UzQjl-GDA`NE7tMQVUZe!Fy-V>^siF^S*ZKS0J9AcwCfKY&2+ww|P%I07Wu1XB>GbSy7G0gow5NI@FLe;YMJ9e2VW}V@sY; zIG03kV>>cvwcjkHLY1Y# zN~U@yG9thRPJ|CiAF+ci)t+oW>Z*)bi8l^i>3e&9kf2VZFC9b)?2)bqPQ1l6XlSQ( zd%7}z`*SoUDDMXCBk)QcKZXrElzqm+qjReRr_i0{WI^V6Th+`U8irJ=_dVI}iTN$|iiDcx@}~(Ga^soA$mD(aE@FZ(U+bNQ$`T2G#Flw+=IHG8P6#J1C`)9% z7E#_>%)V}g8iu{b2u(tNBVv}{aHEw9MW z#yrWRCRLBkBif%W(OQmeY4=;Zap^x8Q=-;exy~0M=sS&VX7BQ}k<28x2XYCBq@Os1 zT-O)s2ymxX-BaN@DC$bvGATd`V@P7c5Ch*^XwG`GGY0og&y)Y2|`F7 z&l$jKW8jQIM_AC}pDdX#1#^G+TDKQL+d zT;2N!Y)!Ja(p`PWsY$Ah9?}5eW-4alClgTIWI~E80N%i6R&kHE8F6BNgUJ(R=evWj zs)-1@?{YM4gZegMq1Up--6WE&?pTQ9Ss0TZ{I8uz2MaIKwxdhY%cAHU zgq?GPV~LJ~7(DQ`2ebkni^L7?`k{UUNe7%b-XS<$!v9iI?|0R-pAHp-n=;ys4C8}_ zC|l3Y^fnZ3m+jgtz#<%o-O1TYb_QP3xy#z`v;kIvG-aS-9|@42@^jGm`+KVN zv3EjR3YuAaxZhTu&?VxoXQ1SW#n*BzE+T@$A5;W>sVfK^!-w92`@1Gg$A(y>y#$!NGFmC3R<4j>W<4c7x32cJvPm)pXPNbcpAW(k(?bn zUF|aB=gVK_U{BN4W#LuvQ8w1!e#QUoDW4D#KD>@aTS8~&?-vOgN3)81hmyTk7#TyDL%g_BA|Hu||`rI=fr;zyw6~>ItNS#wXG?+ocDs zP*UU^3^!_5h4b*c^dpKrPUI-w>&c7vR_z+i_|$&li6VuSIW9$rT(U2>A&TOy^erab zK;Yii4r-E|vE5;UL4C6&JssZPazgu{2b=NQ3vG?Pk>=icKRWFVc~&^uD{Z`cQh@Gc zv1V>py^T@Q)Ioejfz!IpYK(-^F2>qaxquu-Zd-BY3}_VHPP~H!{86L%V3^A zyhCR9qleafd?)fxpA`0em3h|xy?#+l?G|<^jqGF2rj+j&Bnr>$u0nSujbg`S_h7ru zC%kXqRsVQivPoZ8%uY9_QtH#O&TY@B?rgVCuXdf*KDJTu&zI1dAYK~>M6i6DG_%M8d${NqCS5ar|6izs-;2xabnto8}=DJ zZtspe)vegR>`|_g*~4mP@A{3B;cLsw1x2&guQnUM7W$~&hgs6>p2(KAhkE|2U4!?n zdREXjFzDoK(>BfJS7`B&mEZ5)217S_+oM{t%hFW?2`ey1OgDM$KYiN!E#dp!?MR(r zdrBAwpEGmFvm(V-*4t<`i6br3878(YLelYL`2+2js(tPT$}-&y&eMz+uBHk3Nx@%` zIwF*}zO!gu5o*WrSbgO9q)&vTfRbNw36JL-2QPz>8++>WAKez(2eaoA)6mVM~us|NmvNwMz74tgKm)b8;)fAs%0 zuwlK)0~_MoV;2YIwjK@MxuK4?)Kjd0n{+QD)2396Ti{Xz%p##r9&~re!x< zw+;KOZ7q;?AvSlJ)%&X@cluY|9r92;ZvDu>W<7>nvi<3(;G+stXDdJLn$PiBg1niU z6NWF6cf}PQ=?Us3^6>3*vO3CT`Pj7Lg!FyKE@Kx<(c-U?<1TM3)(=MBGkJil5;|3E z)Wx$M$LB@FipGCT*xvSzxyHKWY`v8L*Oxl;0};J?wOvQtH&?G-tg7l`>qz(&2kHIko7`A&qWFU zl^l5r#|U!7UR@6>RR@cye6z}tD3t!;{N=`N^^Lg(a!#sM*CQ`+wQC;nVS;PP=-<+) zJ~;WV{!0c=;diYj--$o)>MtFX4+drV76)ES=6@P?rZ-+wcq0dU0a}#QxJ&e;p-N{) z=MTA2_a?=|Jk_emiLlVt+f~|K^w$!J({26)duzW>ciWl%n0&d@MdzYTmDm_Z{Vw$# z^_H;-Ol=s`W6!sL3ATWlWIM;Rd#jZ5jHGpl8=Di)wSO!rO_ZQ3NG$Vm-BFJ$kJ*;k zd|Bcy(bs6}E-R+hY;NL7`RbB|vc;M=zC966xxqGG?eMT=)8XLU%i0pPy)9nCn(gjp zQw6bJX6zYujg{l^HjzfI-`%u3KHucc@{aV9`Y=7gO8*qOD~m1a~Uj?topCpJnxxYZEy85uF~H>_?Wq} zlgi}hP_RNr{p}$86D;o_cg&B36OdPF`BNfIeLAmzJt@8HUnLLk6N)kj{sSa{B z3Mgcm4n{o8iRWfcbIn)H$o@L<;Vpqycx`GdFiXGm*iCH;|$GEaT znnP5{%zv!=JD0&`xyBf?*z=jFH^QNZ@2A@sTDU}8OT0e&xFt?LF7f$mz2jS5%y{jk z3%Jlo6^4b^~ABOHxsfexMBGMWAv92(_DvkduHB| z!6+Jy-gT4SxlT7zCy_^1LMXN<(`mfa_uXVW!od_;QiftA`W7lckXb!aMUQ&7|91F1e4f@e75nRM=Igo#f2 z9%9nBNhXHfsZl%I+`p-P6nj$8+b3%>V{i@U>Q%6JY6wiD;t^eotICeWNS15Ur(Xoz z?t@9oNZww_7xs2ppJtQ%qm=KY2)CgeCiSFqZMwtBv1L-D{2~vA{3XV8D;|7>%b3EJ z$)wiSc5Er+(c`Ja)+>J~H$z-YQb8#xe>T;T$G=+Z|LOg;n_V@{MWOHB>1Zt%eYO3) z$0nzI8=p`&Jej0 z@!QrZ?{;{58(TPkz1+z6?Soa@TKa_st!w&czP1R~t6)stvnXH`gP&}$)G?5?apJpn z!^d3|-$oHvdwP1skdlaF->smJddqaV_PXn3WD<5{RJ|_UeW$nH<`o9Ul9+T!_i+ES z(y7qXq%FO?xrMF*bpiUy$3z}~sQR#v#UECEsN~IQV|!^ON-N0EYCXKg-ADUuh_3L! zt>aqn2X1pI#2R_ND z>vOj5zDpE5xpFGgi}@|jn=(v!!dsRqHV$#y3@ldX1FXH2b8mNLBP}nU#x(lFwqVyY zM@~ogYh88{4_pc-nSFdm9+e&??(N1(@2Ij~ap?*(bAEE2v-d^Q@qnzc&9&pXkLiwe zb`6)jG$Nq4-AnAWnav6?OpF)agvbyw1|ZI6@n9q>*nE>hqS3$$X?RF59qc8+z@W|G zoM=%YBpiT80k{_?JPhQ)M!_*~e@r-;7B!nXEIcFx#AAaop?CecK@{AS?a8BR_E-si#1z0cu=2C%M5a1Ym1PP+zgbBq2z(FDgObSk7pvnO33UCJ! zQKR65&`>}M7mNq^Dm*C+NE8LK(E|Vqj)9h9$l(vRH46tBf^oR;&`<){HwHpe&6EuY z1DIDRg`ozhR7TNMfRh8LITC%!1{f}&w&0mKNO2HO3}C^)+i0M^ zFr_&iV?x0OG(?aV1E&#i0aOx>1h9QTHWGy93Jne?2Eb7y0syKFJaDWp7ArywPfQA8(5EB{&L;#w?F~~ru zxp9D+gK!UXYPZDzNI1ad0lkw6fZ-V&8!#3KzDEY)GK!whTmV#w5NW7VAo&1}4+6?T zpuB}@oQ1AI^#EZB!$95Q!^ul4DT0LgPbmlzbylc(+=hfxLLj60DfoXATCR1N`4?J_ z1E3Ou$>ESK&miUM@>m4l#WTr%c)%=s{}}_6@J}oJrxpIMZiP#tZ?>xlGyy=L`!DoT zBF8x%*!?_XF+b}*w!z8=o@r;riz2ceieO(Ko~L{fsfMLL>_59iUJrzj>hY4Fbyrgj0D0emcsG>iDjp01Yn2Zjv zm&OE~?fm0eFF!5iqhsmIFM^YIxKc}#JD%~Dojdw|d^mGjq@kv=;tIbvE+uu1O>w~{ zTX*|Cd%huEqTjO|{J3_1H|M^1moHeB(>I%cW9I^l-uUH2T|qxe9;d6Bo1Z@$O;=UG9CklDdUwSDR&31|mI!T)&89 z#at0ni=sR4ap|l%6m(odj8{=u*&~G!0uH<>%NG zceb;@sY$EYOpK5n9tCI00!ShL+!621`KI)!vq}+ z;s9<9A=UwKmkN;ODniKrC_gmBF<{zZ1Tf^z8%7x;B82sx1Gz204RV=EK)AsxST)-x z1`g7;Iai(U?;v;%letD|o}+Po3;B`|D za61A4KZnEaQ5QVQC41%zvuyV}W(YC(KS2jeHt)J#CV?{T-@$d>+Gc?;g+ymTGoGw$ znt{Q`$(Zd=w0uoc%5LJ$#7FS#GiFVVKXziLg%r9xB?nKjKo&}~<`JC^6gpHTEAk{{ zQYt_C9+c=keW9X)?J=7jlM|C8)18KZci5zB*Q-bBYx+xXeRv`=5+Q&4Q2%452X#ku zzc0^;R3A4{(9r(`C*~eeZVR>fc3(_S$mLU0dC%Q2&dS$Ql~$LNJ?(}RJ2cBCTGoUc zT#{Utj*so`%)9XXM6O8+#}UKVg*9gBjxFN1&QUbaR1Q^eDIQbt6HTaL(_!LuHMEI=-G+%+1#k9L(R%)@5@` z$L1f2{h~!vlNy~~bFO*wK#FxkuKh%5?IRK60=U4#G8b!C$z-jB*Oq-Y1$8iH_BY!; zzj@*o&%`e$;S;v=>8R3?wW|iRD!z_79QWgW+Q0nL+YRGP+br%HUOq0a?P`=Ne0e#d zr+;h5#zcqcrw6b4RH}p@zgRvZ7M)EyR^wu{Ts>wj(|%TZv2=Ngy&ba^88;{_EUe6& zb}w&uvz%c?rnJ;@o~);LQ#!hLcD~f=RZNRk2^A4m&k;oFl5Q4G_-oae1SjRMfC*>v z%jj}{l3h)gzrA($YF3A#K5gG`fhV>jm!%1vw8L#v`Ly5p*iN1Ua_jIU4ugr)F!{G< zJYBZli)!${w<}mXhdKZ&*UIA%RG(C9cph0a*Gky`EaoTyV}dgHDiQQIeGVu1Y?{ZiB#&hr><60xKq}VC!{C)@wt{` z_8N&!OI+Ezh_uG)`brC?!;0HNPT8JPIe67LWiKl246Ca2q}>Uv&mQS2-0$|Onjv@D zXxZfq$|tl$Ok5MMRUmYC7m9jakS!O?w5+MLT!+Zt(0nRQzGFX{d6%Wwbv2YJx6GyW z*FPanv~{ne+ZgglY~e*tQDXMee5P>ND{LZon{Tel54=U!tIF@QGiqcn)jH)^@kwMC zSq*ddtJykb>9pR`hf}5Zhfki@Moxyk$~wEtf~qGxO6;iR(Giz@o1XU~x|_M|?T~wz zyd#=exz~M&i9JPO^*LNn!#A)6-?K#CSiZ@;mzX_ydTwdOHQl4w+K!DGiG%mMQ}k_{ zKbw+yc-03CGn9`AH

LTP=u`?lWA+C#+M5QXCb_NB2J+^*KpFNIBFcO&&ZKc(1gu zN&Hr&oU(pSWQVYN@ zUP|n^>*0;xs>hO7{7B(j>-9}~__Gv$LS+qim^Cwp(?bzGjK(9Mv+DaD;3+-lZ`DN< z>juq0ut=n6~JK?{&D&4 z*kp1w&H7S-!D`VFnQCNR>1VH_Yl@SQG94-~nt3QQ(QbOxp)173IJDIJ)!!`(8YM>6 zifL=ocvxekI9Y^suj!~XS8eDH9#-zUZ&CO@O6Z}GMo*8{p1zO#LCcfV+P;LZ3{txy zZsGNrEA-}Sg9ud1!JAcy;luJg$GqERR|=JW_6Tko`;TDVS=*(yCzgz_>c zX<3J`x61}fF(!O1UMl&Htv<|xKRx|{Bafq$TGX-aOz!ltbi_!e%Z2+D_wko+>uOBv zd39ZL^XnqK;z7GSGCa_K(SId#mwD(R;GIHn6$2 zR|;Mrko$3mIE#5VjXs-hGEGasA5mhrP7}B%cc52rzu%W!Qa9a5D0}+Z`Yl3-V>Cph zTeh}stnuTX$k`e#Mc8b(YVwtM{BW3URlzOGCm0@!7bCSX(!4_?k&{Pkc8@n*yCPpu zs+)azR{@dT`N+dBQ@3|-5ztOCnras+kv*6>jAFM<_#1m$ub~I63GHknXG-6= zUq4<{*uQnH)V@HThZ_TAHkU1zAj|X9N_` zzTri>2iIMSdsrf6UeJ;;TI_t4$d2e3@sLR;KiP3Ub8? zS`Xa4L?SeA#cgK~I*s8ilDYIo!U*xHU2)F`e%0_KT6SDd z9jtxlxtgsXtm{Tm3x1_#&FHc}?u= z4ck*hth*a(`}Q7s#iYV((vLqe_&y=SNqqZ>O*n(A5x(n_-Zk7kN6v~#GRk;xpg0h{ zTdGA^TK2d?3_WQ}yimd3oI|n4wob-u-uUe%ilZ}r;t;dWm#T^@O@@-^f`v8%frTUrb?)DuXd6E}Z z;fV6PyAfNe;@>p0ST2hsT<^PSi)*ss*eXZB32kt7=aQEli9Gi-sZGfyFz>{b2vUP3 z9Q(W_YOgEqjApn)CF>ysTcHZQ)%?m0a@v1vdt#IB^DI9sct{~y*q~eppCf)~ z#Y#l=O52Lw4s`9;spDG?sdAk441(vl$MuG?vWs4kt55DPcuy+r6(?QETrKh>dhH~? zP}aIuRF;c!i0ud822$WT#S10j>D+fpjXD}Mjk-64*kzE8ZS(i1zv@=cA}DmTR`aa7 z$=keUd#YWtuH4H{x)kS{A43n02CSLxjg^>qkCxt6mv><&c?7H8Q6Al(g1a7xSwDI9 z7X5VNlZffCrX3A>)+wuD?VKsS>)wnVxR<-*hI6jYsgyMrP@dT&;$+)nrE8seS^I-T zl@v)+0u`l9J1*WAlqh7EJkK(gAuOy=kzYdpEQA=3_Gw|)Y{kp9x25yb>{`kQjzYIm zv-(HiN0jR|$6WVu7AA6V3sb{(#CU~stkb`%4defC0PA<-*s>9Iy?Xu~Dz~gMvr;8< zVT$JqXgbZLyf@aLG<9IQy!fYq*Y6^h_33h*swM`tb?&uf`<7THs8hMG1LoA^Z}?sQ z8l|Z}T(%3|czVa{@}srt8;7i{3P_SQ-(c*|`l%KB#y4Fkelozsk{!6x!KVeARbs1o z8)N!ed;GP=SfFRYstm`C=fgL)D9XPMP2+lewEq+5&bJ>|tSlSqMV|Sl#tu^{)z#4- zdM0*w3)MeP>rkTpH=m>E*W-p+`o%esfvCCCBA!JxA`z9=F*un>0HX0&Lo8Z zw|w-CHHK-on+{y&PH$`PG}f!W<+lp2yyGsyRGB6bQ{IDbM<6`*?kcaGDq(g(i3V&XMsiU4w?;oi>?_+ZZwIV3GMw3WwBBSL z`@zyw^c;8hW_|x!>`AXU+X6|)llm{$n^N}b{fLh~8L>CHvM1xlF_Vi1sIztEtx@ZB zb@5l+tS%~FR4Q1zp7Tjv|5hGbxiHJ*^SUB&oU2N0^ah9@lAm{$aUIa{mvd2XDwdAm z{D!jrq+}^+k+CIT=KQwJ`8!lM2p-r{dZ1Be9bNI!V`cR{D?Q39Li;rzc%OEBgDQV< z$@I(wv)hVB_H6h2%dJD9w zogAeCw}fLtz`8cz=%&s1&iyfAVBPcVN-Y&hxKnU=C~&JgMFC_K4-1;}huZ`b;Tmec!Y!$gI~&LoOcfgaC7Jqs$t=P0Z?IZE0IZ)TP#HdU#&UGP zZ(ukzNCIw2W2_Iy62Mym44e{71&i!Kin-Svz?Do5h67JNBNgy?M?$J0(InvJV2L*^ z0F(p1K+x>tKYp=Lf9li!4hl_SCY-W+t*g?uCEMiD4`ZGx&eioR)OYzy$bgKo2r^J|r9lHqZz6JOYyQ7k4H? z+v+n`ak3*|v!}oTPgVO3U`!lyM}E10mr3AFnLb=ybH!u=qDJWntY1h^$Z87rwq8uKCRfw!g~IuccJVSU+VbeoRWQ zBrTEr+ut z7b(@_Z)95wBK}D0&mU+$nqDpX5S3f~Vv0f`3y*4#OYZSIu*BFQ#11JB@M*-j@&l?f z5(I}EEDi-BfBoSG@4Dxi)QSL1h%Y$DBXoA1G&tIzlwG!~i$%7pjRhQTu$ZSm;dCj> z6aLDh4HII61~UpqKny0pCBwYnSV$R|&00W%5yLpHa4vuqB%Oc6-E3yF`A2EZPy5?} zO*4rX9T@Uk`uWE?%%oqi9mia1i0MCBTRuO@?~A7A$IN?iIX`%j-+6xI;&EdxmIG4E zyp8}S0EPUjM;JI~Q+TcL*H!`iJOJw{ey{{0s8$1n7Ac6g8z@82eSWo zIsnk+f2ZSY3CsSsT@|u_*ORGWbNLGJlqAb8xT}O^H4J!7gxEp=(iIaXRKy!2NuuHj zk-j*9HKkC)B=zB5F_J+b3|vVvP{)!=a7oAy0|KbwWZ=n_45NY5ropWm<2pD5T&3(u#x9`!Q@;NTu@@fPx|sMb-9@Sg2)Aw{lV7FOLo5VBK}qP`_tFGnDB2= ze*sy>fDiWZrUi#G#$htF!+2;EGz^DCfz?aFbclxtXtU{HDio@&qoS^&CJ9DF$yxIA zA~jUNKgl?l5ElZCws+H?@u&-ZhO0s!b%ZSffp7$W(8n5q&_W=I2owTAM3@j6VMyd} z@pLHA>}R|k!gA&wd^1X6#Dg$80^|d~P$4G4Cg2;~BM??$APl*|l0pvmG;@w=;JlxO$cylvICZG@3a1RZ= zARs%HAv@XC1x-@ZvZSj2&igMt{p8Cs$W9&TA_D8?8f9rVqbG<5PYRigHAnmZl+DD- z6z^`QhSg9f<48z*hFp+t|9W{Cb>t6p4(f9rpOCyW`dNZEGY@z4OkMwN{LI%I*=)wn zJVQL)B7@vD=IjkYa@2H<#+&QWJl!pcPA)U`hcib3`$a{&YofJrKqg>=Bv5y1c7e0? zySV;CuvAA4I}=ODwm@0v0FYnJe-k;cJPZX+D$zG<3=Ysx}k$`6xV266eI5< zyR-~)rFpoK!z|5EL7o->BCcu6u(73h#Jg#SKz0Uf65<2YlRBN?I?s+NdL#_RH4y(_ z;M2uCAOcHqAvyz@Jt)8i8SQnk9+ynFR9%C<0dMM|HrwvivF=FiKh^W^?PzJcBijXI z+>vxf+ghNbCG5}{V@{^y%t8MYg+zLqqoX|C?V=gHEXC`;*Y20*H%KoE^o>wkR>!)5 z`Xv2Xy)T)L?g9Ew4|luyeb0Q`TcXbA>sy_SH%Ec|=zyPno`qVNrNz8H(!|ONr)j6| z>4bFhbVo+usA#h1jJ+&jo8)MmrYqSJ1>`W-i^7|`2K}wNB-@d3R8L^@E<_iKD;;V_ zo}irq`9koNKkbX0J>9J|Kz}~NH`JyyT*HB0)G+Rze{UZ}hT1bS0!xK>L5DzN;GgvE zpf2p_SPkuHP%kbRXw3RM`*w&=4ZSct&?{C$>tFE*jrVF8H@)z`mA@d*2&{Qz1U^b# z9cO~H_jGd&@^qW&tC#2l%&{7gVOUKZ4eyR1I#XQ9pgsP*cKh3y?n(A^Lz6t*>_Qnj zqb%-Q=Z{fn2ESGT9vWeP>p0-*rp*}ZJxzm&w$3wS&=R_J#WKb?*B~%vM%vDdX-oB4 z6s);kz`q&?JSmY89`66OF%@HuqPuI_1!6U=X6KcQb-5_bQvE)~GYqSN`d8!q0{)lk zSJm-QTlN6Ffc_`~^z%B5e%72!!ckCB1ZZwxVnrvop`vgUJ|E?jtqvT?KaT8gCm%>n`l9E`v7R#Z$^Zs@h;H4Hk8vEn(rgkpi6t`Yvv(9 zP1Rpj72F^Zk&Hkb^v#(6Gj0)9(9e85L-)}U;CTcpj))9ekUt2DpO@bTx?f;#P>?zF zyaAqHz%S!tg0PC#Q&ojzG(!RIfhdfJ6vibZ1roYM&Yu^XLw$n>1r>;ac(F1C`Bfd~ z_(7r>xfb%nC=^Z63j;Q*Y2)rn2V*zXX33t6O8TlMt80jq-ag0mG72|me z8Yh4Cqd@-Jb;@ z4~Eju)2ADB@1o%vg{P1MXUBDD-b2Pxp*cHQ11if6;$J%uls8|WP)^7&<~~~B0=am8 z076Mne%+t>O=jwYk#AmphCMmp!1EdOi~{DTBrxx0B(imwwGZg=6#C5F&+3z5R5SL+ zh-MUuT5|5kh}VPoiFO8sGUWMJ{5WEOUV&bIrcF0NMAS{|{4>hrt{WX!ZXC`_i4Q diff --git a/examples/notebooks/hap/output/hap_complete_output.csv b/examples/notebooks/hap/output/hap_complete_output.csv deleted file mode 100644 index 5727aa09d..000000000 --- a/examples/notebooks/hap/output/hap_complete_output.csv +++ /dev/null @@ -1,66 +0,0 @@ -Customer Name,Date,Customer ID,Email Address,Phone Number,Shipping Address,Order Number,Issue Reported,Service Representative,Response Time,Resolution,Follow-up Needed,Customer Feedback,Unnamed: 13,hap_score -Ethan Johnson,11/7/24,782011,ethan.johnson@email.com,-4109,"134 Oak Street, Portland, OR 97204",4578342890," Ethan contacted customer service to report a recurring problem with his washing machine, Model EcoWash 300. He mentioned that during the spin cycle, the machine produces a loud, irregular banging noise. Ethan initially thought the noise might be related to something inside the drum, so he checked the drum for foreign objects like coins or small clothing items. He confirmed that no objects were present. However, the noise continues to persist even with minimal loads. He is concerned that the issue could be related to the suspension system or drum assembly, potentially causing damage to the internal components if not addressed soon. Ethan requested a technician visit to prevent further damage and to get a professional diagnosis. ",Jane Cooper,15 minutes," Jane took the time to walk Ethan through a detailed troubleshooting process, including verifying the load size and checking the balance of the washing machine. She explained that unbalanced loads can sometimes cause unusual noise during the spin cycle. However, since the issue persisted even with small loads, she advised him to check the drum suspension system and ensure that it was properly secured. Jane further explained that if the drum or suspension system had become misaligned or the shock absorbers were worn out, this could cause the banging noise. Realizing the issue might require more complex intervention, Jane scheduled a technician visit to perform a more thorough inspection of the internal components, including the suspension springs and drum motor. ","Yes, technician visit scheduled for drum suspension and motor"," Rating: 4 Comments: ""Service was prompt, but I'm worried it might be a bigger issue with the drum. Hope it gets fixed soon."" Source System: CRM --- ### Customer Service Log 2 Customer Information",,0.00019484490621834993 -Sophia Wilson,11/8/24,785673,sophia.wilson@email.com,-5331,"1025 Pine Avenue, San Francisco, CA 94107",4678452100," Sophia reported an issue with her SmartWash 500, which has been malfunctioning during its washing cycle. Specifically, the machine displays the error code ""E4"" after completing the cycle, which indicates a drainage problem. Despite attempting to troubleshoot on her own, including checking the drain filter and ensuring the drain hose was clear of obstructions, the issue persists. Sophia mentioned that there is also water left in the drum after the cycle finishes. She is concerned that it might be an issue with the pump motor or a deeper internal blockage. ",Peter Foster,30 minutes," Peter explained that the E4 error code typically indicates a drainage issue, often due to a clogged or blocked drain hose or filter. After walking Sophia through the steps to clean the filter and ensure no external blockages were present, he confirmed that the issue was still unresolved. He hypothesized that the issue could be related to the pump motor, which may not be functioning properly to expel the water. Peter emphasized that if the motor had failed, the water would not be drained ebiciently, leading to standing water in the drum. He arranged for a technician to inspect the internal pump motor and check for any blockages or malfunctioning parts that might be abecting the drainage. Peter assured Sophia that the technician would be able to further diagnose the problem and ober a solution. ","Yes, technician visit scheduled to inspect the pump motor and"," Rating: 5 Comments: ""Great help from Peter! Hope the technician resolves the issue."" Source System: CRM --- ### Customer Service Log 3 Customer Information",,0.00015312987670768052 -Liam Smith,11/6/24,781924,liam.smith@email.com,-6553,"432 Birch Road, Los Angeles, CA 90001",4589231843," Liam’s QuickWash 700 washing machine has been experiencing an issue where it intermittently stops mid-cycle and does not complete the wash. He reported that the cycle halts after about 15 minutes, and he is forced to restart the machine manually, only for it to happen again. Liam checked the load balance and even tried running the machine with smaller loads, but the problem persists. He suspects that the issue might be related to the control board or an internal sensor malfunction that is causing the cycle to terminate prematurely. ",Emily Davis,20 minutes," Emily took the time to explain to Liam that one possible cause for a washing machine to stop mid-cycle is a faulty control board or a malfunctioning sensor that fails to communicate the proper cycle status. She walked Liam through a series of checks to verify whether the machine was properly loaded or if any specific wash settings might be triggering the premature shutdown. After ruling out the simpler causes, Emily acknowledged that the problem might lie within the machine’s electronic control system. She advised Liam to keep track of the error codes, if any, that appeared on the display. Emily escalated the issue for a more in-depth inspection of the control board and its connections, as well as a sensor diagnostic. She arranged for a technician to visit and diagnose the problem further. ","Yes, technician visit scheduled to inspect the control board and"," Rating: 3 Comments: ""The service was quick, but I’m still unsure if the issue will be fixed. It keeps happening."" Source System: CRM --- ### Customer Service Log 4 Customer Information",,0.00016854520072229207 -Isabella Martinez,11/9/24,783210,isabella.martinez@email.com,-7775,"785 Oak Drive, Chicago, IL 60601",4599872301," Isabella reached out regarding her SmartWash 200 washing machine, which is failing to spin correctly. After completing the washing cycle, her clothes are still wet and heavy. Isabella mentioned that the drum doesn’t seem to be spinning at all, even when she tries to set the machine to a spin cycle manually. She noted that the drainage seems fine, and there are no unusual noises during the cycle, which led her to believe the problem could be with the spin motor or drive belt. ",Peter Foster,40 minutes," Peter first confirmed that Isabella had checked for any obvious external blockages in the drain and the drum, and all appeared fine. He then walked her through testing the spin cycle to see if the machine would respond when set to high spin speed. After determining that the drum was indeed not turning, Peter suspected that the issue might lie with either the spin motor or the drive belt. He explained that if the motor was malfunctioning, it would prevent the drum from spinning at all, even though the rest of the machine seemed to be operating normally. Given the nature of the issue, Peter scheduled a technician visit to further investigate the motor and drive belt system. ","Yes, technician visit scheduled to inspect the spin motor and drive"," Rating: 5 Comments: ""Excellent service and advice. Hopefully the technician can fix it soon."" Source System: CRM --- ### Customer Service Log 5 Customer Information",,0.00015754287596791983 -Noah White,11/10/24,780512,noah.white@email.com,-10219,"912 Willow Lane, Miami, FL 33101",4602741920," Noah’s SuperClean 800 washing machine has been experiencing intermittent water leakage around the door seal during the wash cycle. He observed that after each wash, water begins to pool around the base of the washing machine. Despite checking the door seal for any visible cracks or damage, he has not been able to identify the source of the leak. He also confirmed that the water inlet valve is functioning correctly. Noah expressed concern that the issue might be due to a defective seal or an issue with the drum assembly causing the leakage . ",Laura Simmons,10 minutes," Laura advised Noah to carefully inspect the door seal and ensure that no detergent or fabric softener had built up in the rubber, as that could cause poor sealing and leakage. After confirming that the seal appeared clean and in good condition, Laura suspected that the issue could be related to improper drum alignment or even a small crack in the seal that was dibicult to detect. She scheduled a technician visit to replace the seal and inspect the drum alignment to prevent future leaks. ","Yes, technician visit scheduled for door seal replacement and drum"," Rating: 2 Comments: ""I’m really frustrated. This issue has been ongoing for weeks and still isn’t resolved."" Source System: CRM --- These logs now include more detailed issues and resolutions, which are useful for analysis and decision-making. Sure! Here are 5 more customer service logs with detailed issues and resolutions, including a variety of feedback: --- ### Customer Service Log 6 Customer Information",,0.0008747439715079963 -Mia Green,11/10/24,787654,mia.green@email.com,-2664,"212 Cedar Way, Austin, TX 78701",4698765432," Mia reported that her EcoWash 1000 washing machine has started to vibrate excessively during the spin cycle. She mentioned that it was noticeable even with smaller loads of laundry, and the vibration causes the machine to move slightly across the floor. Mia checked that the machine was level, and the floor seemed solid, so she suspected that the issue could be related to the shock absorbers or suspension system, which might not be absorbing the vibrations properly. ",Kevin Mitchell,25 minutes," Kevin walked Mia through some troubleshooting steps to check if the machine was perfectly level and if the laundry was balanced. After confirming that the machine was indeed level and the load was balanced, Kevin suspected that the issue could be with the shock absorbers or the suspension springs, which could have worn out over time and were no longer absorbing the vibrations. He scheduled a technician visit to inspect and replace the faulty components. Kevin also recommended that Mia avoid overloading the machine until the issue is resolved, as that could further stress the system. ","Yes, technician visit scheduled to inspect and replace shock"," Rating: 4 Comments: ""The service was helpful, but I’m hoping the issue will be fully resolved with the new parts. It’s been a bit of a nuisance."" Source System: CRM --- ### Customer Service Log 7 Customer Information",,0.015778325498104095 -Daniel Thomas,11/11/24,788901,daniel.thomas@email.com,-9219,"520 Maple Road, Denver, CO 80205",4692345678," Daniel's QuickWash 800 washing machine has started to stop halfway through its wash cycle. He noted that it happens frequently, regardless of the type of load or wash cycle selected. The washing machine will start the cycle normally, but then it freezes and the timer stops progressing. Daniel confirmed that the power supply is stable and the machine is correctly plugged in. He suspected that the control board or timer could be malfunctioning, causing the machine to halt unexpectedly. ",Sarah Lewis,15 minutes," Sarah explained that the issue could indeed be related to a malfunctioning control board or timer, as these are the primary components that regulate the wash cycle. Sarah walked Daniel through a few basic steps, such as turning the machine ob for a few minutes and resetting it, but the problem persisted. She recommended that he check for any error codes or flashing lights on the display that might provide more insight into the issue. Sarah escalated the issue for further inspection and arranged for a technician to diagnose the problem with the control board and timer, as they would need to be thoroughly tested and potentially replaced. ","Yes, technician visit scheduled to inspect the control board and"," Rating: 3 Comments: ""The support was helpful, but I’m frustrated the machine keeps stopping. I’ve been waiting for a technician for a while now."" Source System: CRM --- ### Customer Service Log 8 Customer Information",,0.00022087343677412719 -Charlotte Davis,11/12/24,786502,charlotte.davis@email.com,-11330,"1500 Elm Street, New York, NY 10001",4701293847," Charlotte reported that her SmartWash 200 washing machine is experiencing issues with water leakage during the rinse cycle. She noticed water was dripping from the bottom of the door seal while the cycle was running. Charlotte has inspected the door seal and found no visible tears or cracks, but still, the machine continues to leak water. She is concerned that the issue may lie in the water inlet valve or a possible internal gasket problem. ",James Walker,20 minutes," James acknowledged that water leakage during the rinse cycle is often due to a broken door seal or an issue with the water inlet valve. He walked Charlotte through checking for any detergent residue or buildup that might cause improper sealing around the door, but Charlotte confirmed that the seal appeared clean. James explained that sometimes the gasket or internal seals can wear out or become misaligned, causing water to leak out. He recommended a technician visit to inspect both the water inlet valve and the internal seals for potential issues. James also advised Charlotte to check that the door was properly closed before starting the cycle to avoid further leaks. ","Yes, technician visit scheduled to inspect the water inlet valve and"," Rating: 5 Comments: ""The service was very quick and the representative explained everything clearly. I hope the technician can fix the leak soon."" Source System: CRM --- ### Customer Service Log 9 Customer Information",,0.00017898922669701278 -Benjamin Harris,11/10/24,780893,benjamin.harris@email.com,-5220,"934 Maple Ave, Miami, FL 33132",4691273856," Benjamin’s EcoWash 500 washing machine has been displaying an ""E3"" error code, indicating a sensor issue. The error typically appears mid-cycle, and the washing machine stops working. Benjamin attempted a reset of the machine, but the issue continued. He mentioned that the machine would run the cycle for a while, but then it would halt and display the E3 error code without completing the wash. Benjamin believes the problem may be related to a faulty water level sensor or another internal sensor. ",Olivia Taylor,10 minutes," Olivia explained that the E3 error code often indicates a sensor malfunction, especially related to the water level sensor or the pressure switch, which measures the water levels in the drum during a cycle. Olivia walked Benjamin through a series of troubleshooting steps, such as checking the water supply and ensuring the water pressure was subicient. However, since the error persisted, Olivia arranged for a technician visit to thoroughly inspect and test the sensors and wiring to identify the cause of the issue. She also noted that if a faulty sensor was found, it would need to be replaced to restore proper function. ","Yes, technician visit scheduled to inspect and replace faulty"," Rating: 5 Comments: ""The support was excellent. I’m confident the technician will sort out the issue once and for all."" Source System: CRM --- ### Customer Service Log 10 Customer Information",,0.00023553140636067837 -Grace Clark,11/11/24,789234,grace.clark@email.com,-3998,"600 Oakwood Blvd, San Diego, CA 92101",4697682012," Grace’s QuickWash 900 washing machine has been giving her problems with uneven washes. She reported that no matter what cycle she uses, the clothes come out either soaking wet or still have detergent residue on them. Grace tried adjusting the load size and used the proper detergent, but the problem persisted. She suspects that the issue might be related to an imbalance in the drum or a problem with the dispensing system that isn’t releasing the detergent properly. ",Emma Harris,30 minutes," Emma explained that an uneven wash result could be caused by several factors, including a misaligned drum, water supply issues, or even a clogged detergent dispenser. Emma walked Grace through checking the detergent compartment to make sure it was not clogged or blocked by detergent buildup. She also suggested that Grace check if the load was unbalanced, as this can sometimes cause uneven washing results. Since the issue was more complex, Emma scheduled a technician visit to inspect the drum alignment and the detergent dispensing system for blockages or malfunctions. ","Yes, technician visit scheduled to inspect drum alignment and"," Rating: 2 Comments: ""I’m frustrated with this issue. I’ve tried everything, and the machine still doesn’t wash properly. I hope the technician can figure it out."" Source System: CRM --- ### Customer Service Log 11 Customer Information",,0.0004476681933738291 -Aiden Walker,11/13/24,790321,aiden.walker@email.com,-7664,"2324 Ashwood Drive, Dallas, TX 75201",4703409821," Aiden contacted customer service about his UltraSpin 1200 washing machine, which is not heating the water properly during the wash cycle. He noticed that even after setting the machine to a hot wash, the clothes were still cold and soapy when the cycle finished. Aiden confirmed that the water inlet valve was functioning and that the temperature setting on the machine was correct. He suspects that the issue might be with the heating element or a faulty temperature sensor. ",Michelle Adams,18 minutes," Michelle explained that the issue could be caused by a broken heating element or a malfunctioning temperature sensor, both of which would prevent the water from reaching the correct temperature. She walked Aiden through verifying the settings and ensuring that the water supply was properly connected. Since the issue persisted, Michelle recommended that the heating element be inspected, as it might have failed. She scheduled a technician visit to test the element and the temperature sensor and replace any faulty parts. ","Yes, technician visit scheduled to inspect and replace the heating"," Rating: 4 Comments: ""Good support, but I'm hoping the technician can get the heating issue fixed quickly. The laundry’s been piling up!"" Source System: CRM --- ### Customer Service Log 12 Customer Information",,0.00014801646466366947 -Olivia Thompson,11/14/24,791014,olivia.thompson@email.com,-10219,"1030 Riverbend Street, Orlando, FL 32801",4704891342," Olivia reported that her WashPro 700 washing machine is exhibiting strange odors after completing wash cycles. She mentioned that the odor is sour and unpleasant, even though she has regularly cleaned the drum and used the proper detergent. Olivia is worried that the odor might be caused by mold or mildew buildup, but she hasn’t found any visible signs of it. She is concerned that the smell is coming from the drain pump or an issue with the drum gasket, as these are areas that could trap water and promote mold growth. ",Jacob Carter,25 minutes," Jacob explained that mold and mildew odors are often caused by water remaining stagnant in hidden areas, such as the drain pump, drain hose, or drum gasket, which can trap moisture and detergent residue. He recommended that Olivia run a maintenance wash with hot water and vinegar to help eliminate any buildup inside the drum. Jacob also advised checking the drain filter for any debris that might be obstructing proper drainage. Since the issue persisted, Jacob scheduled a technician visit to inspect the drain pump, gasket, and drain hose for any blockages or signs of mildew buildup. ","Yes, technician visit scheduled to inspect the drain pump, gasket,"," Rating: 3 Comments: ""The advice was helpful, but the odor is still there. Hoping the technician can sort this out quickly."" Source System: CRM --- ### Customer Service Log 13 Customer Information",,0.00745440972968936 -Henry Lewis,11/15/24,792785,henry.lewis@email.com,-7775,"875 Pine Street, Philadelphia, PA 19101",4706765432," Henry’s QuickWash 1000 washing machine has started to not respond to any button presses on the control panel. He reported that the machine powers on normally, but when he tries to start a wash cycle, none of the buttons work. The display screen also appears frozen, showing a blank screen after the initial power-up. Henry suspects that the control board or touchpad might be malfunctioning, and that this is preventing the machine from operating properly. ",Emily Scott,35 minutes," Emily explained that when a washing machine’s control panel or touchpad becomes unresponsive, it’s often a sign of a control board issue or a problem with the wiring connections. Emily recommended that Henry unplug the machine for a few minutes to reset the system, which might help if the issue was due to a temporary electronic glitch. Since this didn’t resolve the problem, Emily scheduled a technician visit to inspect the control board, touchpad, and the wiring connections for any issues. She noted that if the control board was faulty, it would need to be replaced. ","Yes, technician visit scheduled to inspect the control board and"," Rating: 4 Comments: ""Service was ebicient, but I really need this issue fixed soon as the washing machine is practically unusable right now."" Source System: CRM --- ### Customer Service Log 14 Customer Information",,0.0001770696835592389 -Ava Robinson,11/16/24,793214,ava.robinson@email.com,-5331,"520 Pinehurst Road, San Jose, CA 95128",4710923845," Ava reported that her SuperClean 800 washing machine is having trouble with the spin cycle, as it is not reaching full speed. Despite running a heavy-duty cycle, the clothes remain wet after the spin cycle finishes. Ava suspects that the issue could be with the drive motor or the spin belt, which she believes may be slipping or failing to engage properly during the spin cycle. She confirmed that the drainage is working fine and that no blockages are present. ",Sarah Morgan,30 minutes," Sarah explained that a washing machine that doesn’t reach full spin speed is often due to a malfunctioning motor or a broken spin belt, both of which can prevent the drum from reaching the correct RPM. Sarah advised Ava to check the spin cycle settings and make sure the machine was not overloaded, as that could also cause the spin cycle to run inebiciently. Since this didn’t resolve the issue, Sarah scheduled a technician visit to inspect the motor, spin belt, and drum assembly. She also advised Ava to avoid using large, heavy loads until the problem was fixed to avoid damaging the system further. ","Yes, technician visit scheduled to inspect the drive motor and spin"," Rating: 5 Comments: ""Great support. I’m confident the technician will figure out the issue and fix it."" Source System: CRM --- ### Customer Service Log 15 Customer Information",,0.000320453749736771 -Lucas Wright,11/17/24,790539,lucas.wright@email.com,-8997,"874 Birch Street, Miami, FL 33139",4709281365," Lucas reported that his WashPro 900 washing machine is consistently overfilling with water during the wash cycle, even when he sets it to a low water level. He mentioned that the machine will fill up beyond the normal water line and cause water to overflow from the door. Lucas suspects that the issue might be due to a faulty water inlet valve or pressure switch that is not regulating the water flow properly. ",Rachel Bennett,28 minutes," Rachel explained that overfilling during the wash cycle is typically caused by a malfunctioning pressure switch or water inlet valve, both of which control the amount of water entering the drum. She recommended that Lucas check for any blockages in the water inlet valve and ensure that the hose was not kinked or obstructed. Since the issue persisted, Rachel arranged for a technician visit to inspect the pressure switch, water inlet valve, and the drainage system to identify the root cause of the overfilling issue. ","Yes, technician visit scheduled to inspect the water inlet valve and"," Rating: 3 Comments: ""Support was helpful, but I’ve been dealing with this overfilling issue for weeks now. I hope the technician can fix it soon."" Source System: CRM --- ### Customer Service Log 16 Customer Information",,0.00016656539810355753 -Lily Adams,11/18/24,795023,lily.adams@email.com,-11108,"1070 Spruce Lane, Houston, TX 77001",4713485932," Lily’s SmartWash 900 washing machine is displaying a ""water flow error"" during the initial filling cycle. She reported that the water does not fill up to the appropriate level, and the machine stops after a few minutes with the error code displayed. Lily confirmed that the water supply is working in her home, and no issues with the pipes were detected. She suspects a malfunction with the water inlet valve or flow sensor. ",Ethan Clarke,15 minutes," Ethan explained that the water flow error is typically caused by issues with the water inlet valve or a faulty flow sensor, both of which control the water supply to the drum. He walked Lily through checking the inlet hose for kinks or obstructions, which could restrict water flow. Since the issue persisted, Ethan escalated the case to the Field Service Management team for an on-site technician visit. The technician would inspect the water inlet valve and flow sensor to identify any malfunction. ","Yes, technician visit scheduled to inspect and replace water inlet"," Rating: 4 Comments: ""The troubleshooting steps were useful, but I hope the technician can fix the issue soon. I need the machine working properly."" Source System: Field Service Management --- ### Customer Service Log 17 Customer Information",,0.00012659716594498605 -George Marshall,11/19/24,796521,george.marshall@email.com,-6331,"453 Birchwood Ave, Chicago, IL 60602",4712457890," George reported that his WashMax 3000 washing machine has been making a loud, grinding noise during the spin cycle. The noise has become progressively louder, and George is concerned that it could be a serious mechanical issue. He suspects the noise may be coming from the motor, drum bearings, or the drive belt. He noted that the machine still spins, but the noise is very disruptive. ",Natalie Rogers,25 minutes," Natalie explained that a loud grinding noise during the spin cycle could be caused by worn drum bearings, a loose drive belt, or a faulty motor coupling. She walked George through checking the drum alignment and ensuring that the machine was not overloaded, which could exacerbate the noise. Since the issue was mechanical and likely internal, Natalie scheduled a technician visit through SAP to inspect the motor, drum bearings, and drive belt for any signs of wear or damage. ","Yes, technician visit scheduled to inspect motor, drum bearings,"," Rating: 3 Comments: ""The support was helpful, but I’m concerned about the cost of fixing the motor or bearings. It’s been quite a frustrating experience."" Source System: SAP --- ### Customer Service Log 18 Customer Information",,0.002657181117683649 -Zoe Carter,11/20/24,797631,zoe.carter@email.com,-5331,"901 Oak Ridge Blvd, Los Angeles, CA 90001",4714579023," Zoe’s QuickSpin 1000 washing machine has an issue with excessive vibrations during the spin cycle, even with small loads. She mentioned that the machine moves across the floor during the spin, and she has already checked the leveling of the unit, which seems fine. Zoe suspects that the problem could be with the shock absorbers or suspension springs. ",Michael Harris,30 minutes," Michael explained that excessive vibrations during the spin cycle are often due to issues with the shock absorbers or suspension springs, which are responsible for stabilizing the drum during high-speed spinning. He walked Zoe through verifying that the machine was level and balanced, which she confirmed. Michael recommended that Zoe avoid overloading the machine and advised that a technician inspect the shock absorbers and suspension springs for wear or damage. He scheduled the technician visit through Field Service Management to assess and replace the faulty parts. ","Yes, technician visit scheduled to inspect and replace shock"," Rating: 4 Comments: ""The support was helpful, but I hope the technician can resolve the vibration issue soon. It's been annoying trying to balance the machine."" Source System: Field Service Management --- ### Customer Service Log 19 Customer Information",,0.0002705753722693771 -Jack Davis,11/21/24,798234,jack.davis@email.com,-9219,"3500 Maplewood Lane, San Francisco, CA 94105",4720293847," Jack reported that his EcoSpin 600 washing machine is not draining water properly at the end of the wash cycle. The machine completes the cycle, but there is still standing water in the drum. Jack has checked the drain filter and confirmed that it’s clean, but the issue persists. He suspects the drain pump or drain hose may be clogged. ",Rebecca Foster,22 minutes," Rebecca explained that improper drainage is often caused by issues with the drain pump or a clogged drain hose, which could be obstructing the flow of water. She guided Jack through inspecting the drain hose for any visible blockages. Since the hose was clear, Rebecca suggested that the drain pump might have a fault. She escalated the issue through SAP for a technician visit to inspect and possibly replace the drain pump and verify the entire drainage system. ","Yes, technician visit scheduled to inspect and replace drain pump."," Rating: 5 Comments: ""The troubleshooting steps were helpful, and the technician visit is scheduled quickly. I appreciate the fast response."" Source System: SAP --- ### Customer Service Log 20 Customer Information",,0.00015676785551477224 -Sarah Mitchell,11/22/24,799012,sarah.mitchell@email.com,-4331,"2180 Oakwood Ave, Phoenix, AZ 85001",4723128475," Sarah reported that her TurboWash 5000 washing machine is experiencing intermittent power issues. The machine powers ob unexpectedly mid-cycle and sometimes won’t turn on again without multiple attempts. Sarah noted that the power cord and outlet are fine, and there are no visible signs of a power surge. She suspects a problem with the power supply board or internal wiring. ",Thomas Hill,40 minutes," Thomas explained that intermittent power issues are typically caused by a faulty power supply board or wiring issues that cause the machine to lose connection. He advised Sarah to check if there were any error codes displayed when the power failure occurred, but Sarah confirmed there were none. Since the issue seemed related to the internal components, Thomas arranged a technician visit through Field Service Management to inspect the power supply board and internal wiring for faults and perform any necessary replacements. ","Yes, technician visit scheduled to inspect and replace the power"," Rating: 3 Comments: ""The support was good, but I’m concerned about the internal wiring issue. I’ve had problems with this machine since I got it."" Source System: Field Service Management --- ### Customer Service Log 21 Customer Information",,0.00030394032364711165 -Ben Thompson,11/23/24,799748,ben.thompson@email.com,-2321,"1155 Redwood Lane, Austin, TX 73301",4724657892," Ben reported that his TurboClean 900 washing machine is making a strange beeping noise after every wash cycle. The machine operates normally during the cycle, but the beeping happens when the cycle finishes, and it continues until the door is opened. Ben confirmed that there is no error code displayed, and the machine is functioning properly otherwise. He suspects it could be a settings issue or something related to software malfunctions. ",Olivia White,20 minutes," Olivia explained that the beeping could be a result of an alert setting indicating that the cycle has completed. She recommended checking the machine’s alarm settings and confirmed that no errors were logged. After troubleshooting, Olivia suggested that the issue might be due to a minor software glitch and escalated the case to SAP for further analysis. A technician was scheduled to check if a firmware update is needed. ","Yes, technician visit scheduled to check for software issue or"," Rating: 4 Comments: ""The beeping is annoying, but I appreciate the fast response. Hopefully, it’s just a simple fix."" Source System: SAP --- ### Customer Service Log 22 Customer Information",,0.0008498365641571581 -Rachel Clark,11/24/24,800214,rachel.clark@email.com,-6886,"2300 Oak Drive, Boston, MA 02115",4730567884," Rachel reported that her WashPro 600 washing machine has a leaking door seal. She observed that after every wash cycle, there is water around the machine’s door, and it pools on the floor. Rachel confirmed that the door is properly closed during each cycle, but water still leaks out. She suspects that the door gasket is damaged or worn out. ",John Smith,18 minutes," John explained that a leaking door seal is usually caused by a worn gasket or improper sealing. He advised Rachel to check for any visible cracks or damage on the door seal. Since Rachel confirmed that the gasket seemed worn, John scheduled a Field Service Management technician to inspect the seal and replace the damaged gasket. ","Yes, technician visit scheduled to replace door seal."," Rating: 3 Comments: ""The support was helpful, but I’ve had this problem for a while. I really hope the technician fixes it this time."" Source System: Field Service Management --- ### Customer Service Log 23 Customer Information",,0.00033511361107230186 -Lisa Roberts,11/25/24,801658,lisa.roberts@email.com,-5220,"540 Maplewood Ave, Denver, CO 80202",4731245793," Lisa’s SuperWash 1000 washing machine is not spinning correctly during the final rinse. She noted that although the machine agitates during the wash cycle, it fails to spin properly and her clothes are coming out soaking wet. She suspects a malfunction with the spin motor or clutch assembly. ",Charles Miller,22 minutes," Charles explained that the issue is likely related to the spin motor, clutch assembly, or even a broken spin belt. He guided Lisa through checking for any obstructions in the drum or any blockages in the drain hose, but since these appeared clear, he scheduled a technician visit through SAP to inspect the spin motor, clutch, and drain system. ","Yes, technician visit scheduled to inspect spin motor and clutch"," Rating: 4 Comments: ""Good support, but I’m getting frustrated that I can’t use the machine properly. Hope it gets fixed soon."" Source System: SAP --- ### Customer Service Log 24 Customer Information",,0.00020283933554310352 -Amy Wilson,11/26/24,802341,amy.wilson@email.com,-7997,"1805 Pine Ridge Dr, Seattle, WA 98001",4732018439," Amy’s UltraSpin 4000 washing machine is displaying a ""drain error"" after completing a wash cycle. The machine stops draining water and remains stuck in the rinse cycle. Amy mentioned that she has tried checking the filter and confirmed it’s clean, but the issue persists. She suspects a problem with the drain pump or clogged hose. ",Stephanie Evans,40 minutes," Stephanie advised Amy that a drain error is often caused by a clogged drain hose or a faulty drain pump. She suggested checking if the hose was kinked, but since it wasn’t, she escalated the issue to Field Service Management for a technician visit. The technician would inspect the pump and the drain hose for potential clogs or malfunctions. ","Yes, technician visit scheduled to inspect drain pump and hose."," Rating: 5 Comments: ""Great service, and I’m relieved that the issue is being handled. Looking forward to the technician's visit."" Source System: Field Service Management --- ### Customer Service Log 25 Customer Information",,0.0001272695226361975 -Mark Harris,11/27/24,803547,mark.harris@email.com,-665,"784 Cedar Street, New York, NY 10001",4733349576," Mark reported that his QuickWash 1200 washing machine’s drum is wobbling during spin cycles. The wobbling is so severe that the machine shakes and vibrates against the wall. Mark confirmed that the machine is level and that it’s not overloaded, but the problem persists. He suspects it could be an issue with the suspension system or balance rings. ",Paul Jacobs,15 minutes," Paul explained that a wobbling drum during the spin cycle is often due to issues with the suspension system, shock absorbers, or balance rings that keep the drum steady during high-speed spins. He advised Mark to try redistributing the load during a wash cycle, but since the issue continued, he arranged for a technician visit via SAP to inspect and repair the suspension system and balance rings. ","Yes, technician visit scheduled to inspect suspension system and"," Rating: 4 Comments: ""I’m glad the issue is being taken seriously. Hopefully, the technician can resolve the wobbling soon."" Source System: SAP --- ### Customer Service Log 26 Customer Information",,0.006668669171631336 -Samuel Green,11/28/24,804412,samuel.green@email.com,-4553,"678 Elmwood Drive, Phoenix, AZ 85001",4734571035," Samuel’s EcoWash 500 washing machine is having issues with water not draining fully during the spin cycle. He reported that the machine completes the wash cycle but leaves a significant amount of water in the drum. He confirmed that the filter is clear, and there are no visible blockages in the drain hose. Samuel suspects the drain pump may be malfunctioning. ",Karen Williams,28 minutes," Karen explained that the issue could likely be caused by a faulty drain pump, which would prevent water from being pumped out during the spin cycle. She advised Samuel to check the drain filter again and make sure there were no blockages. Since the issue persisted, Karen arranged a Field Service Management technician visit to inspect the drain pump and test its functionality. ","Yes, technician visit scheduled to inspect and replace drain pump."," Rating: 3 Comments: ""The machine isn’t draining properly, and I’m getting frustrated. I hope the technician can fix this soon."" Source System: Field Service Management --- ### Customer Service Log 27 Customer Information",,0.00027618688181973994 -Helen Brown,11/29/24,805273,helen.brown@email.com,-8442,"302 Birch Street, Atlanta, GA 30301",4735621948," Helen reported that her SpeedWash 300 washing machine’s detergent dispenser is not working correctly. The detergent doesn’t fully dispense, and she noticed that the machine is leaving detergent residue in the compartment. Helen suspects an issue with the water inlet valve or the dispenser system. ",Benjamin Cook,25 minutes," Benjamin advised Helen that the problem is often due to a clogged detergent dispenser or a malfunctioning water inlet valve. He walked Helen through cleaning the dispenser tray and ensuring the water supply was working properly. After confirming the issue persisted, he scheduled a technician visit through SAP to inspect the water inlet valve and ensure the dispenser was functioning correctly. ","Yes, technician visit scheduled to inspect and replace water inlet"," Rating: 5 Comments: ""Thanks for your help. I’m glad this is being addressed quickly. I’m hopeful the technician can resolve the issue."" Source System: SAP --- ### Customer Service Log 28 Customer Information",,0.00015080420416779816 -Henry Smith,11/30/24,806485,henry.smith@email.com,-9886,"442 Pinehill Road, San Jose, CA 95101",4736742017," Henry’s MaxSpin 1000 washing machine is stopping mid-cycle. The machine powers ob unexpectedly during the wash cycle, and sometimes, it won’t start at all. Henry tried resetting the machine and unplugging it, but the issue persists. He suspects a problem with the power board or electrical connections. ",Monica Taylor,35 minutes," Monica explained that sudden power losses mid-cycle are often caused by a faulty power board or a short circuit in the machine’s wiring. She recommended checking the error codes on the display, but since there were none, she scheduled a technician visit through Field Service Management to inspect the power board and electrical components. ","Yes, technician visit scheduled to inspect and replace power board."," Rating: 2 Comments: ""I’m really disappointed with the frequent breakdowns of this machine. I’ve had to call for service several times."" Source System: Field Service Management --- ### Customer Service Log 29 Customer Information",,0.00022953079314902425 -Diana Evans,12/1/24,807220,diana.evans@email.com,-7553,"2001 Aspen Court, Dallas, TX 75001",4737853099," Diana’s QuickClean 700 washing machine is displaying a ""water level sensor"" error. The machine seems to be overfilling during cycles, and Diana confirmed that the water level sensor may need to be replaced. She tried recalibrating the machine, but the issue persists. ",Jason Lee,18 minutes," Jason confirmed that a faulty water level sensor could be causing the issue and guided Diana through resetting the sensor. Since resetting didn’t resolve the error, he scheduled a technician visit through SAP to replace the water level sensor and calibrate it. ","Yes, technician visit scheduled to replace water level sensor."," Rating: 4 Comments: ""Good support, but I’m hoping the sensor replacement fixes the problem for good."" Source System: SAP --- ### Customer Service Log 30 Customer Information",,0.0003282120160292834 -Julia Green,12/2/24,808451,julia.green@email.com,-2220,"1250 Hilltop Blvd, Miami, FL 33101",4741102358," Julia’s EcoSpin 2000 washing machine has a persistent odor after each wash cycle, even after running an empty cycle with vinegar. She mentioned that the smell seems to come from the drum, and she suspects there may be mold or mildew buildup, despite cleaning the machine regularly. ",Liam Harris,30 minutes," Liam explained that odors are commonly caused by mold or mildew buildup in the door seal or drum. He advised Julia to check the drain pump filter and door seal for residue and moisture. After confirming the issue persisted, he scheduled a technician visit via SAP to perform a thorough cleaning and inspection of the drum, pump, and seals to eliminate the odor. ","Yes, technician visit scheduled to clean and inspect drum and"," Rating: 3 Comments: ""I’m disappointed that the smell keeps coming back. I hope the technician can fix it for good."" Source System: SAP --- ### Customer Service Log 31 Customer Information",,0.000546164286788553 -Darren King,12/3/24,809276,darren.king@email.com,-3998,"9457 Oakwood Drive, Orlando, FL 32801",4742239081," Darren’s SpinMaster 500 washing machine is experiencing dibiculty completing the wash cycle. The machine often gets stuck on the wash phase, and Darren needs to restart the cycle to complete it. He suspects an issue with the motor or timer mechanism. ",Rachel Scott,25 minutes," Rachel explained that a machine stuck on the wash phase is often due to a faulty timer or motor not completing the cycle. She walked Darren through resetting the machine and checking for any error codes. Since the issue persisted, Rachel escalated the issue to Field Service Management for an on-site technician visit. The technician would inspect the motor, timer, and control board to identify the root cause. ","Yes, technician visit scheduled to inspect motor and timer."," Rating: 4 Comments: ""I’m glad the support team is investigating. It’s frustrating to restart the cycle every time."" Source System: Field Service Management --- ### Customer Service Log 32 Customer Information",,0.00023480135132558644 -Martin Edwards,12/4/24,810112,martin.edwards@email.com,-998,"2030 Birchwood St, San Diego, CA 92101",4743247621," Martin’s QuickClean 1500 washing machine is not agitating during the wash cycle. The drum remains still, and the clothes are not being washed properly. Martin checked the agitator and ensured the drum was not overloaded, but the issue persists. ",Sarah White,18 minutes," Sarah explained that a lack of agitation can be caused by a faulty agitator motor, coupling, or belt. She guided Martin through verifying the belt and the agitator for any visible damage. Since the issue continued, Sarah escalated the case to SAP for a technician to inspect the agitator motor and coupling for potential failure. ","Yes, technician visit scheduled to inspect agitator motor and"," Rating: 3 Comments: ""The support was decent, but I really need the agitator fixed. It’s making washing clothes a hassle."" Source System: SAP --- ### Customer Service Log 33 Customer Information",,0.0013983179815113544 -Emily White,12/5/24,811439,emily.white@email.com,-10441,"1287 Sunset Blvd, Los Angeles, CA 90001",4744365920," Emily’s WashSmart 800 washing machine has a noisy spin cycle. The machine makes a loud, high-pitched noise when it reaches full spin speed. Emily confirmed there are no clothes caught in the drum, and the machine seems level. She suspects a problem with the drum bearings or drive belt. ",Greg Stone,27 minutes," Greg explained that the noise is likely caused by worn drum bearings or a loose drive belt. He walked Emily through checking for any visible issues with the belt and drum but confirmed that these appeared fine. Since the noise persisted, he scheduled a Field Service Management technician visit to inspect and possibly replace the drum bearings or drive belt. ","Yes, technician visit scheduled to inspect and replace drum"," Rating: 4 Comments: ""The noise is getting really annoying. Hopefully, the technician can fix it soon."" Source System: Field Service Management --- ### Customer Service Log 34 Customer Information",,0.00024661299539729953 -Patricia Moore,12/6/24,812876,patricia.moore@email.com,-7442,"3564 Oakhill Drive, Atlanta, GA 30301",4745523477," Patricia reported that her SuperWash 1200 washing machine stopped mid-cycle and won’t restart. She mentioned that the machine was working fine until it suddenly turned ob during the rinse cycle. She has tried pressing the reset button and unplugging the machine, but it still won't power back on. ",Jason Brooks,40 minutes," Jason explained that a machine turning ob mid-cycle could be caused by a power board failure, faulty wiring, or overheating protection. He guided Patricia through checking the plug and outlet for issues but confirmed there were none. He then escalated the case to SAP for a technician visit to inspect and possibly replace the power board and test the internal wiring. ","Yes, technician visit scheduled to inspect and replace power board."," Rating: 2 Comments: ""Very frustrating that the machine just turned ob. I’ve had enough of these power issues."" Source System: SAP --- ### Customer Service Log 35 Customer Information",,0.00038764093187637627 -Vanessa James,12/7/24,813492,vanessa.james@email.com,-2442,"2795 Pine View Rd, Chicago, IL 60602",4746601093," Vanessa’s WashPro 2000 washing machine is draining very slowly during the rinse cycle, and the drum is often left with standing water. She confirmed that the filter is clean, and the drain hose appears clear. She suspects a malfunction in the drain pump or a blockage in the internal pump hose. ",Lisa Clark,33 minutes," Lisa advised Vanessa that slow draining could indicate a faulty drain pump or a blockage in the pump hose. She recommended checking the pump filter for any residual debris. Since the issue persisted, Lisa scheduled a technician visit via Field Service Management to inspect and test the drain pump and pump hose for clogs or faults. ","Yes, technician visit scheduled to inspect and replace drain pump."," Rating: 5 Comments: ""I’m glad the technician is coming soon. I need the machine to drain properly before I can use it again."" Source System: Field Service Management --- ### Customer Service Log 36 Customer Information",,0.00015732910833321512 -Sophia Evans,2024-12,814256,sophia.evans@email.com,-8664,"1985 Crestwood Lane, Houston, TX 77001",4747703817, Sophia reported that her TurboClean 500 washing machine is having an issue with water temperature. The machine is stuck on cold wash even when she selects a hot water setting. She suspects the heating element or temperature sensor is malfunctioning. ,Robert Turner,22 minutes," Robert explained that a washing machine stuck on cold wash is typically due to a faulty heating element or temperature sensor. He guided Sophia through checking the water inlet valve to ensure it's working properly. Since the problem persisted, Robert arranged for a technician visit via SAP to replace the heating element and test the temperature sensor for functionality. ","Yes, technician visit scheduled to inspect and replace heating"," Rating: 4 Comments: ""I appreciate the support. I just need the temperature issue fixed quickly so I can use the machine again."" Source System: SAP --- ### Customer Service Log 37 Customer Information",,0.00029154791263863444 -Carla Peterson,12/9/24,815774,carla.peterson@email.com,-10997,"1368 Grand Avenue, San Francisco, CA 94110",4748865105," Carla’s EcoWash 1200 washing machine is not filling with water properly during the wash cycle. The machine starts, but no water enters the drum, leaving the clothes dry. Carla checked the water supply and the inlet valve and confirmed they are both working correctly. ",Matthew Lee,30 minutes," Matthew explained that no water filling the drum could be caused by a faulty water inlet valve or a clogged filter. He walked Carla through checking the valve filter and the supply hoses for blockages. Since the issue persisted, Matthew arranged for a technician visit via Field Service Management to inspect and replace the water inlet valve. ","Yes, technician visit scheduled to inspect and replace water inlet"," Rating: 3 Comments: ""I’m frustrated that the machine isn’t working. I’m hoping the technician can fix the issue soon."" Source System: Field Service Management --- ### Customer Service Log 38 Customer Information",,0.00021283984824549407 -Edward Ross,12/10/24,816042,edward.ross@email.com,-6442,"4507 Pine Hill Road, Portland, OR 97201",4749981340," Edward reported that his SpinWash 1800 washing machine’s door won’t lock properly. The machine starts a wash cycle but stops after a few minutes, showing a door lock error. Edward confirmed that there is no obstruction in the door mechanism, and the door appears aligned. ",Karen Fisher,24 minutes," Karen explained that a door lock error can be caused by a faulty door latch or door lock mechanism. She advised Edward to check for any visible damage to the door components. Since the issue continued, Karen scheduled a technician visit via SAP to inspect and replace the door lock mechanism. ","Yes, technician visit scheduled to inspect and replace door lock."," Rating: 4 Comments: ""The issue seems to be with the door. I hope the technician can get it fixed without much hassle."" Source System: SAP --- ### Customer Service Log 39 Customer Information",,0.0011916260700672865 -Marcus Johnson,12/11/24,817592,marcus.johnson@email.com,-3220,"1584 Redwood Ave, Sacramento, CA 95814",4751112378," Marcus reported that his TurboClean 500 washing machine was not draining at the end of the cycle. He mentioned that there was standing water in the drum after each wash, even though the drain hose was clear. Marcus suspected an issue with the drain pump or filter. ",Anna Gray,28 minutes," Anna advised Marcus to clean the drain filter and inspect the hose for any visible clogs. Since the issue persisted, she scheduled a technician visit via Field Service Management to inspect and replace the drain pump if necessary. ","Yes, technician visit scheduled to inspect and replace drain pump."," Rating: 4 Comments: ""I’m frustrated with this issue, but the support was decent. Hopefully the technician can finally fix it."" Source System: Field Service Management --- ### Customer Service Log 40 Customer Information",,0.00025551579892635345 -Linda Anderson,12/12/24,818130,linda.anderson@email.com,-7108,"1983 Pinecrest Way, Denver, CO 80202",4752203899," Linda’s WashMaster 3000 washing machine is stopping mid-cycle without completing the wash or spin phases. She has tried resetting the machine multiple times, but it keeps shutting ob randomly. She suspects a problem with the timer or control board. ",Oliver Harris,22 minutes," Oliver walked Linda through checking the error codes and suggested resetting the machine multiple times. After confirming the issue persisted, he escalated the case to SAP for a technician visit to inspect the control board and timer. ","Yes, technician visit scheduled to inspect and replace control"," Rating: 5 Comments: ""Thank you for helping. I hope this issue is resolved. The machine has been a nightmare for the past week."" Source System: SAP --- ### Customer Service Log 41 Customer Information",,0.00023222251911647618 -Frank Lewis,12/13/24,819444,frank.lewis@email.com,-10108,"2156 Horizon St, Chicago, IL 60611",4753401755," Frank’s MaxSpin 2000 washing machine is making a loud banging noise during the spin cycle. Frank mentioned that he made sure the machine was properly leveled, but the noise is still very loud and disruptive. He suspects an issue with the drum bearings or suspension springs. ",Kelly Morgan,18 minutes," Kelly explained that the noise could be due to worn drum bearings or suspension springs. She advised Frank to check the suspension springs for any visible damage. Since the issue persisted, Kelly arranged a technician visit via Field Service Management to inspect and replace the drum bearings. ","Yes, technician visit scheduled to inspect and replace drum"," Rating: 3 Comments: ""The noise is unbearable. I can’t stand it anymore. Please get this fixed ASAP!"" Source System: Field Service Management --- ### Customer Service Log 42 Customer Information",,0.001093846163712442 -Nathan Parker,12/14/24,820289,nathan.parker@email.com,-6886,"3327 Maple St, Phoenix, AZ 85001",4754536791," Nathan reported that his QuickWash 1000 washing machine won’t start after selecting a wash cycle. The control panel lights up, but the machine does not initiate any wash actions. Nathan suspected a malfunction with the control board or start switch. ",Jasmine Lee,35 minutes, Jasmine walked Nathan through checking the control panel and ensuring the door was properly closed. She confirmed the issue persisted and scheduled a technician visit via SAP to inspect and replace the control board or start switch. ,"Yes, technician visit scheduled to inspect and replace control"," Rating: 2 Comments: ""I am so sick of this machine. This is the third time I’ve had problems! I just want a replacement!"" Source System: SAP --- ### Customer Service Log 43 Customer Information",,0.002052868250757456 -Amanda Green,12/15/24,821377,amanda.green@email.com,-9219,"9084 Valley Rd, Portland, OR 97223",4755641032," Amanda’s EcoSpin 2000 washing machine is not spinning properly during the final cycle. The clothes are coming out soaking wet, and the machine appears to be underperforming on the spin cycle. She suspects a problem with the drum motor or drive belt. ",Thomas White,20 minutes," Thomas advised Amanda that a malfunctioning drum motor or drive belt could cause incomplete spinning. After troubleshooting, he escalated the issue to Field Service Management for a technician visit to inspect and replace the drum motor or drive belt. ","Yes, technician visit scheduled to inspect and replace drum motor."," Rating: 5 Comments: ""It’s been frustrating, but the support has been helpful. Hopefully, the technician can get this fixed."" Source System: Field Service Management --- ### Customer Service Log 44 Customer Information",,0.000362074701115489 -Kevin Taylor,12/16/24,822651,kevin.taylor@email.com,-5220,"1223 Oakridge Drive, Las Vegas, NV 89101",4756724389," Kevin’s WashPro 1200 washing machine is not dispensing detergent during the wash cycle. Despite adding detergent to the dispenser, it remains in the compartment, and the clothes don’t seem to be cleaned properly. Kevin suspects a blockage in the detergent dispenser system. ",Olivia Moore,30 minutes," Olivia explained that detergent not dispensing could be caused by a clogged dispenser or a malfunctioning water inlet valve. After guiding Kevin through cleaning the dispenser, she confirmed the issue persisted and escalated the issue to SAP for a technician visit to inspect the dispenser system and water valve. ","Yes, technician visit scheduled to inspect and clean dispenser"," Rating: 4 Comments: ""I’m annoyed that the detergent isn’t dispensing correctly. I expect a better product."" Source System: SAP --- ### Customer Service Log 45 Customer Information",,0.002651396906003356 -Alicia Harris,12/17/24,823493,alicia.harris@email.com,-7553,"3121 Elm St, New York, NY 10001",4757834098," Alicia’s SpinMaster 1000 washing machine is producing a burning smell during the spin cycle. She said the smell is noticeable even when no heavy loads are being washed. Alicia is concerned it might be a sign of a more serious issue, possibly with the motor or wiring. ",Daniel Clark,25 minutes, Daniel recommended that Alicia unplug the machine immediately as a safety precaution and not use it until it’s inspected. He scheduled a technician visit via Field Service Management to inspect and potentially replace the motor or wiring that could be causing the burning smell. ,"Yes, technician visit scheduled to inspect and replace motor."," Rating: 5 Comments: ""I’m scared to use it after that smell. Thank you for the fast response. Please fix this!"" Source System: Field Service Management --- ### Customer Service Log 46 Customer Information",,0.001158195547759533 -Julie Roberts,12/18/24,824006,julie.roberts@email.com,-9886,"2217 Sun Valley Dr, Dallas, TX 75201",4758957320," Julie’s EcoWash 1500 washing machine is leaking water from underneath the machine. The leak is noticeable when the machine starts the wash cycle, and it leaves puddles around the machine. Julie suspects a problem with the water pump or drain hose. ",Nathan Collins,30 minutes," Nathan guided Julie to check the water hoses and pump filter for any visible leaks or damage. Since the leak continued, he scheduled a technician visit via SAP to inspect and replace the water pump or hose. ","Yes, technician visit scheduled to inspect and replace pump."," Rating: 1 Comments: ""This is ridiculous! The machine has been leaking for days and no one is fixing it. I’m done with this brand!"" Source System: SAP --- ### Customer Service Log 47 Customer Information",,0.513248085975647 -Brian Mitchell,12/19/24,825137,brian.mitchell@email.com,-1442,"2571 Briarwood Dr, Austin, TX 73301",4760018825," Brian’s SmartWash 700 washing machine won’t drain properly at the end of the cycle. Water remains in the drum after completing the cycle, and the clothes are still soaked. Brian tried checking the drain hose and filter but the problem persists. ",Helen Scott,25 minutes," Helen explained that the issue could be caused by a faulty drain pump or blockage in the drain hose. After troubleshooting, she scheduled a technician visit via Field Service Management to inspect and replace the drain pump. ","Yes, technician visit scheduled to inspect and replace pump."," Rating: 3 Comments: ""I’ve had enough of these problems. If the technician doesn’t fix it soon, I’ll demand a refund."" Source System: Field Service Management --- ### Customer Service Log 48 Customer Information",,0.00032178364926949143 -Thomas Williams,12/20/24,826255,thomas.williams@email.com,-10886,"1152 Ashwood Dr, Minneapolis, MN 55123",4761134120," Thomas reported that his UltraWash 3000 washing machine is vibrating excessively during the spin cycle, causing the whole unit to shake and move across the floor. He mentioned that he had leveled the machine but the issue continues. ",Mark Johnson,22 minutes," Mark explained that an unbalanced load or worn suspension springs could be causing the vibration. He guided Thomas through checking the load distribution and inspecting the suspension springs. Since the issue persisted, Mark scheduled a technician visit via SAP to inspect and replace the suspension springs. ","Yes, technician visit scheduled to inspect and replace suspension"," Rating: 4 Comments: ""I’m glad someone’s coming to fix it, but I’ve had enough of this. I just want the machine to work properly!"" Source System: SAP --- ### Customer Service Log 49 Customer Information",,0.00022851758694741875 -Jessica Bryant,12/21/24,827404,jessica.bryant@email.com,-2442,"4632 Birchwood Rd, San Diego, CA 92103",4762231882," Jessica’s SmartWash 500 washing machine is not agitating properly during the wash cycle. The drum spins slowly, and clothes are not being cleaned as expected. Jessica believes the agitator or motor is malfunctioning. ",Nancy Hall,30 minutes," Nancy walked Jessica through checking the agitator for blockages and confirmed it was moving improperly. Since the issue wasn’t resolved, she escalated the case via Field Service Management for a technician visit to inspect and replace the agitator or motor. ","Yes, technician visit scheduled to inspect and replace agitator."," Rating: 5 Comments: ""The support team was helpful, but I’m really annoyed this happened right after the warranty expired."" Source System: Field Service Management --- ### Customer Service Log 50 Customer Information",,0.00014407523849513382 -Natalie Cooper,12/22/24,828552,natalie.cooper@email.com,-6331,"1056 Kings Rd, Orlando, FL 32801",4763347501," Natalie’s TurboSpin 1200 washing machine is leaking water from the bottom panel during the wash cycle. The water pools on the floor around the machine, and Natalie suspects a problem with the water pump or seal. ",Leo Davis,26 minutes," Leo explained that a water leak could be caused by a damaged water seal or faulty water pump. He advised Natalie to inspect the water hose and drain filter for any visible damage. Since the leak persisted, Leo scheduled a technician visit via SAP to inspect and replace the water pump or seal. ","Yes, technician visit scheduled to inspect and replace water pump."," Rating: 2 Comments: ""This is the second time my machine is leaking. I’m getting tired of this brand. FIX IT."" Source System: SAP --- ### Customer Service Log 51 Customer Information",,0.0002608941576909274 -Adam Rodriguez,12/23/24,829701,adam.rodriguez@email.com,-4331,"2987 Oak Valley Dr, Miami, FL 33101",4764450613," Adam’s QuickWash 1500 washing machine is not spinning at all during the final spin cycle. The clothes are coming out wet and heavy, and the drum does not rotate. Adam has checked the door lock and load balance, but the problem persists. ",Carla Moore,24 minutes," Carla explained that the issue could be due to a broken drive belt or faulty motor. She advised Adam to inspect the drive belt for any visible damage. Since the issue wasn’t resolved, Carla scheduled a technician visit via Field Service Management to inspect and replace the drive belt or motor. ","Yes, technician visit scheduled to inspect and replace drive belt."," Rating: 5 Comments: ""I’m hoping this gets fixed soon. I’ve already had too many issues with this machine."" Source System: Field Service Management --- ### Customer Service Log 52 Customer Information",,0.00043143669608980417 -Barbara Grant,12/24/24,830014,barbara.grant@email.com,-5664,"5743 Riverside Ave, Austin, TX 73301",4765564392," Barbara’s EcoSpin 1000 washing machine is overheating during the wash cycle. The temperature sensor seems to be malfunctioning as the water gets hotter than expected, potentially damaging clothes. Barbara has confirmed that the thermostat and water inlet valve are functioning. ",Daniel Scott,33 minutes, Daniel explained that overheating can be caused by a faulty temperature sensor or malfunctioning control board. He escalated the issue via SAP and scheduled a technician visit to inspect and replace the temperature sensor or control board. ,"Yes, technician visit scheduled to inspect and replace temperature"," Rating: 3 Comments: ""The machine shouldn’t be overheating like this. It’s frustrating, especially since it’s brand new!"" Source System: SAP --- ### Customer Service Log 53 Customer Information",,0.0005323936929926276 -Eric Foster,12/25/24,831179,eric.foster@email.com,-2553,"7455 Hilltop Ln, Los Angeles, CA 90001",4766678245," Eric’s SpinClean 800 washing machine is not draining water at the end of the cycle. The water remains in the drum, and the clothes are still soaking wet. Eric has already cleaned the drain filter and checked the drain hose but the issue persists. ",Helen Taylor,20 minutes, Helen explained that the issue could be with the drain pump or a blockage in the drain hose. She arranged a technician visit via Field Service Management to inspect the pump and hose for any obstructions. ,"Yes, technician visit scheduled to inspect and replace drain pump."," Rating: 1 Comments: ""I’ve had enough of this junk! It’s been a month of issues and no one’s fixing it! Get this thing replaced."" Source System: Field Service Management --- ### Customer Service Log 54 Customer Information",,0.8846611380577087 -Samuel Lewis,12/26/24,832254,samuel.lewis@email.com,-998,"1586 Willow Creek Rd, Dallas, TX 75201",4767789337," Samuel’s WashMaster 4000 washing machine is producing a strange odor during the wash cycle. He suspects the drain pump or filter might be clogged, but he cannot find any visible issues. ",Anna Walker,30 minutes," Anna advised Samuel to clean the drain pump and filter thoroughly. After confirming the issue persisted, she scheduled a technician visit via SAP to inspect the pump and filter system for any internal clogs. ","Yes, technician visit scheduled to inspect and clean pump."," Rating: 4 Comments: ""The odor is unbearable, but at least I have someone coming to fix it."" Source System: SAP --- ### Customer Service Log 55 Customer Information",,0.0009222364169545472 -Rachel White,12/27/24,833411,rachel.white@email.com,-3998,"3185 Creekside Dr, San Francisco, CA 94105",4768890246," Rachel’s MaxSpin 2000 washing machine is not completing the wash cycle. The machine shuts ob halfway through, and she needs to restart it multiple times to finish the cycle. She suspects an issue with the timer or control board. ",Jennifer Miller,29 minutes," Jennifer advised Rachel to check for any error codes or obstructions. Since the issue persisted, she escalated the issue to Field Service Management for a technician visit to inspect and replace the timer or control board. ","Yes, technician visit scheduled to inspect and replace control"," Rating: 2 Comments: ""I don’t care about the technician visit. Just fix this machine! It’s been down too long."" Source System: Field Service Management --- ### Customer Service Log 56 Customer Information",,0.00040924124186858535 -Peter Knight,12/28/24,834558,peter.knight@email.com,-8553,"2450 Glenwood Rd, Seattle, WA 98101",4770003942," Peter’s TurboClean 1500 washing machine is not starting when he presses the power button. The lights on the control panel come on, but it doesn’t respond when he selects a cycle. Peter suspects a problem with the start button or control board. ",Brian King,27 minutes," Brian walked Peter through checking the power connections and error codes. After confirming the issue persisted, he escalated the case via SAP and scheduled a technician visit to inspect the control board. ","Yes, technician visit scheduled to inspect and replace control"," Rating: 4 Comments: ""I’m beyond frustrated at this point. This machine is brand new and it’s already broken!"" Source System: SAP --- ### Customer Service Log 57 Customer Information",,0.0005144605529494584 -Erica Matthews,12/29/24,835744,erica.matthews@email.com,-5331,"9725 Maplewood St, Portland, OR 97205",4771123567, Erica's EcoWash 1000 washing machine is making a loud grinding noise during the wash cycle. She suspects an issue with the drive belt or motor. ,James Carter,28 minutes, James explained that the grinding noise could be due to a worn drive belt or damaged motor bearings. He scheduled a technician visit via SAP to inspect and potentially replace the drive belt or motor. ,"Yes, technician visit scheduled to inspect and replace drive belt."," Rating: 3 Comments: ""It’s been so frustrating. I don’t know how this machine got so loud so quickly. Let’s see if the technician can fix it."" Source System: SAP --- ### Customer Service Log 58 Customer Information",,0.00046848144847899675 -Michael Spencer,12/30/24,836011,michael.spencer@email.com,-10219,"1156 Eagle St, Denver, CO 80203",4772234568," Michael reported that his SpinMaster 2500 washing machine isn’t spinning at all. The drum remains stationary, and clothes are still soaked after the cycle. ",Emily White,24 minutes," Emily suggested that the drum motor could be faulty, or there could be an issue with the drive belt. She scheduled a technician visit via Field Service Management to inspect and replace the motor or drive belt. ","Yes, technician visit scheduled to inspect and replace motor."," Rating: 5 Comments: ""Quick and ebicient response. I hope the technician can fix it soon."" Source System: Field Service Management --- ### Customer Service Log 59 Customer Information",,0.00019660884572658688 -Jennifer Thomas,12/31/24,836895,jennifer.thomas@email.com,-7442,"3446 Sunridge Ave, Phoenix, AZ 85001",4773345569," Jennifer's MaxSpin 3000 washing machine is not draining properly, and water is left standing in the drum after each cycle. She believes there may be a blockage in the drain hose or filter. ",Laura Phillips,21 minutes," Laura advised Jennifer to clean the drain filter and inspect the hose for obstructions. When the issue wasn’t resolved, Laura escalated the case via SAP to schedule a technician visit to check the drain pump and hose. ","Yes, technician visit scheduled to inspect and clean the drain pump."," Rating: 4 Comments: ""The technician better sort this out. I’ve cleaned everything and it’s still broken."" Source System: SAP --- ### Customer Service Log 60 Customer Information",,0.00048811433953233063 -Steve Harvey,1/1/25,837112,steve.harvey@email.com,-6664,"7900 Northview Dr, Seattle, WA 98109",4774456670, Steve reported that his QuickClean 5000 washing machine’s display screen is flickering and showing random error codes. He’s unable to start a cycle because of this malfunction. ,Sam Collins,19 minutes, Sam explained that the issue could be related to the control board or wiring. He scheduled a technician visit via Field Service Management to inspect and replace the control board. ,"Yes, technician visit scheduled to inspect and replace control"," Rating: 2 Comments: ""How hard is it to make a washing machine that actually works? I’m sick of having to call for repairs."" Source System: Field Service Management --- ### Customer Service Log 61 Customer Information",,0.00046817120164632797 -Karen Brooks,1/2/25,838225,karen.brooks@email.com,-2553,"6715 Valley Dr, Chicago, IL 60601",4775567781," Karen’s EcoSpin 2000 washing machine is overheating during the wash cycle. The water temperature is rising too high, potentially ruining clothes. ",Patricia Lee,32 minutes, Patricia explained that overheating could be caused by a malfunctioning thermostat or control board. She scheduled a technician visit via SAP to inspect and replace the thermostat or control board. ,"Yes, technician visit scheduled to inspect and replace thermostat."," Rating: 3 Comments: ""This is not what I expected from a high-end machine. Can you get this sorted out quickly?"" Source System: SAP --- ### Customer Service Log 62 Customer Information",,0.00014980968262534589 -David King,1/3/25,839478,david.king@email.com,-3775,"8903 Pinehurst Dr, Houston, TX 77001",4776678892," David’s TurboWash 800 washing machine is not starting at all. The power button doesn’t respond, and there are no lights on the control panel. ",Brian Martin,25 minutes," Brian suggested checking the power supply and circuit breaker. Since the machine still wouldn’t start, he escalated the case via Field Service Management to schedule a technician visit to inspect the control board and power connections. ","Yes, technician visit scheduled to inspect and replace control"," Rating: 1 Comments: ""This is insane. I’ve only had this machine for a year, and it’s already broken. This brand is trash."" Source System: Field Service Management --- ### Customer Service Log 63 Customer Information",,0.5792514085769653 -Angela Hill,1/4/25,840325,angela.hill@email.com,-8997,"1108 Crestwood Rd, Los Angeles, CA 90015",4777789921, Angela’s SmartWash 7000 washing machine is not spinning clothes dry. The machine goes through the wash cycle but leaves the clothes soaking wet. ,Kevin Clark,27 minutes, Kevin suggested that the spin motor or pump could be malfunctioning. He arranged a technician visit via SAP to inspect and replace the spin motor or pump. ,"Yes, technician visit scheduled to inspect and replace motor."," Rating: 4 Comments: ""I need this fixed. I can’t abord to keep waiting for a solution."" Source System: SAP --- ### Customer Service Log 64 Customer Information",,0.0003836100222542882 -Richard Young,1/5/25,841196,richard.young@email.com,-9553,"4822 Oakview Dr, Dallas, TX 75209",4778891104, Richard’s SpinForce 2500 washing machine is stopping midway through the wash cycle and not draining the water . ,Olivia Stone,34 minutes," Olivia suggested checking for any blockages in the drain hose and pump. Since the issue wasn’t resolved, she escalated the case via Field Service Management to schedule a technician visit to inspect the pump and drain hose. ","Yes, technician visit scheduled to inspect pump."," Rating: 2 Comments: ""I’m so done with this machine. I can’t trust it to work. Just get it fixed already."" Source System: Field Service Management --- ### Customer Service Log 65 Customer Information",,0.0002852928882930428 -Lisa Green,1/6/25,842401,lisa.green@email.com,-5775,"5224 Ashford Ave, Miami, FL 33165",4779912335, Lisa’s TurboClean 1500 washing machine is vibrating excessively during the spin cycle and making an unusual screeching noise. ,Scott Gray,29 minutes," Scott explained that the issue could be related to unbalanced loads or a faulty suspension system. Since the issue persisted, he escalated the case via SAP and scheduled a technician visit to inspect the suspension springs and balance. ","Yes, technician visit scheduled to inspect suspension springs."," Rating: 3 Comments: ""I’ve leveled the machine, but it’s still vibrating like crazy. Something’s clearly wrong."" Source System: SAP ",,0.05251740291714668 diff --git a/examples/notebooks/hap/output/hap_filtered_output.csv b/examples/notebooks/hap/output/hap_filtered_output.csv deleted file mode 100644 index 13f38a25e..000000000 --- a/examples/notebooks/hap/output/hap_filtered_output.csv +++ /dev/null @@ -1,66 +0,0 @@ -Customer Feedback,hap_score -" Rating: 4 Comments: ""Service was prompt, but I'm worried it might be a bigger issue with the drum. Hope it gets fixed soon."" Source System: CRM --- ### Customer Service Log 2 Customer Information",0.00019484490621834993 -" Rating: 5 Comments: ""Great help from Peter! Hope the technician resolves the issue."" Source System: CRM --- ### Customer Service Log 3 Customer Information",0.00015312987670768052 -" Rating: 3 Comments: ""The service was quick, but I’m still unsure if the issue will be fixed. It keeps happening."" Source System: CRM --- ### Customer Service Log 4 Customer Information",0.00016854520072229207 -" Rating: 5 Comments: ""Excellent service and advice. Hopefully the technician can fix it soon."" Source System: CRM --- ### Customer Service Log 5 Customer Information",0.00015754287596791983 -" Rating: 2 Comments: ""I’m really frustrated. This issue has been ongoing for weeks and still isn’t resolved."" Source System: CRM --- These logs now include more detailed issues and resolutions, which are useful for analysis and decision-making. Sure! Here are 5 more customer service logs with detailed issues and resolutions, including a variety of feedback: --- ### Customer Service Log 6 Customer Information",0.0008747439715079963 -" Rating: 4 Comments: ""The service was helpful, but I’m hoping the issue will be fully resolved with the new parts. It’s been a bit of a nuisance."" Source System: CRM --- ### Customer Service Log 7 Customer Information",0.015778325498104095 -" Rating: 3 Comments: ""The support was helpful, but I’m frustrated the machine keeps stopping. I’ve been waiting for a technician for a while now."" Source System: CRM --- ### Customer Service Log 8 Customer Information",0.00022087343677412719 -" Rating: 5 Comments: ""The service was very quick and the representative explained everything clearly. I hope the technician can fix the leak soon."" Source System: CRM --- ### Customer Service Log 9 Customer Information",0.00017898922669701278 -" Rating: 5 Comments: ""The support was excellent. I’m confident the technician will sort out the issue once and for all."" Source System: CRM --- ### Customer Service Log 10 Customer Information",0.00023553140636067837 -" Rating: 2 Comments: ""I’m frustrated with this issue. I’ve tried everything, and the machine still doesn’t wash properly. I hope the technician can figure it out."" Source System: CRM --- ### Customer Service Log 11 Customer Information",0.0004476681933738291 -" Rating: 4 Comments: ""Good support, but I'm hoping the technician can get the heating issue fixed quickly. The laundry’s been piling up!"" Source System: CRM --- ### Customer Service Log 12 Customer Information",0.00014801646466366947 -" Rating: 3 Comments: ""The advice was helpful, but the odor is still there. Hoping the technician can sort this out quickly."" Source System: CRM --- ### Customer Service Log 13 Customer Information",0.00745440972968936 -" Rating: 4 Comments: ""Service was ebicient, but I really need this issue fixed soon as the washing machine is practically unusable right now."" Source System: CRM --- ### Customer Service Log 14 Customer Information",0.0001770696835592389 -" Rating: 5 Comments: ""Great support. I’m confident the technician will figure out the issue and fix it."" Source System: CRM --- ### Customer Service Log 15 Customer Information",0.000320453749736771 -" Rating: 3 Comments: ""Support was helpful, but I’ve been dealing with this overfilling issue for weeks now. I hope the technician can fix it soon."" Source System: CRM --- ### Customer Service Log 16 Customer Information",0.00016656539810355753 -" Rating: 4 Comments: ""The troubleshooting steps were useful, but I hope the technician can fix the issue soon. I need the machine working properly."" Source System: Field Service Management --- ### Customer Service Log 17 Customer Information",0.00012659716594498605 -" Rating: 3 Comments: ""The support was helpful, but I’m concerned about the cost of fixing the motor or bearings. It’s been quite a frustrating experience."" Source System: SAP --- ### Customer Service Log 18 Customer Information",0.002657181117683649 -" Rating: 4 Comments: ""The support was helpful, but I hope the technician can resolve the vibration issue soon. It's been annoying trying to balance the machine."" Source System: Field Service Management --- ### Customer Service Log 19 Customer Information",0.0002705753722693771 -" Rating: 5 Comments: ""The troubleshooting steps were helpful, and the technician visit is scheduled quickly. I appreciate the fast response."" Source System: SAP --- ### Customer Service Log 20 Customer Information",0.00015676785551477224 -" Rating: 3 Comments: ""The support was good, but I’m concerned about the internal wiring issue. I’ve had problems with this machine since I got it."" Source System: Field Service Management --- ### Customer Service Log 21 Customer Information",0.00030394032364711165 -" Rating: 4 Comments: ""The beeping is annoying, but I appreciate the fast response. Hopefully, it’s just a simple fix."" Source System: SAP --- ### Customer Service Log 22 Customer Information",0.0008498365641571581 -" Rating: 3 Comments: ""The support was helpful, but I’ve had this problem for a while. I really hope the technician fixes it this time."" Source System: Field Service Management --- ### Customer Service Log 23 Customer Information",0.00033511361107230186 -" Rating: 4 Comments: ""Good support, but I’m getting frustrated that I can’t use the machine properly. Hope it gets fixed soon."" Source System: SAP --- ### Customer Service Log 24 Customer Information",0.00020283933554310352 -" Rating: 5 Comments: ""Great service, and I’m relieved that the issue is being handled. Looking forward to the technician's visit."" Source System: Field Service Management --- ### Customer Service Log 25 Customer Information",0.0001272695226361975 -" Rating: 4 Comments: ""I’m glad the issue is being taken seriously. Hopefully, the technician can resolve the wobbling soon."" Source System: SAP --- ### Customer Service Log 26 Customer Information",0.006668669171631336 -" Rating: 3 Comments: ""The machine isn’t draining properly, and I’m getting frustrated. I hope the technician can fix this soon."" Source System: Field Service Management --- ### Customer Service Log 27 Customer Information",0.00027618688181973994 -" Rating: 5 Comments: ""Thanks for your help. I’m glad this is being addressed quickly. I’m hopeful the technician can resolve the issue."" Source System: SAP --- ### Customer Service Log 28 Customer Information",0.00015080420416779816 -" Rating: 2 Comments: ""I’m really disappointed with the frequent breakdowns of this machine. I’ve had to call for service several times."" Source System: Field Service Management --- ### Customer Service Log 29 Customer Information",0.00022953079314902425 -" Rating: 4 Comments: ""Good support, but I’m hoping the sensor replacement fixes the problem for good."" Source System: SAP --- ### Customer Service Log 30 Customer Information",0.0003282120160292834 -" Rating: 3 Comments: ""I’m disappointed that the smell keeps coming back. I hope the technician can fix it for good."" Source System: SAP --- ### Customer Service Log 31 Customer Information",0.000546164286788553 -" Rating: 4 Comments: ""I’m glad the support team is investigating. It’s frustrating to restart the cycle every time."" Source System: Field Service Management --- ### Customer Service Log 32 Customer Information",0.00023480135132558644 -" Rating: 3 Comments: ""The support was decent, but I really need the agitator fixed. It’s making washing clothes a hassle."" Source System: SAP --- ### Customer Service Log 33 Customer Information",0.0013983179815113544 -" Rating: 4 Comments: ""The noise is getting really annoying. Hopefully, the technician can fix it soon."" Source System: Field Service Management --- ### Customer Service Log 34 Customer Information",0.00024661299539729953 -" Rating: 2 Comments: ""Very frustrating that the machine just turned ob. I’ve had enough of these power issues."" Source System: SAP --- ### Customer Service Log 35 Customer Information",0.00038764093187637627 -" Rating: 5 Comments: ""I’m glad the technician is coming soon. I need the machine to drain properly before I can use it again."" Source System: Field Service Management --- ### Customer Service Log 36 Customer Information",0.00015732910833321512 -" Rating: 4 Comments: ""I appreciate the support. I just need the temperature issue fixed quickly so I can use the machine again."" Source System: SAP --- ### Customer Service Log 37 Customer Information",0.00029154791263863444 -" Rating: 3 Comments: ""I’m frustrated that the machine isn’t working. I’m hoping the technician can fix the issue soon."" Source System: Field Service Management --- ### Customer Service Log 38 Customer Information",0.00021283984824549407 -" Rating: 4 Comments: ""The issue seems to be with the door. I hope the technician can get it fixed without much hassle."" Source System: SAP --- ### Customer Service Log 39 Customer Information",0.0011916260700672865 -" Rating: 4 Comments: ""I’m frustrated with this issue, but the support was decent. Hopefully the technician can finally fix it."" Source System: Field Service Management --- ### Customer Service Log 40 Customer Information",0.00025551579892635345 -" Rating: 5 Comments: ""Thank you for helping. I hope this issue is resolved. The machine has been a nightmare for the past week."" Source System: SAP --- ### Customer Service Log 41 Customer Information",0.00023222251911647618 -" Rating: 3 Comments: ""The noise is unbearable. I can’t stand it anymore. Please get this fixed ASAP!"" Source System: Field Service Management --- ### Customer Service Log 42 Customer Information",0.001093846163712442 -" Rating: 2 Comments: ""I am so sick of this machine. This is the third time I’ve had problems! I just want a replacement!"" Source System: SAP --- ### Customer Service Log 43 Customer Information",0.002052868250757456 -" Rating: 5 Comments: ""It’s been frustrating, but the support has been helpful. Hopefully, the technician can get this fixed."" Source System: Field Service Management --- ### Customer Service Log 44 Customer Information",0.000362074701115489 -" Rating: 4 Comments: ""I’m annoyed that the detergent isn’t dispensing correctly. I expect a better product."" Source System: SAP --- ### Customer Service Log 45 Customer Information",0.002651396906003356 -" Rating: 5 Comments: ""I’m scared to use it after that smell. Thank you for the fast response. Please fix this!"" Source System: Field Service Management --- ### Customer Service Log 46 Customer Information",0.001158195547759533 -" Rating: 1 Comments: ""This is ridiculous! The machine has been leaking for days and no one is fixing it. I’m done with this brand!"" Source System: SAP --- ### Customer Service Log 47 Customer Information",0.513248085975647 -" Rating: 3 Comments: ""I’ve had enough of these problems. If the technician doesn’t fix it soon, I’ll demand a refund."" Source System: Field Service Management --- ### Customer Service Log 48 Customer Information",0.00032178364926949143 -" Rating: 4 Comments: ""I’m glad someone’s coming to fix it, but I’ve had enough of this. I just want the machine to work properly!"" Source System: SAP --- ### Customer Service Log 49 Customer Information",0.00022851758694741875 -" Rating: 5 Comments: ""The support team was helpful, but I’m really annoyed this happened right after the warranty expired."" Source System: Field Service Management --- ### Customer Service Log 50 Customer Information",0.00014407523849513382 -" Rating: 2 Comments: ""This is the second time my machine is leaking. I’m getting tired of this brand. FIX IT."" Source System: SAP --- ### Customer Service Log 51 Customer Information",0.0002608941576909274 -" Rating: 5 Comments: ""I’m hoping this gets fixed soon. I’ve already had too many issues with this machine."" Source System: Field Service Management --- ### Customer Service Log 52 Customer Information",0.00043143669608980417 -" Rating: 3 Comments: ""The machine shouldn’t be overheating like this. It’s frustrating, especially since it’s brand new!"" Source System: SAP --- ### Customer Service Log 53 Customer Information",0.0005323936929926276 -" Rating: 1 Comments: ""I’ve had enough of this junk! It’s been a month of issues and no one’s fixing it! Get this thing replaced."" Source System: Field Service Management --- ### Customer Service Log 54 Customer Information",0.8846611380577087 -" Rating: 4 Comments: ""The odor is unbearable, but at least I have someone coming to fix it."" Source System: SAP --- ### Customer Service Log 55 Customer Information",0.0009222364169545472 -" Rating: 2 Comments: ""I don’t care about the technician visit. Just fix this machine! It’s been down too long."" Source System: Field Service Management --- ### Customer Service Log 56 Customer Information",0.00040924124186858535 -" Rating: 4 Comments: ""I’m beyond frustrated at this point. This machine is brand new and it’s already broken!"" Source System: SAP --- ### Customer Service Log 57 Customer Information",0.0005144605529494584 -" Rating: 3 Comments: ""It’s been so frustrating. I don’t know how this machine got so loud so quickly. Let’s see if the technician can fix it."" Source System: SAP --- ### Customer Service Log 58 Customer Information",0.00046848144847899675 -" Rating: 5 Comments: ""Quick and ebicient response. I hope the technician can fix it soon."" Source System: Field Service Management --- ### Customer Service Log 59 Customer Information",0.00019660884572658688 -" Rating: 4 Comments: ""The technician better sort this out. I’ve cleaned everything and it’s still broken."" Source System: SAP --- ### Customer Service Log 60 Customer Information",0.00048811433953233063 -" Rating: 2 Comments: ""How hard is it to make a washing machine that actually works? I’m sick of having to call for repairs."" Source System: Field Service Management --- ### Customer Service Log 61 Customer Information",0.00046817120164632797 -" Rating: 3 Comments: ""This is not what I expected from a high-end machine. Can you get this sorted out quickly?"" Source System: SAP --- ### Customer Service Log 62 Customer Information",0.00014980968262534589 -" Rating: 1 Comments: ""This is insane. I’ve only had this machine for a year, and it’s already broken. This brand is trash."" Source System: Field Service Management --- ### Customer Service Log 63 Customer Information",0.5792514085769653 -" Rating: 4 Comments: ""I need this fixed. I can’t abord to keep waiting for a solution."" Source System: SAP --- ### Customer Service Log 64 Customer Information",0.0003836100222542882 -" Rating: 2 Comments: ""I’m so done with this machine. I can’t trust it to work. Just get it fixed already."" Source System: Field Service Management --- ### Customer Service Log 65 Customer Information",0.0002852928882930428 -" Rating: 3 Comments: ""I’ve leveled the machine, but it’s still vibrating like crazy. Something’s clearly wrong."" Source System: SAP ",0.05251740291714668 diff --git a/examples/notebooks/hap/output/metadata.json b/examples/notebooks/hap/output/metadata.json deleted file mode 100644 index ee3fed7b9..000000000 --- a/examples/notebooks/hap/output/metadata.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "pipeline": "pipeline_id", - "job details": { - "job category": "preprocessing", - "job name": "hap", - "job type": "pure python", - "job id": "job_id", - "start_time": "2024-12-05 10:56:36", - "end_time": "2024-12-05 10:56:38", - "status": "success" - }, - "code": { - "github": "github", - "commit_hash": "12345", - "path": "path" - }, - "job_input_params": { - "model_name_or_path": "ibm-granite/granite-guardian-hap-38m", - "annotation_column": "hap_score", - "doc_text_column": "Customer Feedback", - "inference_engine": "CPU", - "max_length": 512, - "batch_size": 128, - "checkpointing": false, - "max_files": -1, - "random_samples": -1, - "files_to_use": [ - ".parquet" - ], - "num_processors": 0 - }, - "execution_stats": { - "cpus": 59.3, - "gpus": 0, - "memory": 12.22, - "object_store": 0, - "execution time, min": 0.031 - }, - "job_output_stats": { - "source_files": 1, - "source_size": 47309, - "result_files": 1, - "result_size": 40282, - "processing_time": 0.518, - "source_doc_count": 65, - "result_doc_count": 65 - }, - "source": { - "name": "/Users/aisha/Documents/GitHub/Personal/DPK/examples/notebooks/hap/input", - "type": "path" - }, - "target": { - "name": "/Users/aisha/Documents/GitHub/Personal/DPK/examples/notebooks/hap/output", - "type": "path" - } -} \ No newline at end of file diff --git a/transforms/universal/hap/python/src/hap_local_python.py b/transforms/universal/hap/python/src/hap_local_python.py index ca5beb144..6c5bcf4fe 100644 --- a/transforms/universal/hap/python/src/hap_local_python.py +++ b/transforms/universal/hap/python/src/hap_local_python.py @@ -17,9 +17,8 @@ from data_processing.utils import ParamsUtils from hap_transform_python import HAPPythonTransformConfiguration -input_folder = os.getenv('INPUT_FOLDER', os.path.abspath(os.path.join(os.path.dirname(__file__), "../test-data/input"))) -output_folder = os.getenv('OUTPUT_FOLDER', os.path.abspath(os.path.join(os.path.dirname(__file__), "../output")) -) +input_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../test-data/input")) +output_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../output")) local_conf = { "input_folder": input_folder, @@ -35,12 +34,12 @@ } hap_params = { - "model_name_or_path": os.getenv('MODEL_NAME_OR_PATH', 'ibm-granite/granite-guardian-hap-38m'), - "annotation_column": os.getenv('ANNOTATION_COLUMN', "hap_score"), - "doc_text_column": os.getenv('DOC_TEXT_COLUMN', "contents"), - "inference_engine": os.getenv('INFERENCE_ENGINE', "CPU"), - "max_length": int(os.getenv('MAX_LENGTH', 512)), - "batch_size": int(os.getenv('BATCH_SIZE', 128)), + "model_name_or_path": 'ibm-granite/granite-guardian-hap-38m', + "annotation_column": "hap_score", + "doc_text_column": "contents", + "inference_engine": "CPU", + "max_length": 512, + "batch_size": 128, } if __name__ == "__main__": @@ -50,3 +49,4 @@ launcher = PythonTransformLauncher(runtime_config=HAPPythonTransformConfiguration()) # Launch the ray actor(s) to process the input launcher.launch() + From 0a19c0c7fe77ae7e5a2692595caa038a5a26b6cb Mon Sep 17 00:00:00 2001 From: Sujee Maniyam Date: Fri, 6 Dec 2024 00:14:21 -0800 Subject: [PATCH 26/31] upgraded to 0.2.3-dev1 release Signed-off-by: Sujee Maniyam --- .../notebooks/html-processing/requirements.txt | 18 ++++++++++++++---- .../{test.ipynb => test_crawl.ipynb} | 0 .../html-processing/{test.py => test_crawl.py} | 0 3 files changed, 14 insertions(+), 4 deletions(-) rename examples/notebooks/html-processing/{test.ipynb => test_crawl.ipynb} (100%) rename examples/notebooks/html-processing/{test.py => test_crawl.py} (100%) diff --git a/examples/notebooks/html-processing/requirements.txt b/examples/notebooks/html-processing/requirements.txt index 3143df746..c43de78ca 100644 --- a/examples/notebooks/html-processing/requirements.txt +++ b/examples/notebooks/html-processing/requirements.txt @@ -1,8 +1,18 @@ -data-prep-toolkit==0.2.2.dev2 -data-prep-toolkit-transforms[html2parquet]==0.2.2.dev3 -data-prep-toolkit-transforms[web2parquet]==0.2.2.dev3 +### 0.2.2.dev3 +# data-prep-toolkit==0.2.2.dev2 +# data-prep-toolkit-transforms[html2parquet]==0.2.2.dev3 +# data-prep-toolkit-transforms[web2parquet]==0.2.2.dev3 + +### 0.2.2 +# data-prep-toolkit==0.2.2 +# data-prep-toolkit-transforms[html2parquet]==0.2.2 +# data-prep-toolkit-transforms[web2parquet]==0.2.2 + +### DPK 0.2.3-dev1 +data-prep-toolkit==0.2.3.dev0 +data-prep-toolkit-transforms[html2parquet]==0.2.3.dev1 +data-prep-toolkit-transforms[web2parquet]==0.2.3.dev1 -# data-prep-connector==0.2.2 ## HTML processing trafilatura diff --git a/examples/notebooks/html-processing/test.ipynb b/examples/notebooks/html-processing/test_crawl.ipynb similarity index 100% rename from examples/notebooks/html-processing/test.ipynb rename to examples/notebooks/html-processing/test_crawl.ipynb diff --git a/examples/notebooks/html-processing/test.py b/examples/notebooks/html-processing/test_crawl.py similarity index 100% rename from examples/notebooks/html-processing/test.py rename to examples/notebooks/html-processing/test_crawl.py From e046a315a45bf403b7ab105a28fbc011c19bf94d Mon Sep 17 00:00:00 2001 From: Aisha Mohammed Farooq Darga Date: Fri, 6 Dec 2024 11:34:14 +0530 Subject: [PATCH 27/31] clear output cells Signed-off-by: Aisha Mohammed Farooq Darga --- .../hap/generate_hap_score_csv.ipynb | 541 +----------------- 1 file changed, 18 insertions(+), 523 deletions(-) diff --git a/examples/notebooks/hap/generate_hap_score_csv.ipynb b/examples/notebooks/hap/generate_hap_score_csv.ipynb index 241dd8b24..1fcf0c8c0 100644 --- a/examples/notebooks/hap/generate_hap_score_csv.ipynb +++ b/examples/notebooks/hap/generate_hap_score_csv.ipynb @@ -63,334 +63,9 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: data-prep-connector in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (0.2.3)\n", - "Requirement already satisfied: scrapy>=2.11.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-connector) (2.12.0)\n", - "Requirement already satisfied: pydantic>=2.8.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-connector) (2.10.2)\n", - "Requirement already satisfied: tldextract>=5.1.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-connector) (5.1.3)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic>=2.8.1->data-prep-connector) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.27.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic>=2.8.1->data-prep-connector) (2.27.1)\n", - "Requirement already satisfied: typing-extensions>=4.12.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic>=2.8.1->data-prep-connector) (4.12.2)\n", - "Requirement already satisfied: Twisted>=21.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.11.0)\n", - "Requirement already satisfied: cryptography>=37.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (44.0.0)\n", - "Requirement already satisfied: cssselect>=0.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.2.0)\n", - "Requirement already satisfied: itemloaders>=1.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.3.2)\n", - "Requirement already satisfied: parsel>=1.5.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.9.1)\n", - "Requirement already satisfied: pyOpenSSL>=22.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.3.0)\n", - "Requirement already satisfied: queuelib>=1.4.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (1.7.0)\n", - "Requirement already satisfied: service-identity>=18.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.2.0)\n", - "Requirement already satisfied: w3lib>=1.17.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (2.2.1)\n", - "Requirement already satisfied: zope.interface>=5.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (7.2)\n", - "Requirement already satisfied: protego>=0.1.15 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (0.3.1)\n", - "Requirement already satisfied: itemadapter>=0.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (0.10.0)\n", - "Requirement already satisfied: packaging in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (24.2)\n", - "Requirement already satisfied: lxml>=4.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (4.9.4)\n", - "Requirement already satisfied: defusedxml>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (0.7.1)\n", - "Requirement already satisfied: PyDispatcher>=2.0.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scrapy>=2.11.2->data-prep-connector) (2.0.7)\n", - "Requirement already satisfied: idna in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (3.10)\n", - "Requirement already satisfied: requests>=2.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (2.32.3)\n", - "Requirement already satisfied: requests-file>=1.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (2.1.0)\n", - "Requirement already satisfied: filelock>=3.0.8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from tldextract>=5.1.2->data-prep-connector) (3.16.1)\n", - "Requirement already satisfied: cffi>=1.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from cryptography>=37.0.0->scrapy>=2.11.2->data-prep-connector) (1.17.1)\n", - "Requirement already satisfied: jmespath>=0.9.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from itemloaders>=1.0.1->scrapy>=2.11.2->data-prep-connector) (1.0.1)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.1.0->tldextract>=5.1.2->data-prep-connector) (3.4.0)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.1.0->tldextract>=5.1.2->data-prep-connector) (2.2.3)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests>=2.1.0->tldextract>=5.1.2->data-prep-connector) (2024.8.30)\n", - "Requirement already satisfied: attrs>=19.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from service-identity>=18.1.0->scrapy>=2.11.2->data-prep-connector) (24.2.0)\n", - "Requirement already satisfied: pyasn1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from service-identity>=18.1.0->scrapy>=2.11.2->data-prep-connector) (0.6.1)\n", - "Requirement already satisfied: pyasn1-modules in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from service-identity>=18.1.0->scrapy>=2.11.2->data-prep-connector) (0.4.1)\n", - "Requirement already satisfied: automat>=24.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (24.8.1)\n", - "Requirement already satisfied: constantly>=15.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (23.10.4)\n", - "Requirement already satisfied: hyperlink>=17.1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (21.0.0)\n", - "Requirement already satisfied: incremental>=24.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from Twisted>=21.7.0->scrapy>=2.11.2->data-prep-connector) (24.7.2)\n", - "Requirement already satisfied: setuptools in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from zope.interface>=5.1.0->scrapy>=2.11.2->data-prep-connector) (75.1.0)\n", - "Requirement already satisfied: pycparser in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from cffi>=1.12->cryptography>=37.0.0->scrapy>=2.11.2->data-prep-connector) (2.22)\n", - "Requirement already satisfied: data-prep-toolkit==0.2.2.dev1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]==0.2.2.dev1) (0.2.2.dev1)\n", - "Requirement already satisfied: numpy<1.29.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.26.4)\n", - "Requirement already satisfied: pyarrow==16.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (16.1.0)\n", - "Requirement already satisfied: boto3==1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.34.69)\n", - "Collecting argparse (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1)\n", - " Using cached argparse-1.4.0-py2.py3-none-any.whl.metadata (2.8 kB)\n", - "Requirement already satisfied: mmh3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (4.1.0)\n", - "Requirement already satisfied: psutil in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (5.9.0)\n", - "Requirement already satisfied: ray==2.36.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.36.1)\n", - "Requirement already satisfied: fastapi>=0.110.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]==0.2.2.dev1) (0.115.5)\n", - "Requirement already satisfied: pillow>=10.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]==0.2.2.dev1) (10.4.0)\n", - "Requirement already satisfied: botocore<1.35.0,>=1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.34.162)\n", - "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (1.0.1)\n", - "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (0.10.4)\n", - "Requirement already satisfied: click>=7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (8.1.7)\n", - "Requirement already satisfied: filelock in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.16.1)\n", - "Requirement already satisfied: jsonschema in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (4.23.0)\n", - "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.1.0)\n", - "Requirement already satisfied: packaging in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (24.2)\n", - "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (5.29.0)\n", - "Requirement already satisfied: pyyaml in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (6.0.2)\n", - "Requirement already satisfied: aiosignal in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.3.1)\n", - "Requirement already satisfied: frozenlist in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.5.0)\n", - "Requirement already satisfied: requests in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.32.3)\n", - "Requirement already satisfied: aiohttp>=3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.11.8)\n", - "Requirement already satisfied: aiohttp-cors in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.7.0)\n", - "Requirement already satisfied: colorful in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.5.6)\n", - "Requirement already satisfied: py-spy>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.4.0)\n", - "Requirement already satisfied: opencensus in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.11.4)\n", - "Requirement already satisfied: pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.10.2)\n", - "Requirement already satisfied: prometheus-client>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.21.0)\n", - "Requirement already satisfied: smart-open in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (7.0.5)\n", - "Requirement already satisfied: virtualenv!=20.21.1,>=20.0.24 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (20.28.0)\n", - "Requirement already satisfied: grpcio>=1.42.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.68.0)\n", - "Requirement already satisfied: memray in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.14.0)\n", - "Requirement already satisfied: starlette<0.42.0,>=0.40.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (0.41.3)\n", - "Requirement already satisfied: typing-extensions>=4.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (4.12.2)\n", - "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.4.3)\n", - "Requirement already satisfied: attrs>=17.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (24.2.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (6.1.0)\n", - "Requirement already satisfied: propcache>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.2.0)\n", - "Requirement already satisfied: yarl<2.0,>=1.17.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.18.0)\n", - "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from botocore<1.35.0,>=1.34.69->boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (2.9.0.post0)\n", - "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from botocore<1.35.0,>=1.34.69->boto3==1.34.69->data-prep-toolkit==0.2.2.dev1->data-prep-toolkit[ray]==0.2.2.dev1) (2.2.3)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.27.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,<3->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.27.1)\n", - "Requirement already satisfied: anyio<5,>=3.4.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from starlette<0.42.0,>=0.40.0->fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (4.6.2.post1)\n", - "Requirement already satisfied: distlib<1,>=0.3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.3.9)\n", - "Requirement already satisfied: platformdirs<5,>=3.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (4.3.6)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2024.10.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.35.1)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.21.0)\n", - "Requirement already satisfied: jinja2>=2.9 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.1.4)\n", - "Requirement already satisfied: rich>=11.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (13.9.4)\n", - "Requirement already satisfied: textual>=0.41.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.87.1)\n", - "Requirement already satisfied: opencensus-context>=0.1.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.1.3)\n", - "Requirement already satisfied: six~=1.16 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.16.0)\n", - "Requirement already satisfied: google-api-core<3.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.23.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.4.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.10)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests->ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2024.8.30)\n", - "Requirement already satisfied: wrapt in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from smart-open->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.17.0)\n", - "Requirement already satisfied: sniffio>=1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from anyio<5,>=3.4.0->starlette<0.42.0,>=0.40.0->fastapi>=0.110.2->data-prep-toolkit[ray]==0.2.2.dev1) (1.3.1)\n", - "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.66.0)\n", - "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.25.0)\n", - "Requirement already satisfied: google-auth<3.0.dev0,>=2.14.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.36.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jinja2>=2.9->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.1.5)\n", - "Requirement already satisfied: markdown-it-py>=2.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from rich>=11.2.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (3.0.0)\n", - "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from rich>=11.2.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.18.0)\n", - "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (5.5.0)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.4.1)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (4.9)\n", - "Requirement already satisfied: mdurl~=0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich>=11.2.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.1.2)\n", - "Requirement already satisfied: linkify-it-py<3,>=1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (2.0.3)\n", - "Requirement already satisfied: mdit-py-plugins in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.4.2)\n", - "Requirement already satisfied: uc-micro-py in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (1.0.3)\n", - "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]==0.2.2.dev1) (0.6.1)\n", - "Using cached argparse-1.4.0-py2.py3-none-any.whl (23 kB)\n", - "Installing collected packages: argparse\n", - "Successfully installed argparse-1.4.0\n", - "Requirement already satisfied: data-prep-toolkit-transforms==0.2.2.dev1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.2.dev1)\n", - "Requirement already satisfied: data-prep-toolkit>=0.2.2.dev1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.2.dev1)\n", - "Requirement already satisfied: bs4==0.0.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.0.2)\n", - "Requirement already satisfied: transformers==4.38.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.38.2)\n", - "Requirement already satisfied: parameterized in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.0)\n", - "Requirement already satisfied: pandas in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.2.2)\n", - "Requirement already satisfied: docling-core==1.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0)\n", - "Requirement already satisfied: llama-index-core<0.12.0,>=0.11.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.11.23)\n", - "Requirement already satisfied: fasttext==0.9.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.2)\n", - "Requirement already satisfied: langcodes==3.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3.0)\n", - "Requirement already satisfied: huggingface-hub<1.0.0,>=0.21.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.26.3)\n", - "Requirement already satisfied: numpy==1.26.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.26.4)\n", - "Requirement already satisfied: sentence-transformers==3.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.0.1)\n", - "Requirement already satisfied: docling-ibm-models==1.1.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.7)\n", - "Requirement already satisfied: deepsearch-glm==0.21.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.0)\n", - "Requirement already satisfied: docling==1.11.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.11.0)\n", - "Requirement already satisfied: filetype<2.0.0,>=1.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.2.0)\n", - "Requirement already satisfied: nltk==3.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.9.1)\n", - "Requirement already satisfied: torch==2.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.4.1)\n", - "Requirement already satisfied: mmh3>=4.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.1.0)\n", - "Requirement already satisfied: xxhash==3.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.4.1)\n", - "Requirement already satisfied: duckdb>=0.10.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.3)\n", - "Requirement already satisfied: networkx==3.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3)\n", - "Requirement already satisfied: colorlog==6.8.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (6.8.2)\n", - "Requirement already satisfied: func-timeout==4.3.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.3.5)\n", - "Requirement already satisfied: emerge-viz==2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.0)\n", - "Requirement already satisfied: beautifulsoup4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from bs4==0.0.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.12.3)\n", - "Requirement already satisfied: pyarrow==16.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (16.1.0)\n", - "Requirement already satisfied: boto3==1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.34.69)\n", - "Collecting argparse (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1)\n", - " Using cached argparse-1.4.0-py2.py3-none-any.whl.metadata (2.8 kB)\n", - "Requirement already satisfied: psutil in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.9.0)\n", - "Requirement already satisfied: docutils!=0.21 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.2)\n", - "Requirement already satisfied: matplotlib<4.0.0,>=3.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.9.2)\n", - "Requirement already satisfied: netwulf<0.2.0,>=0.1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.5)\n", - "Requirement already satisfied: numerize<0.13,>=0.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.12)\n", - "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.1)\n", - "Requirement already satisfied: rich<14.0.0,>=13.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (13.9.4)\n", - "Requirement already satisfied: tabulate>=0.8.9 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.0)\n", - "Requirement already satisfied: tqdm<5.0.0,>=4.64.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.67.1)\n", - "Requirement already satisfied: certifi>=2024.7.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.8.30)\n", - "Requirement already satisfied: docling-parse<2.0.0,>=1.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.6.2)\n", - "Requirement already satisfied: easyocr<2.0,>=1.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.7.2)\n", - "Requirement already satisfied: pydantic<3.0.0,>=2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.10.2)\n", - "Requirement already satisfied: pydantic-settings<3.0.0,>=2.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.6.1)\n", - "Requirement already satisfied: pypdfium2<5.0.0,>=4.30.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.30.0)\n", - "Requirement already satisfied: requests<3.0.0,>=2.32.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.32.3)\n", - "Requirement already satisfied: rtree<2.0.0,>=1.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0)\n", - "Requirement already satisfied: scipy<2.0.0,>=1.14.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.14.1)\n", - "Requirement already satisfied: json-schema-for-humans<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.4)\n", - "Requirement already satisfied: jsonref<2.0.0,>=1.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.0)\n", - "Requirement already satisfied: jsonschema<5.0.0,>=4.16.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.23.0)\n", - "Requirement already satisfied: pyproject-toml<0.0.11,>=0.0.10 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.0.10)\n", - "Requirement already satisfied: Pillow<11.0.0,>=10.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (10.4.0)\n", - "Requirement already satisfied: jsonlines<4.0.0,>=3.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.0)\n", - "Requirement already satisfied: lxml<5.0.0,>=4.9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.9.4)\n", - "Requirement already satisfied: mean_average_precision<2022.0.0.0,>=2021.4.26.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2021.4.26.0)\n", - "Requirement already satisfied: onnxruntime<2.0.0,>=1.16.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.20.1)\n", - "Requirement already satisfied: opencv-python-headless<5.0.0.0,>=4.9.0.80 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.10.0.84)\n", - "Requirement already satisfied: torchvision<1,>=0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.19.1)\n", - "Requirement already satisfied: wheel in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.44.0)\n", - "Requirement already satisfied: autopep8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.3.1)\n", - "Requirement already satisfied: coloredlogs in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (15.0.1)\n", - "Requirement already satisfied: interrogate in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.7.0)\n", - "Requirement already satisfied: scikit-learn in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.5.2)\n", - "Requirement already satisfied: prettytable in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.12.0)\n", - "Requirement already satisfied: py in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.11.0)\n", - "Requirement already satisfied: pycodestyle in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.12.1)\n", - "Requirement already satisfied: pylint in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3.1)\n", - "Requirement already satisfied: pyparsing in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.2.0)\n", - "Requirement already satisfied: python-louvain in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.16)\n", - "Requirement already satisfied: PyYAML in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (6.0.2)\n", - "Requirement already satisfied: PyDriller in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.7)\n", - "Requirement already satisfied: pyperclip in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.9.0)\n", - "Requirement already satisfied: pybind11>=2.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fasttext==0.9.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.13.6)\n", - "Requirement already satisfied: setuptools>=0.7.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fasttext==0.9.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (75.1.0)\n", - "Requirement already satisfied: click in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (8.1.7)\n", - "Requirement already satisfied: joblib in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.4.2)\n", - "Requirement already satisfied: regex>=2021.8.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from nltk==3.9.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.11.6)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.9.0.post0)\n", - "Requirement already satisfied: pytz>=2020.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.2)\n", - "Requirement already satisfied: tzdata>=2022.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.2)\n", - "Requirement already satisfied: filelock in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.16.1)\n", - "Requirement already satisfied: typing-extensions>=4.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.12.2)\n", - "Requirement already satisfied: sympy in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.13.3)\n", - "Requirement already satisfied: jinja2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.4)\n", - "Requirement already satisfied: fsspec in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.10.0)\n", - "Requirement already satisfied: packaging>=20.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (24.2)\n", - "Requirement already satisfied: tokenizers<0.19,>=0.14 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.15.2)\n", - "Requirement already satisfied: safetensors>=0.4.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from transformers==4.38.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.5)\n", - "Requirement already satisfied: botocore<1.35.0,>=1.34.69 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.34.162)\n", - "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.1)\n", - "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from boto3==1.34.69->data-prep-toolkit>=0.2.2.dev1->data-prep-toolkit-transforms==0.2.2.dev1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.10.4)\n", - "Requirement already satisfied: ray==2.36.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.36.1)\n", - "Requirement already satisfied: fastapi>=0.110.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.115.5)\n", - "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.1.0)\n", - "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.29.0)\n", - "Requirement already satisfied: aiosignal in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.1)\n", - "Requirement already satisfied: frozenlist in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray==2.36.1->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.5.0)\n", - "Requirement already satisfied: aiohttp>=3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.11.8)\n", - "Requirement already satisfied: aiohttp-cors in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.7.0)\n", - "Requirement already satisfied: colorful in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.5.6)\n", - "Requirement already satisfied: py-spy>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.0)\n", - "Requirement already satisfied: opencensus in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.11.4)\n", - "Requirement already satisfied: prometheus-client>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.0)\n", - "Requirement already satisfied: smart-open in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (7.0.5)\n", - "Requirement already satisfied: virtualenv!=20.21.1,>=20.0.24 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (20.28.0)\n", - "Requirement already satisfied: grpcio>=1.42.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.68.0)\n", - "Requirement already satisfied: memray in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.14.0)\n", - "Requirement already satisfied: SQLAlchemy>=1.4.49 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.36)\n", - "Requirement already satisfied: dataclasses-json in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.6.7)\n", - "Requirement already satisfied: deprecated>=1.2.9.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.2.15)\n", - "Requirement already satisfied: dirtyjson<2.0.0,>=1.0.8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.8)\n", - "Requirement already satisfied: httpx in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.28.0)\n", - "Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.6.0)\n", - "Requirement already satisfied: tenacity!=8.4.0,<9.0.0,>=8.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (8.5.0)\n", - "Requirement already satisfied: tiktoken>=0.3.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.8.0)\n", - "Requirement already satisfied: typing-inspect>=0.8.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.9.0)\n", - "Requirement already satisfied: wrapt in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.17.0)\n", - "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.4.3)\n", - "Requirement already satisfied: attrs>=17.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (24.2.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (6.1.0)\n", - "Requirement already satisfied: propcache>=0.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.0)\n", - "Requirement already satisfied: yarl<2.0,>=1.17.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from aiohttp>=3.7->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.18.0)\n", - "Requirement already satisfied: scikit-image in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.24.0)\n", - "Requirement already satisfied: python-bidi in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.6.3)\n", - "Requirement already satisfied: Shapely in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.6)\n", - "Requirement already satisfied: pyclipper in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0.post6)\n", - "Requirement already satisfied: ninja in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.11.1.2)\n", - "Requirement already satisfied: starlette<0.42.0,>=0.40.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from fastapi>=0.110.2->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.41.3)\n", - "Requirement already satisfied: MarkupSafe<3.0.0,>=2.1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.1.5)\n", - "Requirement already satisfied: Pygments<3.0.0,>=2.18.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.18.0)\n", - "Requirement already satisfied: htmlmin2<0.2.0,>=0.1.13 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.13)\n", - "Requirement already satisfied: markdown2<3.0.0,>=2.5.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from json-schema-for-humans<2.0.0,>=1.0.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.5.1)\n", - "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from dataclasses-json->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.23.1)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.16.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.10.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.16.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.35.1)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from jsonschema<5.0.0,>=4.16.0->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.21.0)\n", - "Requirement already satisfied: contourpy>=1.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.1)\n", - "Requirement already satisfied: cycler>=0.10 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.12.1)\n", - "Requirement already satisfied: fonttools>=4.22.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.55.0)\n", - "Requirement already satisfied: kiwisolver>=1.3.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from matplotlib<4.0.0,>=3.7.1->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.4.7)\n", - "Requirement already satisfied: simplejson>=3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from netwulf<0.2.0,>=0.1.5->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.19.3)\n", - "Requirement already satisfied: flatbuffers in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from onnxruntime<2.0.0,>=1.16.2->docling-ibm-models==1.1.7->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (24.3.25)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.27.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.0.0->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.27.1)\n", - "Requirement already satisfied: toml in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pyproject-toml<0.0.11,>=0.0.10->docling-core==1.3.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.10.2)\n", - "Requirement already satisfied: six>=1.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.16.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests<3.0.0,>=2.32.3->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.4.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests<3.0.0,>=2.32.3->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.10)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from requests<3.0.0,>=2.32.3->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.2.3)\n", - "Requirement already satisfied: markdown-it-py>=2.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from rich<14.0.0,>=13.7.0->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.0.0)\n", - "Requirement already satisfied: greenlet!=0.4.17 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.1)\n", - "Requirement already satisfied: mypy-extensions>=0.3.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from typing-inspect>=0.8.0->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.0)\n", - "Requirement already satisfied: soupsieve>1.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from beautifulsoup4->bs4==0.0.2->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.6)\n", - "Requirement already satisfied: humanfriendly>=9.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from coloredlogs->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (10.0)\n", - "Requirement already satisfied: anyio in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.6.2.post1)\n", - "Requirement already satisfied: httpcore==1.* in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.7)\n", - "Requirement already satisfied: h11<0.15,>=0.13 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from httpcore==1.*->httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.14.0)\n", - "Requirement already satisfied: colorama in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from interrogate->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.6)\n", - "Requirement already satisfied: wcwidth in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from prettytable->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.2.13)\n", - "Requirement already satisfied: gitpython in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.1.43)\n", - "Requirement already satisfied: types-pytz in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.2.0.20241003)\n", - "Requirement already satisfied: lizard in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.17.13)\n", - "Requirement already satisfied: platformdirs>=2.2.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.3.6)\n", - "Requirement already satisfied: astroid<=3.4.0-dev0,>=3.3.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.3.5)\n", - "Requirement already satisfied: isort!=5.13.0,<6,>=4.2.5 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.13.2)\n", - "Requirement already satisfied: mccabe<0.8,>=0.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.7.0)\n", - "Requirement already satisfied: tomlkit>=0.10.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.13.2)\n", - "Requirement already satisfied: dill>=0.3.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pylint->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.3.9)\n", - "Requirement already satisfied: threadpoolctl>=3.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-learn->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (3.5.0)\n", - "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from sympy->torch==2.4.1->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.0)\n", - "Requirement already satisfied: mdurl~=0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich<14.0.0,>=13.7.0->deepsearch-glm==0.21.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.2)\n", - "Requirement already satisfied: sniffio>=1.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from anyio->httpx->llama-index-core<0.12.0,>=0.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.3.1)\n", - "Requirement already satisfied: distlib<1,>=0.3.7 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.3.9)\n", - "Requirement already satisfied: gitdb<5,>=4.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from gitpython->PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.0.11)\n", - "Requirement already satisfied: textual>=0.41.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.87.1)\n", - "Requirement already satisfied: opencensus-context>=0.1.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.1.3)\n", - "Requirement already satisfied: google-api-core<3.0.0,>=1.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.23.0)\n", - "Requirement already satisfied: imageio>=2.33 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-image->easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.36.1)\n", - "Requirement already satisfied: tifffile>=2022.8.12 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-image->easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2024.9.20)\n", - "Requirement already satisfied: lazy-loader>=0.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from scikit-image->easyocr<2.0,>=1.7->docling==1.11.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4)\n", - "Requirement already satisfied: smmap<6,>=3.0.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from gitdb<5,>=4.0.1->gitpython->PyDriller->emerge-viz==2.0.0->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.0.1)\n", - "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.66.0)\n", - "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.25.0)\n", - "Requirement already satisfied: google-auth<3.0.dev0,>=2.14.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.36.0)\n", - "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (5.5.0)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.1)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (4.9)\n", - "Requirement already satisfied: linkify-it-py<3,>=1 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (2.0.3)\n", - "Requirement already satisfied: mdit-py-plugins in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.4.2)\n", - "Requirement already satisfied: uc-micro-py in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from linkify-it-py<3,>=1->markdown-it-py[linkify,plugins]>=2.1.0->textual>=0.41.0->memray->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (1.0.3)\n", - "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /opt/anaconda3/envs/data-prep-kit/lib/python3.11/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.dev0,>=2.14.1->google-api-core<3.0.0,>=1.0.0->opencensus->ray[default]==2.36.1; extra == \"ray\"->data-prep-toolkit[ray]>=0.2.2.dev1; extra == \"ray\"->data-prep-toolkit-transforms[all,ray]==0.2.2.dev1) (0.6.1)\n", - "Using cached argparse-1.4.0-py2.py3-none-any.whl (23 kB)\n", - "Installing collected packages: argparse\n", - "Successfully installed argparse-1.4.0\n", - "zsh:1: 2.2.2, not found\n" - ] - } - ], + "outputs": [], "source": [ "! pip install data-prep-connector\n", "! pip install 'data-prep-toolkit[ray]==0.2.2.dev1'\n", @@ -407,18 +82,9 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[nltk_data] Downloading package punkt_tab to /Users/aisha/nltk_data...\n", - "[nltk_data] Package punkt_tab is already up-to-date!\n" - ] - } - ], + "outputs": [], "source": [ "import os\n", "import sys\n", @@ -450,17 +116,9 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "./input-parquet\n" - ] - } - ], + "outputs": [], "source": [ "\n", "# create parameters\n", @@ -502,17 +160,9 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Deleted file: ./input-csv/customer_feedback_file.csv\n" - ] - } - ], + "outputs": [], "source": [ "#Clear the existing input-parquet folder\n", "if os.path.exists(input_csv):\n", @@ -531,17 +181,9 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using CSV file: ./input-csv/customer_feedback_file.csv\n" - ] - } - ], + "outputs": [], "source": [ "csv_files = [f for f in os.listdir(input_csv) if f.endswith(\".csv\")]\n", "\n", @@ -564,17 +206,9 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CSV file converted to Parquet format at: ./input-parquet/customer-feedback.parquet\n" - ] - } - ], + "outputs": [], "source": [ "parquet_file_path = os.path.join(input_folder, \"customer-feedback.parquet\")\n", "df = pd.read_csv(csv_file_path)\n", @@ -592,28 +226,9 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "10:55:04 INFO - hap params are {'model_name_or_path': 'ibm-granite/granite-guardian-hap-38m', 'annotation_column': 'hap_score', 'doc_text_column': 'Customer Feedback', 'inference_engine': 'CPU', 'max_length': 512, 'batch_size': 128} \n", - "10:55:04 INFO - pipeline id pipeline_id\n", - "10:55:04 INFO - code location {'github': 'github', 'commit_hash': '12345', 'path': 'path'}\n", - "10:55:04 INFO - data factory data_ is using local data access: input_folder - ./input-parquet output_folder - ./output\n", - "10:55:04 INFO - data factory data_ max_files -1, n_sample -1\n", - "10:55:04 INFO - data factory data_ Not using data sets, checkpointing False, max files -1, random samples -1, files to use ['.parquet'], files to checkpoint ['.parquet']\n", - "10:55:04 INFO - orchestrator hap started at 2024-12-06 10:55:04\n", - "10:55:04 INFO - Number of files is 1, source profile {'max_file_size': 0.04511737823486328, 'min_file_size': 0.04511737823486328, 'total_file_size': 0.04511737823486328}\n", - "10:55:06 INFO - Completed 1 files (100.0%) in 0.008 min\n", - "10:55:06 INFO - Done processing 1 files, waiting for flush() completion.\n", - "10:55:06 INFO - done flushing in 0.0 sec\n", - "10:55:06 INFO - Completed execution in 0.043 min, execution result 0\n" - ] - } - ], + "outputs": [], "source": [ "%%capture\n", "sys.argv = ParamsUtils.dict_to_req(d=params)\n", @@ -623,20 +238,9 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[]" - ] - }, - "execution_count": 53, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "#The specified folder will include the transformed parquet files.\n", "\n", @@ -711,118 +315,9 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Deleted old CSV file: ./output/customer-feedback.csv\n", - "Renamed Parquet file to: ./output/customer-feedback.parquet\n", - "Transformed CSV file saved at: ./output/customer-feedback.csv\n", - "\n", - "Selected Columns (Customer Feedback and HAP Score):\n" - ] - }, - { - "data": { - "text/html": [ - "

\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Customer Feedbackhap_score
0Rating: 4 Comments: \"Service was prompt, but ...0.000195
1Rating: 5 Comments: \"Great help from Peter! H...0.000153
2Rating: 3 Comments: \"The service was quick, b...0.000169
3Rating: 5 Comments: \"Excellent service and ad...0.000158
4Rating: 2 Comments: \"I’m really frustrated. T...0.000875
5Rating: 4 Comments: \"The service was helpful,...0.015778
6Rating: 3 Comments: \"The support was helpful,...0.000221
7Rating: 5 Comments: \"The service was very qui...0.000179
8Rating: 5 Comments: \"The support was excellen...0.000236
9Rating: 2 Comments: \"I’m frustrated with this...0.000448
\n", - "
" - ], - "text/plain": [ - " Customer Feedback hap_score\n", - "0 Rating: 4 Comments: \"Service was prompt, but ... 0.000195\n", - "1 Rating: 5 Comments: \"Great help from Peter! H... 0.000153\n", - "2 Rating: 3 Comments: \"The service was quick, b... 0.000169\n", - "3 Rating: 5 Comments: \"Excellent service and ad... 0.000158\n", - "4 Rating: 2 Comments: \"I’m really frustrated. T... 0.000875\n", - "5 Rating: 4 Comments: \"The service was helpful,... 0.015778\n", - "6 Rating: 3 Comments: \"The support was helpful,... 0.000221\n", - "7 Rating: 5 Comments: \"The service was very qui... 0.000179\n", - "8 Rating: 5 Comments: \"The support was excellen... 0.000236\n", - "9 Rating: 2 Comments: \"I’m frustrated with this... 0.000448" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# Locate transformed Parquet files in the output folder\n", "output_parquet_files = [f for f in os.listdir(output_folder) if f.endswith(\".parquet\")]\n", From 31b7d4d9d8accbb9c54ce927179f7edaeeb25a1d Mon Sep 17 00:00:00 2001 From: Aisha Mohammed Farooq Darga Date: Fri, 6 Dec 2024 11:37:28 +0530 Subject: [PATCH 28/31] Remove unnecessary files Signed-off-by: Aisha Mohammed Farooq Darga --- .../universal/hap/python/src/hap_config.py | 23 ------------------- .../hap/python/src/hap_local_python.py | 7 +++--- 2 files changed, 4 insertions(+), 26 deletions(-) delete mode 100644 transforms/universal/hap/python/src/hap_config.py diff --git a/transforms/universal/hap/python/src/hap_config.py b/transforms/universal/hap/python/src/hap_config.py deleted file mode 100644 index 3dc3568c4..000000000 --- a/transforms/universal/hap/python/src/hap_config.py +++ /dev/null @@ -1,23 +0,0 @@ -# hap_config.py -import os - -def get_hap_params(custom_params=None): - """ - Returns HAP parameters. - If `custom_params` are provided (e.g., from a notebook), they override the defaults. - """ - default_hap_params = { - "model_name_or_path": "ibm-granite/granite-guardian-hap-38m", - "annotation_column": "hap_score", - "doc_text_column": "contents", # Default column for documents - "inference_engine": "CPU", - "max_length": 512, - "batch_size": 128, - } - - # Merge custom_params into defaults, if provided - if custom_params: - default_hap_params.update(custom_params) - - print(f"Final HAP Parameters: {default_hap_params}") # Debugging log - return default_hap_params diff --git a/transforms/universal/hap/python/src/hap_local_python.py b/transforms/universal/hap/python/src/hap_local_python.py index 6c5bcf4fe..8e79dc583 100644 --- a/transforms/universal/hap/python/src/hap_local_python.py +++ b/transforms/universal/hap/python/src/hap_local_python.py @@ -17,9 +17,9 @@ from data_processing.utils import ParamsUtils from hap_transform_python import HAPPythonTransformConfiguration +# create parameters input_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../test-data/input")) output_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "../output")) - local_conf = { "input_folder": input_folder, "output_folder": output_folder, @@ -33,6 +33,7 @@ "runtime_code_location": ParamsUtils.convert_to_ast(code_location), } + hap_params = { "model_name_or_path": 'ibm-granite/granite-guardian-hap-38m', "annotation_column": "hap_score", @@ -42,11 +43,11 @@ "batch_size": 128, } + if __name__ == "__main__": # Set the simulated command line args sys.argv = ParamsUtils.dict_to_req(d=params | hap_params) # create launcher launcher = PythonTransformLauncher(runtime_config=HAPPythonTransformConfiguration()) # Launch the ray actor(s) to process the input - launcher.launch() - + launcher.launch() \ No newline at end of file From 6b58de43b50cecd34c94769abe3025007856935b Mon Sep 17 00:00:00 2001 From: Aisha Mohammed Farooq Darga Date: Fri, 6 Dec 2024 11:39:56 +0530 Subject: [PATCH 29/31] Remove unnecessary files Signed-off-by: Aisha Mohammed Farooq Darga --- examples/notebooks/hap/requirements.txt | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 examples/notebooks/hap/requirements.txt diff --git a/examples/notebooks/hap/requirements.txt b/examples/notebooks/hap/requirements.txt deleted file mode 100644 index 079767b7a..000000000 --- a/examples/notebooks/hap/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -nltk==3.9.1 -transformers==4.38.2 -torch>=2.2.2,<=2.4.1 -pandas==2.2.2 From 512a4e322a2e0477f2e85c02d0ea73ba46a7ce10 Mon Sep 17 00:00:00 2001 From: Aisha Mohammed Farooq Darga Date: Fri, 6 Dec 2024 21:38:11 +0530 Subject: [PATCH 30/31] Fixed variable mismatch Signed-off-by: Aisha Mohammed Farooq Darga --- examples/notebooks/hap/generate_hap_score_csv.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/notebooks/hap/generate_hap_score_csv.ipynb b/examples/notebooks/hap/generate_hap_score_csv.ipynb index 1fcf0c8c0..121b49ac0 100644 --- a/examples/notebooks/hap/generate_hap_score_csv.ipynb +++ b/examples/notebooks/hap/generate_hap_score_csv.ipynb @@ -165,9 +165,9 @@ "outputs": [], "source": [ "#Clear the existing input-parquet folder\n", - "if os.path.exists(input_csv):\n", - " for file_name in os.listdir(input_csv):\n", - " file_path = os.path.join(input_csv, file_name)\n", + "if os.path.exists(input_folder):\n", + " for file_name in os.listdir(input_folder):\n", + " file_path = os.path.join(input_folder, file_name)\n", " try:\n", " os.remove(file_path)\n", " print(f\"Deleted file: {file_path}\")\n", From 06a2936d81b91d34c97cfd88dfed18252a3c1266 Mon Sep 17 00:00:00 2001 From: Aisha Mohammed Farooq Darga Date: Sat, 7 Dec 2024 02:38:36 +0530 Subject: [PATCH 31/31] Removed parquet folder and managed dependencies Signed-off-by: Aisha Mohammed Farooq Darga --- .../notebooks/hap/generate_hap_score_csv.ipynb | 14 ++++++-------- .../hap/input-parquet/customer-feedback.parquet | Bin 47309 -> 0 bytes 2 files changed, 6 insertions(+), 8 deletions(-) delete mode 100644 examples/notebooks/hap/input-parquet/customer-feedback.parquet diff --git a/examples/notebooks/hap/generate_hap_score_csv.ipynb b/examples/notebooks/hap/generate_hap_score_csv.ipynb index 121b49ac0..a42cf0ce2 100644 --- a/examples/notebooks/hap/generate_hap_score_csv.ipynb +++ b/examples/notebooks/hap/generate_hap_score_csv.ipynb @@ -67,10 +67,8 @@ "metadata": {}, "outputs": [], "source": [ - "! pip install data-prep-connector\n", - "! pip install 'data-prep-toolkit[ray]==0.2.2.dev1'\n", - "! pip install 'data-prep-toolkit-transforms[ray,all]==0.2.2.dev1'\n", - "! pip install nltk==3.9.1 transformers==4.38.2 torch>=2.2.2,<=2.4.1 pandas==2.2.2" + "! pip install 'data-prep-toolkit[ray]==0.2.2'\n", + "! pip install 'data-prep-toolkit-transforms[all]==0.2.2'" ] }, { @@ -164,7 +162,7 @@ "metadata": {}, "outputs": [], "source": [ - "#Clear the existing input-parquet folder\n", + "# Clear the existing input-parquet folder\n", "if os.path.exists(input_folder):\n", " for file_name in os.listdir(input_folder):\n", " file_path = os.path.join(input_folder, file_name)\n", @@ -174,9 +172,9 @@ " except Exception as e:\n", " print(f\"Failed to delete {file_path}: {e}\")\n", "else:\n", - " os.makedirs(input_csv)\n", - " os.makedirs(input_folder)\n", - " print(f\"Created folder: {input_csv}\")" + " os.makedirs(input_csv, exist_ok=True) # Updated line\n", + " os.makedirs(input_folder, exist_ok=True) # Updated line\n", + " print(f\"Created folder: {input_csv}\")\n" ] }, { diff --git a/examples/notebooks/hap/input-parquet/customer-feedback.parquet b/examples/notebooks/hap/input-parquet/customer-feedback.parquet deleted file mode 100644 index 1d8bf2e6468b5457e0651836cc448cd25a57a394..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 47309 zcmeEudw3Jo*7xjohR!6D-DH{!Y1)RF(v+sQLz6bW38g7bp_LYBxdzZ?|Wg8o>S0sd-?%I!)ZAPdcMB<2q(OVwLi!g7}|x62<> z82GChzCIN^>WCs1cX^#;h-E%hij9&g$<*(#%~%prOjym%d} zL}F1igLVh|ioNIw7M{IG_X}#yZj*y?7=tNdJsOiYRS7_6>LTG@T{`;QwgFl5>iu#^ z7Cb$2$X3lZ#RLB0<&-rKMgwj<%OC1Qj}gtbhs}b=*DFVSREg0O^g~Z z%39=@9Pq=~hr``4_ME#s4;tr#DT^yhQNUOyN806x*h9nZVvpc~?g|azXpGraym$7d zW~(iZUbRi2g|kBLO{g7B5>(BM#k!PU&p=tT-`gd_*tc~nA+Ms4*-c6)QrFlra}kMZlV==XGMI%yl_0! zsp>sU{D`WDOoVA2GF^{~)a9i&jWUhOE++~!e`u8%tE%@$*WlYzk55U3Un>ww%UHhr zg4zNzUbV7)sH@VtZk4ZY+u72MS9Ux*yrgE!x~}@dn_9MByKwiyoeS>2G0<@L!O)|( zgkJdKr)9P90QfKex%6xko9QA(=`+o`$?<3`98@Ag3uu8Ug_`Sky; zU71B-DhN{w0YVSI2m=1TRcrc$oMyTxX*ph_3ewv^n$sFFDx+3l$(iQ*#lq?bO4fc`yS{AZb#eKo zhZk=;nHZ?tFxYq7w#Od0_n*&fpYX=qEK?f^k;15tAq zh?oBk;@1a2T>6Omug5`bc^bssLmCvQ6Y^`=R}p{b;Nou^!K8_(N&=f4Am; zZqEL{XyRWg)lrM-L;NqLdQjIj8oHvWC`4K~finvpy59!&Wf@O(Y*cljE;1${{LEa9gK}P9(32PBPmj@bHPO z0u-q?2~aXEC}<2P5el`6=yqxmnjv(l07VKYy^^1xW&lEwB7jiJ=OJV%3aAi8>Ppq-g}YnTGZ^eMTS)8IjTWmMTOC0H}OU15n9%1g)UuU|-#&6tRqjaa&Zk2_eNC zO=1s8zU+sWZ&a*L@;_~V%Q&47}PlYRz0&pegdXz~oj0YOJ;qwAt60t~& z0I^iH6K3}o&p*tPlEf_4Ed;3q#fPf*ryy5J_i4-`1p%|1cS1g=2%e>a772i$k~71W zN`#f3>PHDQvjBP;kfm-Fx71yh!raWl$VGFeBbO1{4A7hY|qhlx9@(^1L75g-G@13>2Di_z>Nc$8sV^LtdQ#D} z{V{h={&0K0GRKK zzyyt0udO-!Rzv>vj{IwL)xGckwy}R{#qOs}52Q{r%}D~Hw^QhEzy6IvWbqeN8vR@6 zQLy*}he4R$265x}u(ka1j(YvcH>fo7w_9|TWczX08uTB*<#*l%am$+^JfFh8VbXPz zU;h9XEl1#^XFmkd3EM4M@fCi$Tc(lHqEDD(yXu^e*V(SW5e;mzJ$6SSd>`PZ_I@TI7-We>GC3>Ee7i#RwT>_M`8i6 zG)2PnR-vlGSyE<4`NtbdoGxJoc*BCbM+wE1BB2eOYSoSM`lH@(kucdURFyeODs1q` zvQno|=Z|>1z!{L?LDRxf@VmgxP@;^zproY4A;4V~l`iU{S@jWr4?Jyh7x+w_@SMhJ zLU~C!lXtwf%HdRf^gy^*mbl29_E1@1G0;@C1S`wyZ_z z1+OQvs7Pp;Cpgf8dklg5c4WoQNnyr zm$^!rY19w4`kJ2+>9mVoWQjv;FhcWuol5akVp_{tDf%L4QmOca?JzZ`@eK4z!yPHM zFIlIfvIH%lBx+?XD6A6lFM*+-FI2*4I;c0y4wp-4ReaPqW;W@L&s#v2mNqURou&3~ z9M6$;VdkW{*)5bf9S$l&yUI#(2B>Ite~M=U4Ua8;n)$K59%iXc0UtL2vpyI4R#`lS zuBa?){@!s7(NcF`irYt*S5&(9Q_nPeTfZRa(h{fr2Gb0}onT60FkDmILMe>2!zRB+ zmN^^CXi%g1r76(C#5}?&A1QLHI}3Q^@kp$w{7?!}!sHh@oUW3j8ypLF7k}cMpTfl1 zsxnuF>+ibVb-$#z_rW*qtg6?I=aFr()(cbRaRo5w6+&A)6jfrnC9UVCpl|6?SNUKb z^{sAMT{aRI(awsh;wQ{={Gea0zJ@nR!inz8b5vEjgvnsQW47xX&#o&3jYw@jhdnucpb+{_aW*_vdh4tr+0GbHJ?)?Yrs&tg0 z{VY7S6g0q#wr^{%*_mQHUgU6<3sb`p|I%5VL5fv(#-QBmTsg;v3AV zuBxmMdFchBdS1m;&Ikr?(sE%9t ziEVlqb7c{AM@=1aze$!l8h<6>&x^!5bYj6aDjNn{m;#Wn%Id__`dqrg>1sTiDsV5a zsQ~41xwVAD{WsPlcLo|0)i0&M%E)0$%gV|VS&j3RoVu+ksGzK(3T#RX7}2gCm~&=N z^)0Xz4^L#vD;zFq9jeYu9p>4}s*+Nrb9k-D*Ks6K;66z^N-9Np?F9Gurky0|aG^f7 ztkUUp>r9)guSV3Zq|3>#r0I@uq+fC0UGgVGSg!ise-W9nf$esU7{8>oYWpkO?lRjQ z|EHE6EW!UIruk&fADBkp*5&W+hAIB9ux8&G{H>P!pWE`EpP<5;|M?#M3yT&v(dUr zN>48@*bMUA;{?b}bKoxeA16W}m#n!0r03U{K+gVZJIDb44anO%K7uDre=a2xolU=W zavpqib7eosPp-NJp)Mv-~FrH$CE0Ur%TI)2B;5W^Hk(j4A!C?ip9WEBAqopw@b-Dt&0GEZ^n=jC8-0L7 zqVV=WI1Z@L5ApxbI7AtQcJR^ESBAp=s3Hl>%K$X}F+Xfceei^E9C91#lbErz5cDIx zxm|${1Z4D8HuNb9ao`yBxX~Yi9YCSRPxVFO!6Nt=f)WA*3cQOHpUKP$U^)@GPVsk! zgmC*p#T$!)SG@>s3Bxyqa72g(;e~=1ayQI8b4&?Fp`R$%82b92L?lQ}*F!HuVN>0# zUL~Tyuqja(0k|dH3nBJM5!FL|XuCu22W%Wg50^lsKkARh1h^IwZ2d)$|MA1~q3g1b z36lOODxlD@KtLNveri9Dmsb6=nBz3@Nn@FcgL z?torCUI{OPU`b~v41JXZm-|!m>D*f5OL};b94cn?NyS2KTiVpvRZ@ijCKd@0w*vp5 z3vfj{xZj{8fsS~{tEv?BH5refG}aGopZU{-q$Wvfg2U^FsqnflwLb-st_%TQ9qkUt zeNo%kX-XslGw20P6>>dgxq{pgQ;=RC1mqt>=ZaIQS%xdk5JLecBc^D~zKHA($(?Qo z5tdg?kc4^#lom2}06CRlH&a&|7GjZb92Q2jD-2@?jT$Zp_kw5P4|xM|AL{aafEcux zI$JM5n*!)6@q8MljQKKi4BHisD(GY_Xp#cD9qs^ii^c$gi&i4aeAn2~d?y)|9LtHI zyNU=)>rzNaYi%{lkc@=2hvMdd(h-CAfR`K(7F*CIwsdr>KID&fDQFJSMb(hru=8jN z>5ugdmw$$kC$Bpm>=uIISUBQ-CtDVLiUQg<@4}LnvXdiQ?9@34mXmN42+#Zg{1cxhO(#K;;w1}E# zFJ@sCdJ~rnyv0Uia%7ueco9N#p?KqCY&aC?L&f6s39zKQHi zs$PBr{bGAajkL+#XG7dJ76;qYCkdaxZX}$}6(`O$g0+Okh!^U?sL2uK{PJ@E={KC2 z692#$6*?edW3%hQSatgkU#|zl=Z&F7gu7)7={BMgYAaDhE!rJRjLien0xm4rq27TU zA>;!aCinD7nfEnZk-F_O_a%8Dn8KJ4Rlq3xO0*AevOQ-8y99=-$o@fgFN9$Q&=cEd zyKf+)iv1SHrz{Motb4^aZRYu;B*>4Gq42;JvtR7LmjHt-#(2!=sO6UU223Nv>&31Q)iw>cuydr*S$vk$`-vYQx74) z9tBmiU=Rxu46A0#`R@xWF z6CtlU^z_cSc$9gkpCH5Scq60qifcV)_fo2WDAJt^vEEL61#N~7b%($b2(W$yaFV=>xD)j7Z|+NN#Npv16@-k&>ES7U z-@s2q2fmgu!O=#c|GgWpM4r4in~0`bm) zl@=fUs%4(9+C`GSxbor9D7FLURQ%R<2^o&;dSa&NOk;9?rqONaxoj9|L6C%l2_w-g zEuuXiCelccmuDf3S2DjMW$^^Rrj;dL{3Z=yP0+E=Q;2B)7`e^gRQ+ly z;dlFXduZ6nZ7T<_CVJ6J&N$`Pm;-V=*9yaCoDB1;nH!>Nn41^osjV$xWymz7T+YDv-ID_?oIzP7(?>Yao*D<1W@F z&f(T;xQ(o-$^Kk?sPGjn>M8@9=v^ep?XXfHOy#+is+aa_*5~b+*!Xu{sjNp+>FdCQ=3Elns`$DI{SA@2zFt3LBw>Syf43uZ915SUPLXq zSmSw=i25(HeyQ1$y6Y=F04Z3zYy9uPT9tQ07Sbu#s0*yS8$H8zM8ZMaJYzc$On`9m zs@S=6I}2NBV9!bWJE+ET6+=il2k|Zq4G&>{s-8@Un~09x14JlsJrN!JiH<3 zHZz-}jg2X^HyprsaX5paR6Rj z)!L%PR~YCSk#Or(1_64bEa~#M`)z$CfY8)Lkr4KUBhljPHf^kW!P3g1P340ebtamF zA7=?cbkgxqxBE7gsAFN%`5>;E#o_Pt!+(Dmm*~7_Z>KnB1i__fR@ND?=W3&M~=Xa%H;V>**pv9{hrc;%_lU zQhHL3S?*?7=vR01{o+_Nm^;9T3iF=O3Hi=gmn6I*|Dd0$CT^?Wq^_E!!=GtE1)|+y z{HzA_EzR5>2O|>_`0<#C&ivwV4Gg@#8zS)m&yn)aDRgl0OWd0ps=B;V%Vis!Sq3sD zbM=|?!ulexz9GeTD|~%JwZ7!3GsNayKEn zIx?d>0Aoqi!3ATe+|BHcF?@u}O>ihi&1C^5#rAHI|B+k4Sk7R`Xuy)r;u|dabw*YR zb;`VG=5}Fe=O}8fP4s`)5;rIZk$d%OZZpk-Hxc4z(YEiw_T(0iCZgP?40#Ioq=gs| z-_j@9e$~~f?s409wp`BeMiw8kaL2G-fW1C~uQGF^O++t#O221sp+hIlNIlZCcqt%3 z++*P;S@4M5zWy?9YBtVEH=ITIi_??7C0>w@ zw{m1Gtb3SV5$@svE4RE`UBg}%6}QpzY(pyN}fc^Lax+CaK~99 zSGpFhkK5{y^SofgSrju3FX5?OluPWzPg`J#C%(;&gu5ZI98>JO@+avW_wf}v|J8c1 z-b%0aCv#&NYM=)BTUp&p@-OJO*56W?YxD2cTdvXJLmHgL(m;twoJT@JodVn1nv=7L z?)G=1bo@;weoTXRasDpXPlPNrQJHrz@#@cz{ShnhT2_iX^hr`>-8wxelX8x}SzaQP zE@(Fr?L&=3kPou53C|cqc1H00Z0nRU(hEo`An_!QUuDG~Kvq-Xo@H}?GH_drtQrBq z4_d|1y4(G)OP5*+|B0{R2ApLs0WS>lG!S0_wnhk-q}yAREw1t6)67L{zpc599&`~! ze2@k-EaO{p$Y_ylL)F{P9%!KE6v`RMI&U(^W{}-}{7klWjfIJu!ODSd^tmhPNW2@{ zcvu>WxQd(~E<6YG=dug>5FtA)Ozr`b)M?!GCc4~TW?7)=|0nmDiF;4iu}x!mjRq(= za2;UcTTC?y;Xi9=j(7kZ)`%ke`U;;g1tV@N8Sb!^fO+&Eyoyv5>()#{5_Qr8$TFB^ zsZVeEdeqXbCZgN68{R%w^e5@3)m5bXYZ~H8wjQFqX>aa>@QW7l zS8%C)hJy#E^2-=UlckWg{7v7#ho7&-kEL@*GWdISsuvmJzR6-17N({X67#g!p3jfQ zIu=i*6JHHg$gY}IX@DuYqZ-^fh6tHre$je&I{&%oEKH@o&Amp;VD7;&izkwSFm5$L z<1QA@(I*M9HRmET#PMduBklZsX2R9-W(wD?EBxHbt;m4Me%s0iG|W~aTHcJTl9p)4 z(+#-`O@xqZ$*`qy8^)6U;6ObIW^he53oc^_Z_;whtcngV;LJe)8UF4-?stOBQ~#*` zIF*XyhcMrqPJ_pcn=H5`oA?^Ef3#mBzyht_XvrqL9Y;87xEIR!2}g!=+i5gx_3 z&t!qf8ajLlI#u(1ihZ^6RO^m3Mn+2)--fHl5b^w5#>PM4%xaTxR(^i98Nwd7SbojJ zPnoz|4fshGn{9YmMy?^haK1$!4R^$P-PXcXm`q7HSDs9?;vdHlftuY2cKIV$#<0JHB z>5-S8Y~`P}KzQhD2wq5nUA%x*sj$?`L>tb)coIA3>vT^U5j!$*@LLNsqB=+Z78o{s zXC||8lLY^=ykHQ=6nntU?S#luzV3UJEnNv8#CCO*d^l6T53IkJE{# z3x6;%Yg~Er1r4=;`;vBVg|^1=k6P=E<3KUn#A4DNk%RfC!uQ_xR*LAp5#@r1Z{(TQ~D`xX|M+d_39!@7p%tMZOW zXpR`2Z%?~EFKyX&ihBC{vSYbN%HdsJaW35~M_N|U`LoR}b9m1VE+vEWq?7IKTqDyw z#rEpp#AA6xuR#PY!%GWc^iJSiS;6r$uV#V9wFtk%fO)k{)F2(c-Ub@AJ4X-Xv4F(f z-?A^&Fa^HFH|uQoaId8oH{n{0d(*MUOe#MA+v0NTOHBTorDkjG;4ZSy}(ZU0h} zdql%^=MoX_C4dXzPzcYq5n)`&5C*Z5t%qRp@XLc${D+#B>1V)<+b$#n^4`t%+qfq( zEUdwD&6x7(h~I#rdG2G0-SYWToKk+a+@Yb?qQUvDEGbBE6Le%q!M(ZExe%>}Sg#kL z>Ck+KpZ3ZD?h#vJsrYztaA$qZIf%#+P07*%yUF@GCY&t~r11{Tz00t{%3+J;Q!Ocb zA$f)<hvR-^ zh)CgC#U7t~r7lJ&+__m&aau!y%-zq(hlq$ZJp&v9N%&DrQL8XwmZx(st|A=sQ_Uf#rEA8d5@Y2tq$g;-ogccA`y1chJq{G~7fkdykEY?B z#^7pPmX0NcyUUne1X-lcuHtSyaWn*yUNq5wmG}b~+7T2%e*n?e(oBm*vQSr*yGY!x zJ9f9jMS5kfuk6(;#ZJFn9A&uDKc`>uFOSzXDHNKVd&oGnN!o8Dip6o{40)iZwQvm4 zmS3T3`lrA>qb0i)d^AhCtYq)J=0nWwO=C&JYk0J9;(ORb=Nc?0)3f~$F^&6t3T`RE zFBtGgI-*^=F2$L_4zwhOx3%IKmRu@#_lrY=czcff&i9_0c=om+>T$o0ULeNdlNj$Z z!2aG~#IZ7>NOb6v6(|-k3-j@|^wvIV=t{hhkqK^PIvWMHQu!QB&U016wuhk0gZ9KM z?ky+(g2`z&+UK?0r5$=AF>J}wSne^QFQ}`zpBO^0>a7Efx@qT4Ej-T5TZ>&9&ZLoV z&z++&2)W>Xj^)qO5dTc9tf_NxWh^exaD~R?qqzFonKQTv z46eqo3J;GZ0+_Iq(JuTtM}+)?yJK9C#+FK=i#uzqD1lLa-C&>V(u(Cxm*nOwv&^B< zl8NQ8lW>pItXfR-{hjWT)^BuZ(tspB!7ZU}-!Na~j@B>SKpB4JUSlk0+li3X;(|Gz zP{vIbJ?9cd+~utMdi4JK3t0$AO3c0dr1X0)VlK68Fv>NkOj>9`M(1TIw9+Re4zLiy z>WUvM{D+p{<$4-QfAE6R@&+A#%6NFI-PV*lGXYMo~GPV%Inm`UKVg zFcGzUOgq2Ll8hpyQ0H+6_26-DcMcQ$1?hZK2Dh#%_m%R+Mrw-Xb_)^R^)h8%yn+Sl zY9|L;q!lfv%YvFLXsMJsFvaawJ{I3)Zk4u~xeI9giMHu#3UJnCR{R9L;7W6`bLayU zAiIkEZG|T}YN_%Q64%euy&jy2BL&V^Q{630ReuH8VI@UPnMlH^#6&;~kZr1Q^Ok3r z);qJ6A8X2V$`o9hVQFK;Occde(pl}c`EgeVkg85J;atALaDCuGxLcx+ceV8y>@2QaI~7S;Q}H3)v!biJFGV zBsZ*aOLNXmUw>x5(0pKz-ksBMztM3$_oj^$0(g-KhS6#pCq6wh3*V$yF6DO%xa~~9 zmyZaix0{6X^49d8?!t#hV5wJ|=s`4B&U|-mnji1+|7QD`mGb)`qI_lue{DjN; zY=*&`x!$xU*Uu5@ViuH>VP;Z))#0{#s}=+~h}+>a+@b|s*!Fm2cP;;H`b42_Cn9A( z|DA&=w0D$Ol;cmUh!AGoG$e%h<$`C1lz|CJcQ>ldKj3h#naCBJXvj9o#jyGx&9j^m zxgT^yk@Zyv(YLd8w$oaY!>!Wb@dkb}ZO!Eu(CK)EMbz=9Xzo$H=T#Q?STS5;+hOD* zG#Zbd;+Gh?jb+r&QVyv+&0osQ>uR(DdPQ}3G_UUlYZ|~^IR9W(@Eq{Fo|H0D>ZXoj z{(fT0@a)3NwXDkff=xbmdu>Zy2GQF*z__84{Py&Y=`=BytI=9_WCl}}o%|F|#)WQ! zA~QdU!s5}1PQFiv-?LyLgS!9Nm0Z4F+<;1%o6w=%=VGT8KdrT1W^|Y>CpZ-helawj zRSPd$=M-?qEs5IpxwK_p1sUqhzhrFr^ExuxjbGB^*;VDXatM+`pz$@#Pb=V-XHpMB z^n$Lc^NwxV$Z#8=)CZV3zcq8FC|@uy>+AD>sG1 z*Gk+%H^88}IXdEP^r5~*<9f2^|C-DLT-oan}XFAjZ0FcsJ^ zu{sr&L&IIEc1E}PuKQE@=f)5x(7QC$+SOFoEx3ym74h=H2l(Bir9H&4w@|LX&;F$J zH}uQ9M6YyW67!;%Z~HR$J(`S!@tamcq6WE$LAk|JjJda2u0n@@VDOz8)_b$~1qL*| zVM{7u8QOljCb4n5^)-6IeWP%y(Ry9_!oifL!SusD+!7PfKJc7#qlcSS0)f#)Gtr4t zi-<0~s|vrCiBn*lh$83i2JS+^J+HVGM&bp#rRka)e1vC%P}nHm1B!f$9)+CK(*fX0 z9i@b{=LyyWl>*H=DcNz^UW z@!U}bme*0Ffz5pAMC0BIKn;G@x+^V5#x(B3rz%M~Btq^J7v$$3t)*U8b!-?t%NLEc zJ-6&79crmAPper@?WJ%8b5u42Yn@_^eHZ?M!xvdR-&od}aF{ONjqzm~v`ckz-lcc% zsZPrvg5r2yCH}H;56^v(?G^!+ ztu(oJ5CQSZ9amb5b=zL#0^pBYP8Pr_?0<|1ak>KQv!&c71~0Ybjxy@Lr5=eqL{VFH zR}2jrF66fAK!?|(U*vm8d0f6@!h+|?a#()N#nLH2t-8AFP=2lsZ8}m1w{1q(>63+| z4S{^Z@3dqQ&N%{YuOXt=TAX+s7v_qqcW1Akm^h$^yq+9+b>f^#5~p&xzd4Khlp}e+ z*PM=v#&Z`88|2*n35kB|{dD7Zt6%3n9|K9&V;Pp$xrQ4x+)JEpxi~l1S~_t1kd^yM zX!wk$maKkA`a4HFB6a|+7E2BDuhIKQwV2LO3h^Et5y;=+0wncfgMJ0@78Lwx?qD^* z8va2Se-6iQG;%kUaIc&}c_975UBY5|s@w{pTjnF&JDywUYC((J3Hm1%m2jz#?3@^m^D4s)KY@|H9Rllg6Xn=9cv*Pxs9Y7w`Odar-l z5CHszr$$jfx<^ZYOOfix1_RkxWj}fN5(w3XWD@Sb7Tv(AJa^06qw_D&5*01W$MBiy zeYLs|_{F1T&-Pbgp*(#S5wma3xHUw!%{@F(EP-Tfq<@y)D>My8s`z24 z2v`GySrW`AP_jh-kAY5dq8w@^!oLqRD5uKY0;B^q-O%I_rpKTAthN#nT+R z*q3_d6v=<=Z4#djA?Cz+5T?4*L8*Ot*ad@{~WQ!?-c2hOJ3vBQX7v zwcyY1PvNh;1LYq&h4NqVs@&oKuiZ(~(Er!&{|DOrmqgv~(oIilrKHkb;`fs} z)CZ27-H|*P1kEf`kHmn(K$2y;sIElKc*DUNBWGAJ?b$(j&2Wf_96}q^b3oii0a6Pgg~=@%;Tlg?D})u4Tav`G zaKc4vxD(3O;xrsck#o7ZRcTkeUU{OXVB~O=ks~I0buUhxtJW;5r%R}ZvWy(Og1nVU zmJGDN8R&0t4vILsGNwRC431Yx9vu?ZiUB>)GlVAeKGk1x7T7j8GX?GiUZX9PeVT$2 z61U_nQU|X&41`w4M1ny**)p7>0;i`aXa>0Da!~gy4WsX1Y<4upZZ~bugQHwJq5ci{ zvZ$D8G$v2Th@#h$5q~?J%uoQwtw2e!nhXjxjy9h@eFH5;HO%As29=pf-2eq{aK;Jn zL;BD}5#qm*oSXD#!4UTqKY(sBH$h|%$kc}BVm~ogoR7{!wPu(a^-L2=Pt=PIM1a{O z2EdWFf5?WRPxg6`nukgh<|pKaL}O%;AZ`JsNC>@~*8&X4KA6csK%C4@Q^JDhLv%$l zWYe&SuuVoy%sg_IJo5?av&MJDPyTfvNbyM&$W#5Y(`T;K`gJugr%<1xm;4LJ8C~vF zq8pBQf!MmBJBVttXPmBKcAUX1P#X~Q+nW4Wc|J$ui>qx%%GB9QDoV1;|5-%|CO7Gt z0kP&Omh63#ymFdRDjM?)>U+>R)t8a{X&DePx$8EgpjCURz zpZ#*_dGA!*`o)<4r_%|dOcf3gq|+r; z5~QP2y;SPVRDJix)#Az1dI&Yn0;XOuk)6rEk z>KX^O+g7x#nrOaa1H2c8b=}~fhE><9(m{ZYCd~5xe$2`#ivKV*|GC>!^6wu=j;7|n zX~bU+=DyXbrbjpbQNA$z9GXQ!Q{glpu#AAg7bRgv%pXMfIDjiKWv2@pVqHm|AXEm9e9PxLBX7R9L5|BsEa4s2VpFtJE~p#;yMja+9?XK-7bpia5aNn92|vfyQ22gUA6x zH`FOkv^8Z*LaPD~T8a!P$3A{bP(7~-iZ~X>{m{lBgickQ3T*u)z{EIpTu~8U_>;=K&Yoei6q;Z8>2m-YNsBxx5od>k-02R_FD>_z1&kK{HdBiB`89&35MVc?iF^`lFDE9Qc zBG4H~xu@CV@er87aG=M1W64zTh5Ddk1pER2(grw#$Lp#3y@E!b0NFRHja4fM?19SP z{UDY639bMtqM;_k_69m0*MMXCG_ZLC)FiT3wq+F$utC_HbCw0uZ7+HLNp$qz?tYn2 zm};P6Qa9+rwk7ggWsvcLf`XV@u>#50{)bfRwo2HbSSn5m{={#Nm68cpSi8Bbj zyrh-~2+mz8(5ji6sfT9YlzGawWpy6ZJcRlt-fV=6P$%MK5exfKbajD$fgU_Cx!wI6 z#QZ2P^%ARAI}}?;IhvSH_~yBQ?tjI_?RE{-NCZ4LulGS34ysk$SGrU5Nk?w&eKhRs zXqbhuipxvDg}2E?`Xmf%aE-e6Zz7un49qU500G|*d`~}$65jsV)NgLD1RS_>=ujAn zFQ6=9a5MBtK4w+)Fpb)+@TJ?busw6-q6L(N^jG#GCzc3_aLpmYmgL;N3xU&1*=FfC5 zRwn&dKqy}5B_J)=MSEVbm+U=dA1(?fa~5Eo;Q^wA7VdNnTx&~}uTQ$KP}l)yon;5l z2VRm|yd&v$vP#q|udOAz+|=Q9jitNiW&5GHN$HMtE$%)nTPD)wwiI9y`+x`;MLZFb zD~+%xf%`g;Mgk?07C-QmHvdY;{9fI=p`)pL?nLM6DrV19T)|eOB-{(NalwrPKA>`= zRYLr%PCKO7Ne+~C->uML38F>w>in71a?^}5putPidi)s%G5%> zLK!OB^Ah1ZJckVW2j8E)65S({hbPkta0PCLxsM4k&4)^6VWsIz+1(DCvF1*x~q%358+`!9KJOmLQxGZ5OWZT8G%9%e;qle;_DSq6o zL6abHc&hmw>Q|R_sher;XTb>ei0{*#put43)$@caD}EsFE42_N>8r%wOICh}?-}7# z8>J)8e@`c#FV;drV!4I}^G~%}meRKSAYNwwiA}Pb*JgLf?GX1BqTFbPP=>c=t7KT% zVY_F(#zx}p?RbrrkYD|tZ>IM+h;GBe#1H5xmCAwNpttJ@ujNG=jiC}^Ewgd`B7Qsr z=LA6{q!UhOK(Ism6E#pd@YQDdNjCfQLmE6&SNFLt{}K&%j9pw0L^bLF-lffh#R40N8i8ue){XnUa)TOY zeMT2~7D;Clz}ojEiZW-5pB(l%rxVcl*Dreo>TK{EI#geip<6qVszkb>emoe6# zwMQzbDa@($7r@R1Dka}fDAzR*m$_%1CeU?LVt|>?c#8Ly&rPj6z?8p{Cf)*baNRgP z)Q$-<*OH#C2GsmPIfCsPhkl{w(7@Wr&X)R%UB?B zY&M{BsQ-xJ3-sLe>A|tAPZI8Quc5(h#{=ngS}p&9Zh$7`PN$U{sRC&nE2r#!Ab(Ff z3kMr5!Uxm$t`jeVB!#^;^KIt5Kh-8-KR1vAO+U<-PZygU+0YoYeIj4w_;g4&JV9Oa ze%zo36nCVA2C})d9}Va1Uo*s_DMqyrE9CPGplyvyGJ8)HTWLnV5uKUu(i|zmmvQ2R z>T+g?7w?o8K&4c&^yA3KxhF6>3$M&Dh9IgaN4Ps!^kib4w4MW_`6Rl}F2NG9U4=>p zM@za?r$3!vt}P5hHvorFyP3v`y`CKOLG>m=?3!I>-$g_XUk}z;-(#y>g&$;C=BA71 zBxcCx@F#Rhz8*FLd$B`Sl0}2uy9`S3udBSFdoNvlmV3Fn0Hf}^j3eQWnlM5Wq2S}< zpPDyF7ju@MaN$!LpvEg0(~G&pZ-fsmLoN3F;VHW6fjNV9FfWt!W+AHbmAm$S4azU$ zd3yVaoErR;mTHAE8H;QJ9PtIDv0l6PeB(%*fg8m%en-fK=cL!P@YRc{_!x8tv)mX> z)(zP{;CeOgDVbLM5r-bI-3p9o%dnyKn2Emw?^s0?86N4_PtLw%(-rq})VjnC)Rp#g z`JhUdWNv5Gvb4lqLF}UCb@fQPcT>)Ptk*K0$GvgH8AI(0QgSL@WVvA?hzW`J>Je2 zfp-MK;|{172gCnqdM;rG+VjP{JC`Ube*!_$!Ln!vCsiY;ulC$((>M;Z>Fzcdb^f%MQv=Ts1@R%`%*lg5i1G@&6XMl&mY~o)odfgh3*&ZL3|xoy}^u~ zEJqm7Au#_-@TUeyzuz`VgKiNoFEq1<&!k3^O6=8e@nW=!|I#pcM!uUbztgDdhR!|N ztY)_$7*qGROmr^c7e8$|Glh^(OanGKpPH^#jM(nw=9xCVg4P0qBM5km*QfgtQMBzO z5SuMC`G#%js|&fi)O01cg(W(v4SH`_C3`)CGiTWkuf3aa17pW(;E!ccEAbHx|0?6( zziH{K8BBETCh>lM6iP%6$MB^bzJacJ8*RGS)hPWk%Hu~D*Vqx!&h25aArlHQz2K}_ zjxpTLEKiSS-dW~juiN41pP4n$R@_Hrtv#PRV6?1a<)GTH>28A_j`0n**Ss_e@~QzJ zH8Bs)%~dN(po(R{N<;<_F3#j$rTgdDr{G_Vd?jZF{M{J>UitkRGx(9fi3;KGVTYBj z)pExSG`P52cr;OD$;!azW%kRSANjlKz5C@gG@v!S80_SKurQGmz`ln%KnZ0+@qt0cuFWKI7{R_<(k8 z=g<^+bziSPC9HvSF1 z`brJi+vV2Udc+LNRXX4qJ;__oVOh1dg}YO09H}wM&B__r!oL721eoivhpPBr?R^Ve zROR08yXFFG&E;iSz!C-+XN}At2qUuulu6MwI)j6Pf-+o6Z3YG$<)U1Lvck3{wl%HP ztgI-ltn6;3ZEb66scC6x+cFc)ymV8`$}V>P?;2E8+TVB1x4-@S&N;Dn);bIH-f-rT_BNcM!F4MCW*DGxOjz=sK8h+b znj>q*`2^TgR|F~u^A~T8XTOPs4c6zIrsFd)&xOit?dBmUCF54^aZc>-8b~o0U_N8s z?vg{(?s^LUO(pKq9uvxt^6QS<`K`O?TopMCEf+@@KA>}qlm(7UOm17G2KdWHEQ?}) zh~kcNxqY$ywGh(+Zr|7JN)9jPvi}~^zQ|L>6>bQj3&-afo)25Br^RM3e|9c!_A0Bn zR*mp_h(*otcNVmX890)`+gZGY-ZjzO%KgmDoww=*&ga-C z!t^#RHFE3Vq^CRAnS4fb&9Mkap9s_E8tz$b`@JUzC;yHe6*B$Ec&Y(98vv&s>Vqn^ zg8}RI1mEeT;=9r~pr)-?1PM8Lew-8LeSY2BY?l_#(r!8*Ki~Wqo5qMkLGX2}x}b#y z$w(X@4t2Zl>9!Om)rY*kTUgU-t#w6+eK2LXPZR&l^3JUrj~YI$ag0eaoHIYfbSzch#*v_8~sc~~B7bGTSn%@@H{9Ej@NK-Y<&Wz{I@Es%Z zU6_W&xFMEN{&e6DJTn4+6`#ziQ?rC{13+#Y*hk{{5kp;|cU}SdTtQWlJTLzO>(UTn zDF<3FFyqwKrovidMudHO0wj1MK5L2LIH$&%CQIU(lkAQN=MLrTsZ;pDMSZ;ZQ*v(p z8Gk`T&Jb{*uN}$2j6&!!KvTo-hQr35n_{;1QEc!f4Hs?>W7nGbH5%9gdA)DoPYpCY zinsBWL$QE3iI%Opj>AO1~64b~S~_E-xP?a?}kdV~MQb56a6NQAI0X6_#e(924ihlObO&=bV* z=4X-(o9*8@odMg{YW{33`iYtxeorWW^hUZmSNV43_!FZAGs0g8IPd?RIfEUp2 z0P;r#s3~U-INR|Fm1%T;>*rKiux4LlWxeI7Qv@8*K-AlEbuR;;@ji(`Y5I|GdaY}MXhd^d1>U$-A-9HBm!XW zc?%$rU!vk~p%OsP?0&pA3SgVHCcPq=V^z?w*mk{6I())!_Y*d!;6XYKQMACX>m$LM z1I9~c4QlYO_sCQQgOt{%f)F; zr=o$SiuVn%*_NCN*T^8ki5%`2A3x^g11gL5?a$2jJO*54^x z`1P=e!;dFq%R=r!-Jqp0Ihd;Z*=2HsS+DrwZpL!UEFB9ZWI2!`pYom6%}g7&VO z3hH5)agJv3$UE($kn`0W>{JQ$zWM3A@0C%^lR&(0f+keP7qHjIBfld?NjcqQ1!OqO z*yRegk6QmXMgtU`Ms^v`{S?N2qhiO!xaz~_ez{9>m|!`P(4@8n&Zz*HoB4ShF4qdL zrs_}Q?XjD7aMgWHc>=5jGK=~`!^a=;)(UFa82)hkY_44`?#rG;;dO#=9!vn{YUb8G z&F0CP31A<_L)Dg*0uX^Vt-;Uq*QWX3;w!oDBdyQLTvl57OUEhHsvXfa?_^W!CmCb5 z4>d=?QaKLqQ70L=J7d`?5nSG&%!6{b0ya;Tb7kh|93!atA8M&}<{@Hr(op^lWr2k= z@?gzMeqZ8aEfW7mkM}k?zEnqgLG;q+^X?K#)l`M~_0>yFZ!nXOsk775`8ZT(eQA*M zsQ!92USmR4Uw^~hmdesGWF8+0Q*}ja*&#JI%d)sm@)^Mg6pi)xG#ys~@R0g|uW8qW z&O-pO4Ij<*k79P(PsvhW#FKPj3~A!O#O)DnE5h+}fTIzNXGY^f{&wUr9OJ{k(~whKJSJYBhz%2> zvX%1sU;1{m&7RA*CEb<0EPU0c`ZYMbdG%;qtWl=(59tpZ;OH3t8tWTj(gQLxji%vr zED;+(*_k6R8DMspBuOvNzPPO4UAM|6N9nb$p7uD}s?5rg|iizk-kCvD?!IM)%i;4cei zXWC^{js788;tg3=&@hhw8oPD@<2D>-V&V!n#F~pX&!nyEnA|OhS}pN^-bw*;L;%Qn zHN2J$Nw@FTSi1V+77cVaAwtb!BOYhsv_}2c@umXCuv10#12VJ!@Zw^LCuILhzhd>; zeEd#C)M#v_^&j?CZUJN1k9`fVX};Mb_wV`iT~i1)C0LfXBpJPld?93R1wT`NP;H0fU_h7;9;Sg~fhIgZqw^Ma4x@mD_28_oCu=^{ zx6?&0;182l)v#U_tB%tT#|2Nbo9V#zHPdJy4|O#yez$ZHwPt}*dA8|m{!Tnn?R+S3 zreHUf?#NMw9~Y=-#xf^Pb@qH8a|lXF;*##epU{@}A^1VHb!VbQO9)_N_)hMO&haP( z=Ai2$FmSVgfarC~@z3!KA`Mo&riMxIn(>oEQA+y=_VqrTTm|de{pqmTUoZsav31EA z3%EyF+utawc|KnmRbpG%^(*d-;8ts0iz(_C{eDLKk+6u$o);!`DEXx1LOSYZ<%i09 zVF$!~q`ceGuGWtxc8KZxI1NEoWam@VyX^32TuAFbVJvTJ@ILG>H5p^t^IWEwx!33I z;hyIU-gmqzv-NkjCohQ2nCyLr{gz!kZciU93`^Ev^ewk^u;n|Jc{_OvWB)j$a#M)q zE>Oi{$EMq6v{#qyj7&Zo&2yGClK%$2Zu_T>jk1{^8Wc7#2qtIIT)Eo5Q)Vp;P2G!9 z3J!Czv~xE;*~f6$LPTI1x#zRY59WQQ81KdQxXz#Sb|&LtF%&@9?tav1iIUqqQ2aRc z34BYWa5p75X=OQVgVbwr zB`PMpsoY*qsk_oD_scRD^~uPLKg{PVpQPSXt~9+Awn{QTx9}5yHM=<*HM(}na3jx; zDfr-=4VB}k`csB>k7a(J?6lB==L|v;9yJj8v!6(IPR`qeCS<-V3UA7|0F(T_rgO0V ztQs`e5*ylg+2?7N4>c5Phux|@XOd&{TaJ`)^I!Ds8tact_9uv+o^iYD7*b9)+@aC4 z7~R0%bP*GieKQ5CfXOa=8jD`m7VZsSb)EYk_K^zndpKhVaDWmh=TZI#+?7q`v55}L zdZPqf@ebEB80Tj2XQ^85-M-p%rmM4_D^3#&QG;@S;Y<3tC020VbYfv0_3(Ex>mtfw zkU>#f_#3$sVUATw_M8e=jxtn&+(2#F1d2(9QOpsA_ZV=hS~*1D+81E@%7n#>C7RLA zqG_|v>4AOnsxZrNnlkcv%5iw4%8;ooeJRP2?4g+*bJ7%f&sq4T`s_`}coM)7 z62vF!ZGwg=Z(=8d>V=M3li3lDvqmJZrNwZ^H%M%4Rx0rs!<-M%?r<6N+}sO_K&2%> z+U3{O*EE^5nT{zufXplxK%JGRO+YS);&4jvq;SLLL6i4WZ&# zae(D!HU9{8y9Ss={r#Ymh3jJMw455ud6*?dYu@t2sI1c>xo@vKnNaYXhDdU={iEiN zR!(sZLJqTRa-n*|v&_aNL(862?oX5@g9JM;C&c*3ASMae%cb5r()3FAGE^TP$9p$*gB;&PM&9%%2l0Z~Imr{wVi+I2M554rV`n zcoGEs8}+S0ZvYzM>M?|^6Tq+xuop#cox?5DbYcrHx<$53gS*D+i{janQFtL^7^!po z9EaD96_T_aKh0T(U8_UwIz~9B;E(0LN+$TB-ne$H%QM7sA`O2xKtDc`KfruxZx3xh z)m5(T&n4>cbe-_R=-eklI(Q6#uN%0)2-BNt{AMoI$GbYJjQcQR{xDftDJ2ftx~zT_ zB{U#?R;=@0W(ylnL-?`$2z_+6HCfG`h#<_ISH|YQHP8sKJ0K(j6c%?5!J9^F8oZT$ zd|eb+(t)e!bStOxt@z1&8Yo)&w&5ti-JZ6q)$11KB;UisQJ6}@>>rtmuHzPJ znL`4YYE~hi{(N?vq~u~X?(yEZU<7{@P9WLk5iFwTjy6{%6?Tc+E4HpR%wo(>CyO*2 zcAbAV_NKzAgGn)%iWQ>`^K|$*ruN1@e5|OQW|uP$Di?!YLHRi3;|R5n6Qgs-hXnpE z#8dW{!+(1KpBoChK+<7Skbz^;gZBJDYyB-lfKe6Wb|$3#{=1eQQF(J+50&{jFGe@Pbj6--u8sZ->2m zZ8=_~Ip&BAI=?FOF{W==@l*j0Fe^Ui)O|GszYQ%T&V;wdv?BLS03zUozPOWN?;W=~ zEmNyx?~d>yU#x;!m^)4}Ekz!+-LNoHe-on^>t|Pwv#ywcyfvjnW2qW1EIgU5*Y*n> z!fy=1d$ss0jw%+8YrRXI;+ge^QK|7OJGTY4v`<-86u5Qr6a_QOnk9q5aw!c*6YpIZ zEMxAaHc|VjLr@F@#KGL%*j_KEwk92H%GW=sGHo^`&!?e8ZHcR1%Y00w5G!!QIEy`; zx;}oW`P_yY7p9nbxfL>I{hS!eH$xV`1m~zS-=qzXkD)%*?nh@j@A$gSo<>=dA}bHz z+*LYpUFM6lFhb?5$s8?D3iG-*xv6#aGHTi}y=p7iMKjhRD3Cm%ulu3^toxsKV08*7gG$h0O3tJGxkco^gzgR~8N$gQ8G z(Lbwq^bNOGg(V+O$CEAAzbXpfL;3f}E{xD*j?l2~FzSB$-7@q(^|``PkMA$m=jyoI ziVC-ca>qGKamx0i@#i1gLOIg4`*!oMi!&ddE9Vc`KSXE;1q|!a_F@H?;Pf`N-gH;2-6-P3`RHY8`G~x5wMDUmq1MPB9!$ z0M{j;J&13Ov^=V|+%ksFJ*=>irAMhp^)ItAof%6E zZ>cP~T1ROZ^E26^J(yw|rVC8Wg>AV&Usy&pFQcftl=pQo9f*ChkA}cJ9rwraJDBG( z+E&OMKi$a6Vps1Mp2skQ*H5A^ObyMP8tNJlmTn{{jczQR)z~}Ds&G43i*>;W7@3h~C0GAfk0quC4p&;Ne?P&Xef*wge z&ESyUm7sv#=stp@gRKxjp8>L2lAo2-jW0s&g7|6zwj)BqL`oNMRA32_aBoCq@-kG6 z80jTQ3nWew;v|Se$;bpJY=LTm#S5B*{GfeIxDvszA!0CH*TY=XGiYEGf&Fj-sLcW@ z62b{W1PMO{o+H}FMBy2dCEX){xhpk)Q!bXVgsUQwh1Y8Q!n;94<<<6F5XOaQpb{Yb zAeOG1*Fg+saNJI$5xYxSLgieHzPr2$giFFeLB&CHxP;G>3|Iu8=>=DoK{W>{8W@bC zPXZ@~z^5-3yR^cBI)icttVgYsO-ZwtkaGxzP71nb;t0f!NIb)hR!PGcSUM6K4}!nC z7$Hf7m_q7-FolBnLINo5rH}}K{y$SdB&*$->XBMWnV0A<`@1v#FXzIX{y=`23!^F| zpuZ-9oDA|{)-RKDPpqLVPi>Q1c03`$b^q*tfA+usulrxGxb(kRqrOm$jVh5q{&V`A z9zlFfmV#*YHPYxkh6^&>Tn)b_A27mD9QL%iWR0AnH2 zyrW(nf%w7^Eig&c!w7y+zZm4h*FQj6=b^|TM0LH0P=(kJ4E}INZVOvD9t4^xYJW6H zP800tdKaSXpgRjwv~#L4B?$a!Oxe*-=8T+_AP!o;09lIwo0hd|{r4aS=na5D3iJyA z3`BsZDW~S6tiH&)em=6kkN7~OG!teoeNh$}oJzeTMoH3NV7Tp`-RkNymO4;PEiyLU zw((8GZ%)cr?uaG^_VTI2m(E6uH)jn~j*{fnH{Xd+)7sII-SYUk>pNxE`IPyN)fibH zSU}mvxR<0!L{BI)NznU&ALTk}DpB5Up>UuH5knb3 zVPZzFzYSS)srD@oIH;)GHvE0TOGtc`qO1<&`~;-?%BMfPoyxxxeQi8zQosT#sbKd! zVamaeAFceqQ@b4n3^_f%2%7Vw?`#n@(wer1FF0}mvNhtCg z7fK0YP9$k@`?fD-Zi8~i`Y&m35|h2@=5-xs-$j6&sFxO%jn=eR-$lyJ>r)k_hOQkW$uv+hdZ`xqu&4*8`t;LpLoSVlQn#}k z(}5ufwg+(CU^kjB9r z4__02Ndu08zd;$T+@Y|Vc02~Gr$YGHXeHm^0}gLteKu@nB=AJar>7*%7z-ennoAr9 ziQp2bi~;0*3l#&s$@s0gXsp%fz9^d$bR+=d(caBK&MJhDr>vWiP(c};z*q;GVQpj7 zJCS6}_Ew0al780_5!hDcx?wZmgrQcsUL2!nsQXwaLm7q7BE?E-(kDm&LuAqfI1Tku zi-%U$gOe5Dy91jBR@|Vt_(Knu7J$G7*CJKYsyGz2C5^DlPTUiGPW1O3Wpule3=jDJP;P@xTz6l7eJ)GH^CE@NVTo0LM#btIWWZ z3UHD^9EYR=`i}oT#R#s+>itmouxC+bfe+{DI>{}Jx1gX-d5{0OId>rD6<%B71%Z2L zN{xVnuQXOdukrzQ-)?MsU5+vfpQ6xb)`>FVc11QP_ihGGNVPRu4l1fMs72mw=Alht z?pm`GT#Q#RU6cjZx-CXx{VU_itKHah6tZ_boq1nB^5BxL34u&hvo_x z`pcV7;CuoyoR{t$Hke-og2LX_6zKdHZb#@GD7558D<{pE z+XpqSjbh%1^cfqEUC3GimZ@i{g9Q(-HqyYuUt^Nc&p3w$X|p@FP^fC_=ctj7DEkSu zO$Fy*wn8`;-fumRgn&YqLp%|tzO`VoBC$rS5?iT{hP^BX6jlDDH2?u)?M&+0lXKjk zh^xi(7XjzX10Z-?O%=h3JHNYxMmEncG}>nU7&&fKoSe(wo;gK99f&4M*f5HuzMz3# zTYvI7Y9(c~pGVGToZ;dN>=QH~8p~Eu8Zt&yx)cydKL#DJ?m)syYVqy;$#~@t5`b*a zp4my)`u&{==eDFS#w9cju56U=LxncX%`bQ=0pM*jfGYz04OUC16Z)K&Y~Cb)!AJTj zA>?cZy=MIs2?F;(82K-HYV3eX4-B=ht6XFBY-nW26H^3CL7^0xaaPacDgTHio| zPlhI}_}Q^BWY1!{!tZtdxV#{a$Rv|3q_Pk5M*CNsj{(j@%bXL+g@S6@tuoJFY4pAI zI1+A^d3W{iK`%Tu1?VGqsv3~^K89PyEEw8uZ|)3X&WOvY6R6hxI`S7SB`39bE3?k# zNKpVQ7(i0zmVwY1*-qp2+BvaGPbZB|S${&pXB72Q-u>QTuA`I_R6hOnUC%4LzJdiH z?%lwSW5l}_j~yy4B8eHJdqrixor#7~kT|K?cN;Yrpi8ig4fqQeG6at_=NsgCmR2At zS+8+?K~;W(eE*>IRVx3T8luR-F2b3|C`DtUVVX))TU*yk9fTuzCKNo9fVl?GR4Ioq zSbpIwHSDbCJsN#vJxd9HRq$u0osuokJ1 z7O;tfrJOlMR{}`PuGJ9IXzpn=|1%1dWV&edqxBP1ze}be#EhC(*vc?Co0`Rd2fhB6 z92S9o?b@XMs~<>%vELwlk5A&N~7XA86d`wnm!+7uAWZ-nuuApi_R?q;3qCE3Ew zl;?dKodhR>^_!`pg~*2=WEDqlv)#w8gkD!~UcR+d|AR)Hr#~Ntrsn=0-u}G8->~U= zr3gHun%oCuZEGoCmKM_R9gTKdzed6Ve&rvXJuj9PBILd7oDf%bo( zqF^2bY-m%n=U5b2*b%YtP0DkE7E)>928I3v&7ESoObzn^KpcQ&1aJZYj6M>4_bU4nrEgT$kfFFV+4S!$F6zb=go&5X4q!4hm0Lrcc!(K-_IC|mB zl;=E+KDWM$9QE>o9K&fA^!(r`r!2Cl#JZDmJuLG!<=4vb7%fa1HJ%U9kyj|sNm@`~ z;XzsCL`drN2C-OY1Ru-wbIVpAM>Vz`Y^g?J^f?#BU35d0d|I3I19g(jCb#UMoS*H7 z8resqFRh!YTsVpT0jM|)(6Z__?N8<+i2IJ2P54p3{}L-Hp{_MxZNw^N>TgM!UEhIt zj}hCnW^2uEg|PwaK>NNLbg~;*Sn~7YUMXdpmZ0`)uBz+*I|g2yN3&pQfswA1%CZ^5t0OdIIfSHQz{3z zw)yc}wjtlk;H%P8MGM1J!fcA!y5wiWShWT)ku{y~0-X)kIE^4aI5-^4ux{Wo?Rkg# z2p~A^XJ|M%<9y*d$;lNrjdhl~{z@Gx0nd`2n`j|C7GlQH>hSmRT%88ycK_ttbpU#h zHY)hE##!1aXTIVd=|hR^JRNLsfL(#l_Q5lwL32L$FeLt({?0T<&JY!ugaZnHG*l3(X<+H2H78C;|;L zPnmrRquZ6JgAAXX-~JNd>z)?M#U;sM4r=2Ts>VFBA-3PBhqN66|CzuGRGuM zjg$I-&C!6;jSl|{7~kdGDGJSNx(h*^eMe(x0_*=GXJ5Mjd*VLg}7#8tt%tfrQ}-cC8Aa_{WZ8 zZ5aD&yy{J z(>j2+VH&p1tfq6-bvz{Z1J90&KT}>*gHQG47q!1cS9n`FO+Ot_ z@Ia39po)4IlvDz^mNp*;=8^t@2F`h4D0iJi3s+OSv*?;bj#(;hbtKR~xKGTpQ!JzU zQoGuJ!A3Z(oSPa6E-5PfHJvOt#~_5t+LvCV{AjGKfO($>S@6HVa4iB$f@tydT?FYEZPGr0{vfrxqzfO5_X>`!~5fZ%eO-mOa zW8cB}H|>IBF*{L>Fhc9StMha2FNtt8tsifgxdboh8%g-tjjioN6@mJEyX8=|9A$V$ z(ZcTv;hB(`)%-LVR$)C{!A=VUAC0XUt)%A%47*f@V-c|W->|RCpAQ-UE_aF1J<+vS zIp0G9fMR{m_b?^qB;TM0;2P*QH#nz5to{Ma%PFun?shR?oQju4Mb{BZQEe-yRpq`B zZ%2?F%%yK+qC!er#r6Lz4;U%F}VX+HX6#N_aGs1~Cg< z6N``b=kCz!*T2r4#g$KRsd z!2ZO+*lk&$Za>%*fs9>aYWgp>MiLq!X!xYcE0;$=N)u?IRxSJ@ui5U05uz^YE&V>N zqM|P86*L`-DX_(BVfLlo!&Vp~!8swc_V%AFt$pE0zQX{}Rg>L1>OzEZp{M|VVl@NuT))kXAHbZ_XjzAaB?IGEuHJhgKHL-^4pD1q^t#U{^L{=o zTYp9^^hw+|1HY6ENVDh|{nq43H*%&V_hi=|G{22PnVyrO!g^LXC)a^?1~i93@pv|& z)8sQWY;EW9x+|!o;iVCV2enXs+<9Aq7wAzSV+|fCW@hTFpp8_^is7b3nBN5t zu*He`4JI_)F+;ZS5z6ypC^}??u#!W=H|pS23P!+lI&QZS2)EVxmuc&(3Ye^+9ks#+ zle;OtaIIp&sgBROk~Q@pHR7t49RaZ3pu)r9Xr=WC68KQlTO10U+{yNd@gCYbp9V*Y zmulP%ySTKz+s zuhYali!y5073@Yt36y z%6l_!1xYQEg)8l{*{g@4EBg55oY{Kj%njoL6erP639gQaEkzPUkp>emO2Gr&-cEx< z%2e`1_wh8bK}mB)P=OKlsIy4X1R7zK0o6V!9Qc4s-4@%5YcutS=ajt~@-F8z11ydg zodx&EDcB_cmHxiBY*So5sSLjqp{v{46g;q8d4%H5PO$v2L1Ec@Rx;!M*^s1W@_%(h z>eUR6qGV&?*aET8&fs5NRAf(at2rPlf#^6ne3Gx0zrP2qM-`KSLUb|F>LLIO020D( zv<(Wjn=_Ef#v2ufqBNi@qP^4*^C3ht8Vkx> z!8?jkVAs8PK>?4dFFooW)%~&)j=icc`mFEytY-XFq8JnO0v>$$8XxypJm1r6c<}kF zge@;VO%nYFa%Vjs(E=3UvUFF^J3T-GJuhE#tkU!5wcYjNT@@*q9@T-kIaEws%vY8z zVV8URm4zIvv9DO}Z(){gS_@4MYg9ySbtG7g{q@lpq^)Z}OArn~tD5|6z0{>;f1k^N zOZ5~C^RHU&A0d`GoaKU!1R5%&3F)aeAv(l1&^A!W=q;oz#>=kTdsc8TK=9oDaj4}v zgXL5-cy90fn20ta6ipvKLBtmk;=sH7wO4zK=Wl~9r&xPMvkAuZR6O@oxMiu$a-WGL zJ+iINZ-qSyl*b5YtpRX%TUQeu?S+!D)?X}{%XfN<@q7E_oL$izgAu+P%6+J`oE>6$ zGy}3$_1o{#Uo^e(gWxxkwQH~T7SZRP%UQdokq0B&JAivwZ~1wGW!q4aJ9E}xA;XyG z1--8pV!oaLhg##|901(^0VZc$p%LzFR8;gqAi zF2y)fG-jpS(gydIte?F-^53(2U8>p>wB_?DmU~Je|93xoJdTVw(e$YwkLt)blK*S3 z_7>5fa&(Kc`6d{8wF-e%@pv8&!DRbZPTQ z_L(Mlw~`Ez*`{KYd{yd0?f=1TP?tt zW~Uf0$4>UL*G78r>c2LwOF4Cqx;DzIZeA5LpaC+|ug&A7%l|BkS8S@Tjrf||S4I6_ zEa0B|{c9t?`tDV6U&`N=ReS03Kg;%nseu`Q7_Pc#g}65I-k!KB0sqC!_uO4w8~@dJ zuZsIpF8>cQ@4mwK$F)&kd;hA)|BD&Fh|Vm$0$g@&I{x^%e{FiMF#24ZjB9TH%VY0m zpY$4ak9+^xc&`QoUmJUGPyB1saA{n-XwSYj5xqY0uT4b{L)^9TUv=}Um@7dXzt_RH z$BnDUA!!OEaNoLmm~?J3RTJk=^{r&Oo6!o?v$__N3V_@ZNGiE%%e308X(pJOOx@A< zxG7lSf2P?eS|b(~jw`kY4Ykqt#}6cb1FFV|qF4a`kw5sJE;_12mk5~0kyX+)hvSO- zE##u}()}^w=-_YoltPj2yG5}@g!kZI5{M(+0hi&oDCRW4^Nzvpn(C&q98Y?Qkl*YZ z?P>JY<^{$D9KMnXEmiKC?rYUqGki6L)&3k$n|oSaRk1Ct!96l}nopQz^A%?VuMKJl z_-b5@WfR<)aBoyaNq(A7uvM1_9OFuhZPo4o39oEMS@EptB|^qjUvUw{BL)fvdD<#+ zvKq@u+*K2U?~|}wN{ch5mzLyLR*bH0=?>H8tEq-K3%kS0u=z?H86|@XGs<&8SNI zmXEIPS?|rI0BbBA^uLkLY6qaWGOFE9$a68IGw=uH*!y$6wY?d>+5#azSRYU}widtV zkIMN!)$S&GbE;d)bD;0FUR6fD^pPTA>WuQy4(NmBRRzU`)5~i-ZRM^1)^<6*9Qr@$ z%aHc~^wmGA_gU))n&nY?XBMbJI0)e zWhJ=+wE1ec3)&{>M}nsb#;P=LN!cHcMMZ+AwF1V!?sStrRqd-K{k2;7H;++JZuu?H zzuF)zMc#ihwnJUCmFHwNl?z2xo}6kjH~c%tQj$(Gr=@v|vzq>B{vl~3^B9z4voAPK z{JYvg4#eBw8&uv{Q6j?J=c$Hyc={iuv#=(s!IzUYXl#k61;(}Nvf_VxyX`qQ<&+hB zrkB>Zrg?+o(0?#Tm67xYN{jRB|Go2or%9TViVHH`qnyDxsF(58Q=B2qab=_Gs`4fV z=d|9&tg@O}&7~#(Y#u0U@CmMeKBszuYbO}Pq3>q3l6e~D*amNLZfj{tZiX~(*E%xb zdn>dJX`HR7@wAo;Qu~Cpm3vlxAbr-@X<~rP&+g0fz5wG&o7C3H+&|6Tlov>Eb=Rhm zb)+bc?p`0j`fn8ZJ+5G+D2}Wn%R(<%9*!cvN2QasV+&bUx(i^L=qMn+UF7#TvS=JB zEh_!wD_pwxSvG#qz=7mWL7ueo%$06QU!`x-TdphXg4}>~LByrO^pOiyBg+R?4jf2s zISR=IaXR^s)~n=;C`wm*t{DrZev>=3RA_+PBMZp;X_wMd)N6XAq&v%+VIFC&XtmkO zM>@ul_6&1|6v-^<{fPqx4jcl%oi6ElvT(kbYw4*v_#|`4Ff3E!M^4hW)r7*UV zIghLf%3wVQ>)9CzU%OUMo$V+0L zRGtOoZ3r(%6wAd?Ieu k88cF)4KMLHM|Qi$;8r*0E5H-|C;yKgPXL#oVxjGS0iSD!ApigX