diff --git a/requirements.txt b/requirements.txt index 24d98b9..8473206 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -fondant==0.6.2 +fondant==0.7.0 notebook==7.0.6 \ No newline at end of file diff --git a/src/components/text_cleaning/requirements.txt b/src/components/text_cleaning/requirements.txt index a05cf10..fb0f4f6 100644 --- a/src/components/text_cleaning/requirements.txt +++ b/src/components/text_cleaning/requirements.txt @@ -1 +1 @@ -fondant[component]==0.6.2 +fondant[component]==0.7.0 diff --git a/src/pipeline.ipynb b/src/pipeline.ipynb index 1cf2dcd..6800617 100644 --- a/src/pipeline.ipynb +++ b/src/pipeline.ipynb @@ -82,7 +82,7 @@ "metadata": {}, "outputs": [], "source": [ - "!pip install -r requirements.txt" + "!pip install -r ../requirements.txt" ] }, { @@ -100,7 +100,6 @@ "metadata": {}, "outputs": [], "source": [ - "import fsspec\n", "from fondant.pipeline import ComponentOp, Pipeline\n", "from pathlib import Path\n", "\n", @@ -187,7 +186,7 @@ " \"column_name_mapping\": {\n", " \"text\": \"text_data\"\n", " },\n", - " \"n_rows_to_load\": 10\n", + " \"n_rows_to_load\": 100\n", " }\n", ")\n", "\n", @@ -302,8 +301,8 @@ "metadata": {}, "outputs": [], "source": [ - "from fondant.compiler import DockerCompiler\n", - "from fondant.runner import DockerRunner\n", + "from fondant.pipeline.compiler import DockerCompiler\n", + "from fondant.pipeline.runner import DockerRunner\n", "\n", "DockerCompiler().compile(pipeline, output_path=\"docker-compose.yaml\")\n", "DockerRunner().run(\"docker-compose.yaml\")" @@ -324,7 +323,7 @@ "metadata": {}, "outputs": [], "source": [ - "from fondant.explorer import run_explorer_app\n", + "from fondant.explore import run_explorer_app\n", "\n", "run_explorer_app(\n", " base_path=BASE_PATH,\n", @@ -373,7 +372,7 @@ "%%writefile components/text_cleaning/fondant_component.yaml\n", "name: Text cleaning component\n", "description: Clean text passages\n", - "image: text-cleaning-component:latest\n", + "image: ghcr.io/ml6team/text_cleaning:dev\n", "\n", "consumes:\n", " text:\n", @@ -412,7 +411,7 @@ "logger = logging.getLogger(__name__)\n", "\n", "\n", - "class TextCleaningComponent(PandasTransformComponent): \n", + "class TextCleaningComponent(PandasTransformComponent):\n", " def __init__(self, *_):\n", " \"\"\"Initialize your component\"\"\"\n", "\n", @@ -422,7 +421,9 @@ " return \"\\n\".join(non_empty_lines)\n", "\n", " def transform(self, dataframe: pd.DataFrame) -> pd.DataFrame:\n", - " dataframe[(\"text\", \"data\")] = dataframe[(\"text\", \"data\")].apply(lambda x: self.remove_empty_lines)\n", + " dataframe[(\"text\", \"data\")] = dataframe[(\"text\", \"data\")].apply(\n", + " lambda x: self.remove_empty_lines\n", + " )\n", " return dataframe" ] }, @@ -477,7 +478,7 @@ "outputs": [], "source": [ "%%writefile components/text_cleaning/requirements.txt\n", - "fondant[component]==0.6.2" + "fondant[component]==0.7.0" ] }, { @@ -582,7 +583,7 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -596,9 +597,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.10.12" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/src/requirements.txt b/src/requirements.txt deleted file mode 100644 index e2c6cf9..0000000 --- a/src/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -fondant[docker]==0.6.2