diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index 144672ad..d4a4e1c8 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -7,6 +7,10 @@ FROM gcr.io/kaggle-images/python-lightgbm-whl:${BASE_IMAGE_TAG}-${LIGHTGBM_VERSI {{ end }} FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} +#b/415358342: UV reports missing requirements files https://github.com/googlecolab/colabtools/issues/5237 +ENV UV_CONSTRAINT= \ + UV_BUILD_CONSTRAINT= + ADD kaggle_requirements.txt /kaggle_requirements.txt # Freeze existing requirements from base image for critical packages: @@ -27,15 +31,10 @@ RUN uv pip install --system -r /requirements.txt RUN uv pip uninstall --system google-cloud-bigquery-storage # b/394382016: sigstore (dependency of kagglehub) requires a prerelease packages, installing separate. -RUN uv pip install --system --force-reinstall --prerelease=allow kagglehub[pandas-datasets,hf-datasets,signing]>=0.3.9 - -# b/408284143: google-cloud-automl 2.0.0 introduced incompatible API changes, need to pin to 1.0.1 - -# b/408284435: Keras 3.6 broke test_keras.py > test_train > keras.datasets.mnist.load_data() -# See https://github.com/keras-team/keras/commit/dcefb139863505d166dd1325066f329b3033d45a -# Colab base is on Keras 3.8, we have to install the package separately -RUN uv pip install --system google-cloud-automl==1.0.1 google-cloud-aiplatform google-cloud-translate==3.12.1 \ - google-cloud-videointelligence google-cloud-vision google-genai "keras<3.6" +# b/408284143: google-cloud-automl 2.0.0 introduced incompatible API changes, need to pin to 1.0.1, +# installed outside of kaggle_requirements.txt due to requiring an incompatibile version of protobuf. +RUN uv pip install --system --force-reinstall --prerelease=allow kagglehub[pandas-datasets,hf-datasets,signing]>=0.3.12 \ + google-cloud-automl==1.0.1 # uv cannot install this in requirements.txt without --no-build-isolation # to avoid affecting the larger build, we'll post-install it. @@ -43,10 +42,8 @@ RUN uv pip install --no-build-isolation --system "git+https://github.com/Kaggle/ # b/408281617: Torch is adamant that it can not install cudnn 9.3.x, only 9.1.x, but Tensorflow can only support 9.3.x. # This conflict causes a number of package downgrades, which are handled in this command -# b/302136621: Fix eli5 import for learntools RUN uv pip install --system --force-reinstall --extra-index-url https://pypi.nvidia.com "cuml-cu12==25.2.1" \ - "nvidia-cudnn-cu12==9.3.0.75" scipy tsfresh scikit-learn==1.2.2 category-encoders eli5 - + "nvidia-cudnn-cu12==9.3.0.75" RUN uv pip install --system --force-reinstall "pynvjitlink-cu12==0.5.2" # b/385145217 Latest Colab lacks mkl numpy, install it. @@ -56,10 +53,10 @@ RUN uv pip install --system --force-reinstall -i https://pypi.anaconda.org/intel RUN uv pip install --system "tbb>=2022" "libpysal==4.9.2" # b/404590350: Ray and torchtune have conflicting tune cli, we will prioritize torchtune. -RUN uv pip install --system --force-reinstall --no-deps torchtune +# b/415358158: Gensim removed from Colab image to upgrade scipy +RUN uv pip install --system --force-reinstall --no-deps torchtune gensim # Adding non-package dependencies: - ADD clean-layer.sh /tmp/clean-layer.sh ADD patches/nbconvert-extensions.tpl /opt/kaggle/nbconvert-extensions.tpl ADD patches/template_conf.json /opt/kaggle/conf.json @@ -171,13 +168,13 @@ RUN mkdir -p /root/.jupyter && touch /root/.jupyter/jupyter_nbconvert_config.py mkdir -p /etc/ipython/ && echo "c = get_config(); c.IPKernelApp.matplotlib = 'inline'" > /etc/ipython/ipython_config.py && \ /tmp/clean-layer.sh -# Fix to import bq_helper library without downgrading setuptools +# Fix to import bq_helper library without downgrading setuptools and upgrading protobuf RUN mkdir -p ~/src && git clone https://github.com/SohierDane/BigQuery_Helper ~/src/BigQuery_Helper && \ mkdir -p ~/src/BigQuery_Helper/bq_helper && \ mv ~/src/BigQuery_Helper/bq_helper.py ~/src/BigQuery_Helper/bq_helper/__init__.py && \ mv ~/src/BigQuery_Helper/test_helper.py ~/src/BigQuery_Helper/bq_helper/ && \ sed -i 's/)/packages=["bq_helper"])/g' ~/src/BigQuery_Helper/setup.py && \ - uv pip install --system -e ~/src/BigQuery_Helper && \ + uv pip install --system -e ~/src/BigQuery_Helper "protobuf<3.21"&& \ /tmp/clean-layer.sh diff --git a/config.txt b/config.txt index b9cff861..cfe8026a 100644 --- a/config.txt +++ b/config.txt @@ -1,5 +1,5 @@ BASE_IMAGE=us-docker.pkg.dev/colab-images/public/runtime -BASE_IMAGE_TAG=release-colab_20250219-060225_RC01 +BASE_IMAGE_TAG=release-colab_20250404-060113_RC00 LIGHTGBM_VERSION=4.6.0 CUDA_MAJOR_VERSION=12 CUDA_MINOR_VERSION=5 diff --git a/kaggle_requirements.txt b/kaggle_requirements.txt index 845abb55..22b26470 100644 --- a/kaggle_requirements.txt +++ b/kaggle_requirements.txt @@ -20,6 +20,7 @@ arrow bayesian-optimization boto3 catboost +category-encoders cesium comm cytoolz @@ -27,11 +28,12 @@ dask-expr # Older versions of datasets fail with "Loading a dataset cached in a LocalFileSystem is not supported" # https://stackoverflow.com/questions/77433096/notimplementederror-loading-a-dataset-cached-in-a-localfilesystem-is-not-suppor datasets>=2.14.6 -datashader deap dipy docker easyocr +# b/302136621: Fix eli5 import for learntools +eli5 emoji fastcore>=1.7.20 fasttext @@ -42,6 +44,13 @@ fuzzywuzzy geojson # geopandas > v0.14.4 breaks learn tools geopandas==v0.14.4 +gensim +google-cloud-aiplatform +# b/315753846: Unpin translate package. +google-cloud-translate==3.12.1 +google-cloud-videointelligence +google-cloud-vision +google-genai gpxpy h2o haversine @@ -112,12 +121,15 @@ qtconsole ray rgf-python s3fs +scikit-learn==1.2.2 # Scikit-learn accelerated library for x86 scikit-learn-intelex>=2023.0.1 scikit-multilearn scikit-optimize scikit-plot scikit-surprise +# b/415358158: Gensim removed from Colab image to upgrade scipy to 1.14.1 +scipy==1.15.1 # Also pinning seaborn for learntools seaborn==0.12.2 git+https://github.com/facebookresearch/segment-anything.git diff --git a/tests/test_automl.py b/tests/test_automl.py index 63c34c69..9a048b14 100644 --- a/tests/test_automl.py +++ b/tests/test_automl.py @@ -8,7 +8,9 @@ def _make_credentials(): import google.auth.credentials - return Mock(spec=google.auth.credentials.Credentials) + credentials = Mock(spec=google.auth.credentials.Credentials) + credentials.universe_domain = 'googleapis.com' + return credentials class TestAutoMl(unittest.TestCase): diff --git a/tests/test_datashader.py b/tests/test_datashader.py deleted file mode 100644 index ad3afe15..00000000 --- a/tests/test_datashader.py +++ /dev/null @@ -1,42 +0,0 @@ -import unittest - -from common import p100_exempt - -class TestDatashader(unittest.TestCase): - - @p100_exempt # b/342143152: Uses cuDF(>=24.4v), which is no longer capitble with p100 GPUs. - def test_pipeline(self): - # based on https://github.com/pyviz/datashader/blob/master/datashader/tests/test_pipeline.py - import numpy as np - import pandas as pd - import datashader as ds - import datashader.transfer_functions as tf - - df = pd.DataFrame({ - 'x': np.array(([0.] * 10 + [1] * 10)), - 'y': np.array(([0.] * 5 + [1] * 5 + [0] * 5 + [1] * 5)), - 'f64': np.arange(20, dtype='f8') - }) - df.f64.iloc[2] = np.nan - - cvs = ds.Canvas(plot_width=2, plot_height=2, x_range=(0, 1), y_range=(0, 1)) - - pipeline = ds.Pipeline(df, ds.Point('x', 'y')) - img = pipeline((0, 1), (0, 1), 2, 2) - agg = cvs.points(df, 'x', 'y', ds.count()) - self.assertTrue(img.equals(tf.shade(agg))) - - color_fn = lambda agg: tf.shade(agg, 'pink', 'red') - pipeline.color_fn = color_fn - img = pipeline((0, 1), (0, 1), 2, 2) - self.assertTrue(img.equals(color_fn(agg))) - - transform_fn = lambda agg: agg + 1 - pipeline.transform_fn = transform_fn - img = pipeline((0, 1), (0, 1), 2, 2) - self.assertTrue(img.equals(color_fn(transform_fn(agg)))) - - pipeline = ds.Pipeline(df, ds.Point('x', 'y'), ds.sum('f64')) - img = pipeline((0, 1), (0, 1), 2, 2) - agg = cvs.points(df, 'x', 'y', ds.sum('f64')) - self.assertTrue(img.equals(tf.shade(agg))) diff --git a/tests/test_gcs.py b/tests/test_gcs.py index eb15ea5f..94da58c9 100644 --- a/tests/test_gcs.py +++ b/tests/test_gcs.py @@ -8,7 +8,9 @@ def _make_credentials(): import google.auth.credentials - return Mock(spec=google.auth.credentials.Credentials) + credentials = Mock(spec=google.auth.credentials.Credentials) + credentials.universe_domain = 'googleapis.com' + return credentials class TestStorage(unittest.TestCase): diff --git a/tests/test_keras.py b/tests/test_keras.py index 22cb6f9f..5dc4610d 100644 --- a/tests/test_keras.py +++ b/tests/test_keras.py @@ -9,10 +9,11 @@ class TestKeras(unittest.TestCase): def test_train(self): - # Load the data and split it between train and test sets - (x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data( - path='/input/tests/data/mnist.npz' - ) + path = '/input/tests/data/mnist.npz' + with np.load(path) as f: + x_train, y_train = f['x_train'], f['y_train'] + x_test, y_test = f['x_test'], f['y_test'] + # Scale images to the [0, 1] range x_train = x_train.astype("float32") / 255