From c8e7617c6550d26f4b26da435fa28962fccca532 Mon Sep 17 00:00:00 2001 From: NIne-WIngEd Date: Wed, 19 Aug 2026 21:34:20 -0500 Subject: [PATCH] Implement multiclass classifiers with CuPy Signed-off-by: NIne-WIngEd --- python/cuml/cuml/multiclass/multiclass.py | 319 ++++++++++++++++++---- python/cuml/tests/test_multiclass.py | 114 +++++++- 2 files changed, 375 insertions(+), 58 deletions(-) diff --git a/python/cuml/cuml/multiclass/multiclass.py b/python/cuml/cuml/multiclass/multiclass.py index 5b5c8e5dbb..e9db631496 100644 --- a/python/cuml/cuml/multiclass/multiclass.py +++ b/python/cuml/cuml/multiclass/multiclass.py @@ -1,17 +1,28 @@ # SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # +import warnings + import cupy as cp +import cupyx.scipy.sparse as cp_sp from cuml.common.doc_utils import generate_docstring from cuml.internals.base import Base from cuml.internals.mixins import ClassifierMixin -from cuml.internals.outputs import exit_internal_context, mlfunc -from cuml.internals.validation import check_inputs +from cuml.internals.outputs import ClassLabels, mlfunc +from cuml.internals.validation import check_inputs, check_is_fitted + + +class _ConstantPredictor: + def predict(self, X): + return cp.zeros(X.shape[0], dtype=cp.int32) + + def decision_function(self, X): + return cp.zeros(X.shape[0], dtype=cp.int32) class _BaseMulticlassClassifier(ClassifierMixin, Base): - """Shared base class for multiclass classifiers""" + """Shared base class for multiclass classifiers.""" def __init__( self, @@ -29,25 +40,68 @@ def _get_param_names(cls): @property def classes_(self): - return self.multiclass_estimator.classes_ + check_is_fitted(self) + return self._classes - @generate_docstring(y="dense_anydtype") - @mlfunc(set_input_type=True) - def fit(self, X, y) -> "_BaseMulticlassClassifier": - """ - Fit a multiclass classifier. - """ - import sklearn.multiclass + @staticmethod + def _predict_binary(est, X): + from sklearn.base import is_regressor - opts = { - "ovo": sklearn.multiclass.OneVsOneClassifier, - "ovr": sklearn.multiclass.OneVsRestClassifier, - } - if (cls := opts.get(self.strategy)) is None: - raise ValueError( - f"Expected `strategy` to be one of {list(opts)}, got {self.strategy}" + if is_regressor(est): + return cp.asarray(est.predict(X)).ravel() + + try: + return cp.asarray(est.decision_function(X)).ravel() + except (AttributeError, NotImplementedError): + return cp.asarray(est.predict_proba(X))[:, 1] + + @staticmethod + def _threshold_for_binary_predict(est): + from sklearn.base import is_classifier + + if hasattr(est, "decision_function") and is_classifier(est): + return 0.0 + return 0.5 + + def _fit_ovr(self, X, y): + from sklearn.base import clone + + X, y, self._classes = check_inputs( + self, + X, + y, + dtype=("float32", "float64"), + y_dtype=None, + accept_sparse=True, + reset=True, + mem_type="device", + return_classes=True, + ) + + n_cls = len(self._classes) + + if n_cls == 1: + warnings.warn( + f"Label not {self._classes[0]} is present in all " + "training examples.", + stacklevel=2, ) - X, y = check_inputs( + self.estimators_ = [_ConstantPredictor()] + return self + + ids = (1,) if n_cls == 2 else range(n_cls) + + self.estimators_ = [ + clone(self.estimator).fit(X, (y == i).astype(cp.int32)) + for i in ids + ] + return self + + def _fit_ovo(self, X, y): + from sklearn.base import clone + from sklearn.utils import get_tags + + X, y, self._classes = check_inputs( self, X, y, @@ -55,15 +109,81 @@ def fit(self, X, y) -> "_BaseMulticlassClassifier": y_dtype=None, accept_sparse=True, reset=True, - mem_type="host", + mem_type="device", + return_classes=True, ) - with exit_internal_context(): - wrapper = cls(self.estimator, n_jobs=None).fit(X, y) + n_cls = len(self._classes) + + if n_cls == 1: + raise ValueError( + "OneVsOneClassifier can not be fit when only one class is " + "present." + ) + + pw = get_tags(self.estimator).input_tags.pairwise + + if cp_sp.issparse(X): + X = X.tocsr() + + self.estimators_ = [] + self.pairwise_indices_ = [] if pw else None + + for i in range(n_cls): + for j in range(i + 1, n_cls): + idx = cp.flatnonzero((y == i) | (y == j)) + Xi = X[idx] + + if pw: + Xi = Xi[:, idx] + self.pairwise_indices_.append(idx) + + yi = (y[idx] == j).astype(cp.int32) + + self.estimators_.append(clone(self.estimator).fit(Xi, yi)) - self.multiclass_estimator = wrapper return self + @staticmethod + def _ovr_decision_function(pred, score, n_cls): + n = pred.shape[0] + + conf = cp.zeros((n, n_cls)) + votes = cp.zeros((n, n_cls)) + + k = 0 + + for i in range(n_cls): + for j in range(i + 1, n_cls): + conf[:, i] -= score[:, k] + conf[:, j] += score[:, k] + + votes[pred[:, k] == 0, i] += 1 + votes[pred[:, k] == 1, j] += 1 + + k += 1 + + conf /= 3 * (cp.abs(conf) + 1) + + return votes + conf + + @generate_docstring(y="dense_anydtype") + @mlfunc(set_input_type=True) + def fit(self, X, y) -> "_BaseMulticlassClassifier": + """ + Fit a multiclass classifier. + """ + if self.strategy == "ovr": + return self._fit_ovr(X, y) + + if self.strategy == "ovo": + return self._fit_ovo(X, y) + + raise ValueError( + f"Expected `strategy` to be one of ['ovo', 'ovr'], " + f"got {self.strategy}" + ) + @generate_docstring( return_values={ "name": "preds", @@ -77,16 +197,45 @@ def predict(self, X): """ Predict using multi class classifier. """ - X = check_inputs( - self, - X, - dtype=("float32", "float64"), - accept_sparse=True, - mem_type="host", - ) + check_is_fitted(self) + + if self.strategy == "ovr": + X = check_inputs( + self, + X, + dtype=("float32", "float64"), + accept_sparse=True, + mem_type="device", + ) - with exit_internal_context(): - return cp.asarray(self.multiclass_estimator.predict(X)) + if len(self.estimators_) == 1: + est = self.estimators_[0] + scr = self._predict_binary(est, X) + cut = self._threshold_for_binary_predict(est) + idx = (scr > cut).astype(cp.intp) + else: + scr = cp.column_stack( + [self._predict_binary(est, X) for est in self.estimators_] + ) + idx = cp.argmax(scr, axis=1) + + return ClassLabels(idx, self._classes) + + if self.strategy == "ovo": + scr = self.decision_function(X) + + if len(self._classes) == 2: + cut = self._threshold_for_binary_predict(self.estimators_[0]) + idx = (scr > cut).astype(cp.intp) + else: + idx = cp.argmax(scr, axis=1) + + return ClassLabels(idx, self._classes) + + raise ValueError( + f"Expected `strategy` to be one of ['ovo', 'ovr'], " + f"got {self.strategy}" + ) @generate_docstring( return_values={ @@ -101,28 +250,86 @@ def decision_function(self, X): """ Calculate the decision function. """ - X = check_inputs( - self, - X, - dtype=("float32", "float64"), - accept_sparse=True, - mem_type="host", + check_is_fitted(self) + + if self.strategy == "ovr": + X = check_inputs( + self, + X, + dtype=("float32", "float64"), + accept_sparse=True, + mem_type="device", + ) + + if len(self.estimators_) == 1: + return cp.asarray( + self.estimators_[0].decision_function(X) + ).ravel() + + return cp.column_stack( + [ + cp.asarray(est.decision_function(X)).ravel() + for est in self.estimators_ + ] + ) + + if self.strategy == "ovo": + X = check_inputs( + self, + X, + dtype=("float32", "float64"), + accept_sparse=True, + mem_type="device", + ) + + ids = self.pairwise_indices_ + + if ids is None: + Xs = [X] * len(self.estimators_) + else: + Xs = [X[:, idx] for idx in ids] + + pred = [] + conf = [] + + for est, Xi in zip(self.estimators_, Xs): + p = est.predict(Xi) + + if isinstance(p, ClassLabels): + p = p.indices + + pred.append(cp.asarray(p).ravel()) + conf.append(self._predict_binary(est, Xi)) + + pred = cp.column_stack(pred) + conf = cp.column_stack(conf) + + scr = self._ovr_decision_function( + pred, + conf, + len(self._classes), + ) + + if len(self._classes) == 2: + return scr[:, 1] + + return scr + + raise ValueError( + f"Expected `strategy` to be one of ['ovo', 'ovr'], " + f"got {self.strategy}" ) - with exit_internal_context(): - return cp.asarray(self.multiclass_estimator.decision_function(X)) class OneVsRestClassifier(_BaseMulticlassClassifier): """ - Wrapper around Sckit-learn's class with the same name. The input can be - any kind of cuML compatible array, and the output type follows cuML's - output type configuration rules. + One-vs-rest multiclass classifier using device-resident multiclass + orchestration. The input can be any kind of cuML compatible array, and + the output type follows cuML's output type configuration rules. - Before passing the data to scikit-learn, it is converted to host (numpy) - array. Under the hood the data is partitioned for binary classification, - and it is transformed back to the device by the cuML estimator. These - copies back and forth the device and the host have some overhead. For more - details see issue https://github.com/rapidsai/cuml/issues/2876. + The data and generated binary targets remain on the device while fitting + the underlying binary estimators, avoiding unnecessary device-to-host + transfers. For documentation see `scikit-learn's OneVsRestClassifier `_. @@ -161,15 +368,13 @@ class OneVsRestClassifier(_BaseMulticlassClassifier): class OneVsOneClassifier(_BaseMulticlassClassifier): """ - Wrapper around Sckit-learn's class with the same name. The input can be - any kind of cuML compatible array, and the output type follows cuML's - output type configuration rules. - - Before passing the data to scikit-learn, it is converted to host (numpy) - array. Under the hood the data is partitioned for binary classification, - and it is transformed back to the device by the cuML estimator. These - copies back and forth the device and the host have some overhead. For more - details see issue https://github.com/rapidsai/cuml/issues/2876. + One-vs-one multiclass classifier using device-resident multiclass + orchestration. The input can be any kind of cuML compatible array, and + the output type follows cuML's output type configuration rules. + + Each pairwise binary problem is constructed on the device and the + resulting votes and confidence values are combined with CuPy, avoiding + unnecessary device-to-host transfers. For documentation see `scikit-learn's OneVsOneClassifier `_. diff --git a/python/cuml/tests/test_multiclass.py b/python/cuml/tests/test_multiclass.py index a31a7d8fd2..231570dbb1 100644 --- a/python/cuml/tests/test_multiclass.py +++ b/python/cuml/tests/test_multiclass.py @@ -1,14 +1,126 @@ -# SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # +import cupy as cp import numpy as np import pytest +from sklearn import multiclass as sk_multiclass +from sklearn.base import BaseEstimator +from sklearn.exceptions import NotFittedError from cuml import LogisticRegression as cuLog from cuml import multiclass as cu_multiclass from cuml.testing.datasets import make_classification_dataset +class _DeviceOnlyClassifier(BaseEstimator): + def fit(self, X, y): + self.dev_ = isinstance(X, cp.ndarray) and isinstance(y, cp.ndarray) + self.rows_ = X.shape[0] + self.bin_ = bool(cp.all((y == 0) | (y == 1))) + return self + + +@pytest.mark.parametrize( + ("cls", "n_est", "n_rows"), + [ + (cu_multiclass.OneVsRestClassifier, 4, 8), + (cu_multiclass.OneVsOneClassifier, 6, 4), + ], +) +def test_multiclass_device_fit(cls, n_est, n_rows): + X = cp.asarray( + [ + [-4.0, -1.0], + [-3.0, -1.0], + [-1.0, 1.0], + [0.0, 1.0], + [2.0, 2.0], + [3.0, 2.0], + [5.0, 3.0], + [6.0, 3.0], + ], + dtype=cp.float32, + ) + y = cp.asarray([0, 0, 1, 1, 2, 2, 3, 3], dtype=cp.int32) + + m = cls(_DeviceOnlyClassifier()).fit(X, y) + + assert len(m.estimators_) == n_est + assert all(est.dev_ for est in m.estimators_) + assert all(est.rows_ == n_rows for est in m.estimators_) + assert all(est.bin_ for est in m.estimators_) + + +@pytest.mark.parametrize("num_classes", [2, 3]) +@pytest.mark.parametrize( + ("cu_cls", "sk_cls"), + [ + ( + cu_multiclass.OneVsRestClassifier, + sk_multiclass.OneVsRestClassifier, + ), + ( + cu_multiclass.OneVsOneClassifier, + sk_multiclass.OneVsOneClassifier, + ), + ], +) +def test_multiclass_sklearn_parity(cu_cls, sk_cls, num_classes): + Xtr, Xte, ytr, _ = make_classification_dataset( + datatype=np.float32, + nrows=400, + ncols=10, + n_info=4, + num_classes=num_classes, + ) + ytr = ytr.astype(np.float32) + + cu = cu_cls(cuLog()).fit(Xtr, ytr) + sk = sk_cls(cuLog()).fit(Xtr, ytr) + + np.testing.assert_array_equal(cu.predict(Xte), sk.predict(Xte)) + np.testing.assert_allclose( + cu.decision_function(Xte), + sk.decision_function(Xte), + rtol=1e-4, + atol=1e-3, + ) + + +@pytest.mark.parametrize( + "cls", + [ + cu_multiclass.OneVsRestClassifier, + cu_multiclass.OneVsOneClassifier, + ], +) +def test_multiclass_not_fitted(cls): + m = cls(cuLog()) + X = cp.zeros((2, 2), dtype=cp.float32) + + with pytest.raises(NotFittedError): + _ = m.classes_ + + with pytest.raises(NotFittedError): + m.predict(X) + + with pytest.raises(NotFittedError): + m.decision_function(X) + + +def test_ovr_single_class(): + X = cp.arange(12, dtype=cp.float32).reshape(6, 2) + y = cp.full(6, 7, dtype=cp.int32) + + with pytest.warns(UserWarning, match="Label not 7"): + cls = cu_multiclass.OneVsRestClassifier(cuLog()).fit(X, y) + + assert len(cls.estimators_) == 1 + assert bool(cp.all(cls.predict(X) == 7)) + assert bool(cp.all(cls.decision_function(X) == 0)) + + @pytest.mark.parametrize("strategy", ["ovr", "ovo"]) @pytest.mark.parametrize("nrows", [1000]) @pytest.mark.parametrize("num_classes", [3])