From cb366301da0df429ed532eb5434ea3dfd6d8fd64 Mon Sep 17 00:00:00 2001 From: ambroiseodt Date: Thu, 9 Jan 2025 16:14:24 +0100 Subject: [PATCH 01/11] Add new scorer: MaNoScorer --- README.md | 12 ++-- docs/source/all.rst | 3 +- skada/metrics.py | 117 +++++++++++++++++++++++++++++++++++++ skada/tests/test_scorer.py | 28 +++++++++ 4 files changed, 154 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3dae92fe..06be2658 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ The following algorithms are currently implemented. Any methods that can be cast as an adaptation of the input data can be used in one of two ways: - a scikit-learn transformer (Adapter) which provides both a full Classifier/Regressor estimator - - or an `Adapter` that can be used in a DA pipeline with `make_da_pipeline`. + - or an `Adapter` that can be used in a DA pipeline with `make_da_pipeline`. Refer to the examples below and visit [the gallery](https://scikit-adaptation.github.io/auto_examples/index.html)for more details. ### Deep learning domain adaptation algorithms @@ -84,7 +84,7 @@ details, please refer to this [example](https://scikit-adaptation.github.io/auto First, the DA data in the SKADA API is stored in the following format: ```python -X, y, sample_domain +X, y, sample_domain ``` Where `X` is the input data, `y` is the target labels and `sample_domain` is the @@ -112,7 +112,7 @@ pipe = make_da_pipeline(StandardScaler(), CORALAdapter(), LogisticRegression()) pipe.fit(X, y, sample_domain=sample_domain) # sample_domain passed by name ``` -Please note that for `Adapter` classes that implement sample reweighting, the +Please note that for `Adapter` classes that implement sample reweighting, the subsequent classifier/regressor must require sample_weights as input. This is done with the `set_fit_requires` method. For instance, with `LogisticRegression`, you would use `LogisticRegression().set_fit_requires('sample_weight')`: @@ -143,7 +143,7 @@ cv = SourceTargetShuffleSplit() scorer = PredictionEntropyScorer() # cross val score -scores = cross_val_score(pipe, X, y, params={'sample_domain': sample_domain}, +scores = cross_val_score(pipe, X, y, params={'sample_domain': sample_domain}, cv=cv, scoring=scorer) # grid search @@ -239,8 +239,10 @@ The library is distributed under the 3-Clause BSD license. [33] Kang, G., Jiang, L., Yang, Y., & Hauptmann, A. G. (2019). [Contrastive Adaptation Network for Unsupervised Domain Adaptation](https://arxiv.org/abs/1901.00976). In Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (pp. 4893-4902). -[34] Jin, Ying, Wang, Ximei, Long, Mingsheng, Wang, Jianmin. [Minimum Class Confusion for Versatile Domain Adaptation](https://arxiv.org/pdf/1912.03699). ECCV, 2020. +[34] Jin, Ying, Wang, Ximei, Long, Mingsheng, Wang, Jianmin. [Minimum Class Confusion for Versatile Domain Adaptation](https://arxiv.org/pdf/1912.03699). ECCV, 2020. [35] Zhang, Y., Liu, T., Long, M., & Jordan, M. I. (2019). [Bridging Theory and Algorithm for Domain Adaptation](https://arxiv.org/abs/1904.05801). In Proceedings of the 36th International Conference on Machine Learning, (pp. 7404-7413). [36] Xiao, Zhiqing, Wang, Haobo, Jin, Ying, Feng, Lei, Chen, Gang, Huang, Fei, Zhao, Junbo.[SPA: A Graph Spectral Alignment Perspective for Domain Adaptation](https://arxiv.org/pdf/2310.17594). In Neurips, 2023. + +[37] Xie, Renchunzi, Odonnat, Ambroise, Feofanov, Vasilii, Deng, Weijian, Zhang, Jianfeng and An, Bo. [MaNo: Exploiting Matrix Norm for Unsupervised Accuracy Estimation Under Distribution Shifts](https://arxiv.org/pdf/2405.18979). In NeurIPS, 2024. \ No newline at end of file diff --git a/docs/source/all.rst b/docs/source/all.rst index 1df6de32..a562929d 100644 --- a/docs/source/all.rst +++ b/docs/source/all.rst @@ -13,7 +13,7 @@ Main module :py:mod:`skada` :no-members: :no-inherited-members: -Sample reweighting DA methods +Sample reweighting DA methods ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. .. autosummary:: @@ -211,6 +211,7 @@ DA metrics :py:mod:`skada.metrics` SoftNeighborhoodDensity CircularValidation MixValScorer + MaNoScorer Model Selection :py:mod:`skada.model_selection` diff --git a/skada/metrics.py b/skada/metrics.py index 821ab10f..f6da7b2c 100644 --- a/skada/metrics.py +++ b/skada/metrics.py @@ -2,6 +2,7 @@ # Remi Flamary # Oleksii Kachaiev # Yanis Lalou +# Ambroise Odonnat # # License: BSD 3-Clause @@ -782,3 +783,119 @@ def _score(self, estimator, X, y=None, sample_domain=None, **params): ice_score = ice_diff return self._sign * ice_score + + +class MaNoScorer(_BaseDomainAwareScorer): + """ + MaNo scorer inspired by [37]_, an approach for unsupervised accuracy estimation. + + This scorer used the model's predictions on target data to estimate + the accuracy of the model. The original implementation in [37]_ is + tailored to neural networks and consist of three steps: + 1) Recover target logits (inference step), + 2) Normalize them as probabilities (e.g., with softmax), + 3) Aggregate by averaging the p-norms of the target normalized logits. + + To ensure compatibility with any estimator, we adapt the original implementation. + If the estimator is a neural network, follow 1) --> 2) --> 3) like in [37]_. + Else, directly use the probabilities predicted by the estimator and then do 3). + + See [37]_ for details. + + Parameters + ---------- + p : int, default=4 + Order for the p-norm normalization. + threshold : int, default=5 + Threshold value to determine which normalization to use + See Eq.(6) of [37]_ for more details. + greater_is_better : bool, default=True + Whether higher scores are better. + + Returns + ------- + score : float in [0, 1]. + + References + ---------- + .. [37] Renchunzi Xie et al. MaNo: Matrix Norm for Unsupervised Accuracy Estimation + under Distribution Shifts. + In NeurIPS, 2024. + """ + + def __init__(self, p=4, threshold=5, greater_is_better=True): + super().__init__() + self.p = p + self.threshold = threshold + self._sign = 1 if greater_is_better else -1 + + def _get_criterion(self, logits): + """ + Compute criterion to select the proper normalization. + See Eq.(6) of [1]_ for more details. + """ + proba = self._stable_softmax(logits) + proba = np.log(proba) + divergence = -np.mean(proba) + + return divergence + + def _stable_softmax(self, logits): + """Compute softmax function.""" + logits -= np.max(logits, axis=1, keepdims=True) + exp_logits = np.exp(logits) + exp_logits /= np.sum(exp_logits, axis=1, keepdims=True) + return exp_logits + + def _taylor_softmax(self, logits): + """Compute Taylor approximation of order 2 of softmax.""" + tay_logits = 1 + logits + logits**2 / 2 + tay_logits -= np.min(tay_logits, axis=1, keepdim=True) + tay_logits /= np.sum(tay_logits, axis=1, keepdims=True) + return tay_logits + + def _softrun(self, logits, criterion, threshold): + """Normalize the logits following Eq.(6) of [37]_.""" + if criterion > threshold: + # Apply softmax normalization + outputs = self._stable_softmax(logits) + + else: + # Apply Taylor approximation + outputs = self._taylor_softmax(logits) + + return outputs + + def _score(self, estimator, X, y, sample_domain=None, **params): + if not hasattr(estimator, "predict_proba"): + raise AttributeError( + "The estimator passed should have a 'predict_proba' method. " + f"The estimator {estimator!r} does not." + ) + + X, y, sample_domain = check_X_y_domain(X, y, sample_domain, allow_nd=True) + source_idx = extract_source_indices(sample_domain) + + # Check from y values if it is a classification problem + y_type = _find_y_type(y) + if y_type != Y_Type.DISCRETE: + raise ValueError("MaNo scorer only supports classification problems.") + + # Recover target probabilities + if not isinstance(estimator, BaseEstimator): + # 1) Recover logits on target + logits = estimator.module_(X[~source_idx], **params) + + # 2) Normalize logits to obtain probabilities + criterion = self._get_criterion(logits) + proba = self._softrun(logits=logits, criterion=criterion) + else: + # Directly recover predicted probabilities + proba = estimator.predict_proba( + X[~source_idx], sample_domain=sample_domain[~source_idx], **params + ) + + # 3) Aggregate following Eq.(2) of [37]_. + score = np.mean(proba**self.p) ** (1 / self.p) + + return self._sign * score diff --git a/skada/tests/test_scorer.py b/skada/tests/test_scorer.py index 328a605f..c9b48ee8 100644 --- a/skada/tests/test_scorer.py +++ b/skada/tests/test_scorer.py @@ -2,6 +2,7 @@ # Remi Flamary # Oleksii Kachaiev # Yanis Lalou +# Ambroise Odonnat # # License: BSD 3-Clause @@ -23,6 +24,7 @@ CircularValidation, DeepEmbeddedValidation, ImportanceWeightedScorer, + MaNoScorer, MixValScorer, PredictionEntropyScorer, SoftNeighborhoodDensity, @@ -92,6 +94,7 @@ def test_supervised_scorer(da_dataset): [ PredictionEntropyScorer(), SoftNeighborhoodDensity(), + MaNoScorer(), ], ) def test_scorer_with_entropy_requires_predict_proba(scorer, da_dataset): @@ -351,6 +354,30 @@ def test_mixval_scorer_regression(da_reg_dataset): scorer(estimator, X, y, sample_domain) +def test_mano_scorer(da_dataset): + X, y, sample_domain = da_dataset.pack_train(as_sources=["s"], as_targets=["t"]) + estimator = make_da_pipeline( + DensityReweightAdapter(), + LogisticRegression().set_fit_request(sample_weight=True), + ) + + estimator.fit(X, y, sample_domain=sample_domain) + + scorer = MaNoScorer() + score_mean = scorer._score(estimator, X, y, sample_domain=sample_domain) + assert isinstance(score_mean, float), "score_mean is not a float" + + +def test_mano_scorer_regression(da_reg_dataset): + X, y, sample_domain = da_reg_dataset.pack(as_sources=["s"], as_targets=["t"]) + + estimator = make_da_pipeline(DensityReweightAdapter(), LinearRegression()) + + scorer = MixValScorer(alpha=0.55, random_state=42) + with pytest.raises(ValueError): + scorer(estimator, X, y, sample_domain) + + @pytest.mark.parametrize( "scorer", [ @@ -360,6 +387,7 @@ def test_mixval_scorer_regression(da_reg_dataset): SoftNeighborhoodDensity(), CircularValidation(), MixValScorer(alpha=0.55, random_state=42), + MaNoScorer(), ], ) def test_scorer_with_nd_input(scorer, da_dataset): From 540d5219d2b07d28dd0be7f961cb07139181d235 Mon Sep 17 00:00:00 2001 From: ambroiseodt Date: Fri, 10 Jan 2025 11:55:54 +0100 Subject: [PATCH 02/11] Update MaNoScorer for deep NN case --- skada/metrics.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/skada/metrics.py b/skada/metrics.py index f6da7b2c..6893c5e0 100644 --- a/skada/metrics.py +++ b/skada/metrics.py @@ -806,8 +806,10 @@ class MaNoScorer(_BaseDomainAwareScorer): ---------- p : int, default=4 Order for the p-norm normalization. + It must be non-negative. threshold : int, default=5 - Threshold value to determine which normalization to use + Threshold value to determine which normalization to use. + If threshold <= 0, softmax normalization is always used. See Eq.(6) of [37]_ for more details. greater_is_better : bool, default=True Whether higher scores are better. @@ -829,6 +831,9 @@ def __init__(self, p=4, threshold=5, greater_is_better=True): self.threshold = threshold self._sign = 1 if greater_is_better else -1 + if self.p <= 0: + raise ValueError("The order of the p-norm must be positive") + def _get_criterion(self, logits): """ Compute criterion to select the proper normalization. @@ -850,8 +855,9 @@ def _stable_softmax(self, logits): def _taylor_softmax(self, logits): """Compute Taylor approximation of order 2 of softmax.""" tay_logits = 1 + logits + logits**2 / 2 - tay_logits -= np.min(tay_logits, axis=1, keepdim=True) + tay_logits -= np.min(tay_logits, axis=1, keepdims=True) tay_logits /= np.sum(tay_logits, axis=1, keepdims=True) + return tay_logits def _softrun(self, logits, criterion, threshold): @@ -884,11 +890,15 @@ def _score(self, estimator, X, y, sample_domain=None, **params): # Recover target probabilities if not isinstance(estimator, BaseEstimator): # 1) Recover logits on target - logits = estimator.module_(X[~source_idx], **params) + logits = estimator.infer(X[~source_idx], **params).cpu().detach().numpy() # 2) Normalize logits to obtain probabilities criterion = self._get_criterion(logits) - proba = self._softrun(logits=logits, criterion=criterion) + proba = self._softrun( + logits=logits, + criterion=criterion, + threshold=self.threshold, + ) else: # Directly recover predicted probabilities proba = estimator.predict_proba( From 5786d6ad467ba9863b952546c9b6f5d5b4251aab Mon Sep 17 00:00:00 2001 From: ambroiseodt Date: Fri, 10 Jan 2025 11:56:20 +0100 Subject: [PATCH 03/11] Update MNoScorer test: regression, invalid p-norm, both normalization --- skada/tests/test_scorer.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/skada/tests/test_scorer.py b/skada/tests/test_scorer.py index c9b48ee8..653ab2a7 100644 --- a/skada/tests/test_scorer.py +++ b/skada/tests/test_scorer.py @@ -367,13 +367,27 @@ def test_mano_scorer(da_dataset): score_mean = scorer._score(estimator, X, y, sample_domain=sample_domain) assert isinstance(score_mean, float), "score_mean is not a float" + # Test softmax normalization + scorer = MaNoScorer(threshold=-1) + score_mean = scorer._score(estimator, X, y, sample_domain=sample_domain) + assert isinstance(score_mean, float), "score_mean is not a float" + + # Test softmax normalization + scorer = MaNoScorer(threshold=float("inf")) + score_mean = scorer._score(estimator, X, y, sample_domain=sample_domain) + assert isinstance(score_mean, float), "score_mean is not a float" + + # Test invalid p-norm order + with pytest.raises(ValueError): + MaNoScorer(p=-1) + def test_mano_scorer_regression(da_reg_dataset): X, y, sample_domain = da_reg_dataset.pack(as_sources=["s"], as_targets=["t"]) - estimator = make_da_pipeline(DensityReweightAdapter(), LinearRegression()) + estimator = make_da_pipeline(DensityReweightAdapter(), LogisticRegression()) - scorer = MixValScorer(alpha=0.55, random_state=42) + scorer = MaNoScorer() with pytest.raises(ValueError): scorer(estimator, X, y, sample_domain) From e3a3df165dcdb2a674f4050d7a7f6f1d4461e844 Mon Sep 17 00:00:00 2001 From: ambroiseodt Date: Fri, 10 Jan 2025 11:56:32 +0100 Subject: [PATCH 04/11] Add MaNoScorer deep tests --- skada/deep/tests/test_deep_scorer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/skada/deep/tests/test_deep_scorer.py b/skada/deep/tests/test_deep_scorer.py index 2922201e..5725d235 100644 --- a/skada/deep/tests/test_deep_scorer.py +++ b/skada/deep/tests/test_deep_scorer.py @@ -1,4 +1,5 @@ # Author: Yanis Lalou +# Ambroise Odonnat # # License: BSD 3-Clause @@ -16,6 +17,7 @@ CircularValidation, DeepEmbeddedValidation, ImportanceWeightedScorer, + MaNoScorer, MixValScorer, PredictionEntropyScorer, SoftNeighborhoodDensity, @@ -29,6 +31,7 @@ PredictionEntropyScorer(), SoftNeighborhoodDensity(), CircularValidation(), + MaNoScorer(), MixValScorer(), ImportanceWeightedScorer(), ], @@ -66,6 +69,7 @@ def test_generic_scorer_on_deepmodel(scorer, da_dataset): PredictionEntropyScorer(), SoftNeighborhoodDensity(), DeepEmbeddedValidation(), + MaNoScorer(), ], ) def test_generic_scorer(scorer, da_dataset): @@ -101,6 +105,7 @@ def test_generic_scorer(scorer, da_dataset): [ DeepEmbeddedValidation(), ImportanceWeightedScorer(), + MaNoScorer(), ], ) def test_scorer_with_nd_features(scorer, da_dataset): From bbf01435592819a79f1bf307f463825ac8041b37 Mon Sep 17 00:00:00 2001 From: ambroiseodt Date: Tue, 18 Feb 2025 16:47:59 +0100 Subject: [PATCH 05/11] Add static decorator --- skada/metrics.py | 78 +++++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 38 deletions(-) diff --git a/skada/metrics.py b/skada/metrics.py index 6893c5e0..a6ec7702 100644 --- a/skada/metrics.py +++ b/skada/metrics.py @@ -834,44 +834,6 @@ def __init__(self, p=4, threshold=5, greater_is_better=True): if self.p <= 0: raise ValueError("The order of the p-norm must be positive") - def _get_criterion(self, logits): - """ - Compute criterion to select the proper normalization. - See Eq.(6) of [1]_ for more details. - """ - proba = self._stable_softmax(logits) - proba = np.log(proba) - divergence = -np.mean(proba) - - return divergence - - def _stable_softmax(self, logits): - """Compute softmax function.""" - logits -= np.max(logits, axis=1, keepdims=True) - exp_logits = np.exp(logits) - exp_logits /= np.sum(exp_logits, axis=1, keepdims=True) - return exp_logits - - def _taylor_softmax(self, logits): - """Compute Taylor approximation of order 2 of softmax.""" - tay_logits = 1 + logits + logits**2 / 2 - tay_logits -= np.min(tay_logits, axis=1, keepdims=True) - tay_logits /= np.sum(tay_logits, axis=1, keepdims=True) - - return tay_logits - - def _softrun(self, logits, criterion, threshold): - """Normalize the logits following Eq.(6) of [37]_.""" - if criterion > threshold: - # Apply softmax normalization - outputs = self._stable_softmax(logits) - - else: - # Apply Taylor approximation - outputs = self._taylor_softmax(logits) - - return outputs - def _score(self, estimator, X, y, sample_domain=None, **params): if not hasattr(estimator, "predict_proba"): raise AttributeError( @@ -909,3 +871,43 @@ def _score(self, estimator, X, y, sample_domain=None, **params): score = np.mean(proba**self.p) ** (1 / self.p) return self._sign * score + + def _get_criterion(self, logits): + """ + Compute criterion to select the proper normalization. + See Eq.(6) of [1]_ for more details. + """ + proba = self._stable_softmax(logits) + proba = np.log(proba) + divergence = -np.mean(proba) + + return divergence + + def _softrun(self, logits, criterion, threshold): + """Normalize the logits following Eq.(6) of [37]_.""" + if criterion > threshold: + # Apply softmax normalization + outputs = self._stable_softmax(logits) + + else: + # Apply Taylor approximation + outputs = self._taylor_softmax(logits) + + return outputs + + @staticmethod + def _stable_softmax(logits): + """Compute softmax function.""" + logits -= np.max(logits, axis=1, keepdims=True) + exp_logits = np.exp(logits) + exp_logits /= np.sum(exp_logits, axis=1, keepdims=True) + return exp_logits + + @staticmethod + def _taylor_softmax(logits): + """Compute Taylor approximation of order 2 of softmax.""" + tay_logits = 1 + logits + logits**2 / 2 + tay_logits -= np.min(tay_logits, axis=1, keepdims=True) + tay_logits /= np.sum(tay_logits, axis=1, keepdims=True) + + return tay_logits From 0ed73e5ab6fd006f5d393ef74c78e9a465592c44 Mon Sep 17 00:00:00 2001 From: ambroiseodt Date: Thu, 20 Feb 2025 15:39:42 +0100 Subject: [PATCH 06/11] Update metrics to deal with skorch update --- skada/metrics.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/skada/metrics.py b/skada/metrics.py index 68da760c..0a6b27bd 100644 --- a/skada/metrics.py +++ b/skada/metrics.py @@ -12,11 +12,13 @@ import numpy as np from sklearn.base import clone +from sklearn.base import clone from sklearn.linear_model import LogisticRegression from sklearn.metrics import balanced_accuracy_score, check_scoring from sklearn.model_selection import train_test_split from sklearn.neighbors import KernelDensity from sklearn.pipeline import Pipeline +from sklearn.pipeline import Pipeline from sklearn.preprocessing import LabelEncoder, Normalizer from sklearn.utils import check_random_state from sklearn.utils.extmath import softmax @@ -397,6 +399,7 @@ def _score(self, estimator, X, y, sample_domain=None, **kwargs): ) has_transform_method = False + if not isinstance(estimator, Pipeline): # The estimator is a deep model if estimator.module_.layer_name is None: @@ -850,7 +853,7 @@ def _score(self, estimator, X, y, sample_domain=None, **params): raise ValueError("MaNo scorer only supports classification problems.") # Recover target probabilities - if not isinstance(estimator, BaseEstimator): + if not isinstance(estimator, Pipeline): # 1) Recover logits on target logits = estimator.infer(X[~source_idx], **params).cpu().detach().numpy() From 3ac3ea761a94bea9fad627d28fc7de1dd3c237f4 Mon Sep 17 00:00:00 2001 From: ambroiseodt Date: Thu, 20 Feb 2025 15:43:07 +0100 Subject: [PATCH 07/11] Update metrics to match skorch update --- skada/metrics.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/skada/metrics.py b/skada/metrics.py index 0a6b27bd..d6fb2ca5 100644 --- a/skada/metrics.py +++ b/skada/metrics.py @@ -12,13 +12,11 @@ import numpy as np from sklearn.base import clone -from sklearn.base import clone from sklearn.linear_model import LogisticRegression from sklearn.metrics import balanced_accuracy_score, check_scoring from sklearn.model_selection import train_test_split from sklearn.neighbors import KernelDensity from sklearn.pipeline import Pipeline -from sklearn.pipeline import Pipeline from sklearn.preprocessing import LabelEncoder, Normalizer from sklearn.utils import check_random_state from sklearn.utils.extmath import softmax From f28065078da2a9a979ccf3e658f724546512710c Mon Sep 17 00:00:00 2001 From: ambroiseodt Date: Fri, 21 Feb 2025 10:19:07 +0100 Subject: [PATCH 08/11] Update test for MaNo in the deep case --- skada/deep/tests/test_deep_scorer.py | 11 +++++++++-- skada/metrics.py | 5 ++++- skada/tests/test_scorer.py | 1 + 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/skada/deep/tests/test_deep_scorer.py b/skada/deep/tests/test_deep_scorer.py index 5725d235..56dac1f2 100644 --- a/skada/deep/tests/test_deep_scorer.py +++ b/skada/deep/tests/test_deep_scorer.py @@ -196,7 +196,14 @@ def test_dev_scorer_on_source_only(da_dataset): assert ~np.isnan(scores), "The score is computed" -def test_dev_exception_layer_name(da_dataset): +@pytest.mark.parametrize( + "scorer", + [ + DeepEmbeddedValidation(), + MaNoScorer(), + ], +) +def test_dev_exception_layer_name(scorer, da_dataset): X, y, sample_domain = da_dataset.pack_train(as_sources=["s"], as_targets=["t"]) X_test, y_test, sample_domain_test = da_dataset.pack_test(as_targets=["t"]) @@ -214,4 +221,4 @@ def test_dev_exception_layer_name(da_dataset): estimator.fit(X, y, sample_domain=sample_domain) with pytest.raises(ValueError, match="The layer_name of the estimator is not set."): - DeepEmbeddedValidation()(estimator, X, y, sample_domain) + scorer(estimator, X, y, sample_domain) diff --git a/skada/metrics.py b/skada/metrics.py index d6fb2ca5..788cad85 100644 --- a/skada/metrics.py +++ b/skada/metrics.py @@ -850,8 +850,11 @@ def _score(self, estimator, X, y, sample_domain=None, **params): if y_type != Y_Type.DISCRETE: raise ValueError("MaNo scorer only supports classification problems.") - # Recover target probabilities if not isinstance(estimator, Pipeline): + # The estimator is a deep model + if estimator.module_.layer_name is None: + raise ValueError("The layer_name of the estimator is not set.") + # 1) Recover logits on target logits = estimator.infer(X[~source_idx], **params).cpu().detach().numpy() diff --git a/skada/tests/test_scorer.py b/skada/tests/test_scorer.py index 653ab2a7..611c760c 100644 --- a/skada/tests/test_scorer.py +++ b/skada/tests/test_scorer.py @@ -40,6 +40,7 @@ SoftNeighborhoodDensity(), DeepEmbeddedValidation(), CircularValidation(), + MaNoScorer(), ], ) def test_generic_scorer(scorer, da_dataset): From 563c86c41df45c19f48b6d4a3f5cea2c756ca2ef Mon Sep 17 00:00:00 2001 From: ambroiseodt Date: Fri, 21 Feb 2025 11:39:45 +0100 Subject: [PATCH 09/11] Add attribute to detect type of normalization --- skada/metrics.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/skada/metrics.py b/skada/metrics.py index 788cad85..64d9f058 100644 --- a/skada/metrics.py +++ b/skada/metrics.py @@ -817,7 +817,7 @@ class MaNoScorer(_BaseDomainAwareScorer): Returns ------- - score : float in [0, 1]. + score : float in [0, 1] (or [-1, 0] depending on the value of self._sign). References ---------- @@ -831,6 +831,7 @@ def __init__(self, p=4, threshold=5, greater_is_better=True): self.p = p self.threshold = threshold self._sign = 1 if greater_is_better else -1 + self.chosen_normalization = None if self.p <= 0: raise ValueError("The order of the p-norm must be positive") @@ -892,10 +893,11 @@ def _softrun(self, logits, criterion, threshold): if criterion > threshold: # Apply softmax normalization outputs = self._stable_softmax(logits) - + self.chosen_normalization = "softmax" else: # Apply Taylor approximation outputs = self._taylor_softmax(logits) + self.chosen_normalization = "taylor" return outputs From 4c940dd9eddd5099274500a215f26f489de0f458 Mon Sep 17 00:00:00 2001 From: ambroiseodt Date: Fri, 21 Feb 2025 11:40:06 +0100 Subject: [PATCH 10/11] Add pipeline test and choice of normalization --- skada/deep/tests/test_deep_scorer.py | 90 +++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/skada/deep/tests/test_deep_scorer.py b/skada/deep/tests/test_deep_scorer.py index 56dac1f2..a704fd08 100644 --- a/skada/deep/tests/test_deep_scorer.py +++ b/skada/deep/tests/test_deep_scorer.py @@ -203,7 +203,7 @@ def test_dev_scorer_on_source_only(da_dataset): MaNoScorer(), ], ) -def test_dev_exception_layer_name(scorer, da_dataset): +def test_exception_layer_name(scorer, da_dataset): X, y, sample_domain = da_dataset.pack_train(as_sources=["s"], as_targets=["t"]) X_test, y_test, sample_domain_test = da_dataset.pack_test(as_targets=["t"]) @@ -222,3 +222,91 @@ def test_dev_exception_layer_name(scorer, da_dataset): with pytest.raises(ValueError, match="The layer_name of the estimator is not set."): scorer(estimator, X, y, sample_domain) + + +def test_mano_softmax(da_dataset): + X, y, sample_domain = da_dataset.pack_train(as_sources=["s"], as_targets=["t"]) + X_test, y_test, sample_domain_test = da_dataset.pack_test(as_targets=["t"]) + + estimator = DeepCoral( + ToyModule2D(proba=True), + reg=1, + layer_name="dropout", + batch_size=10, + max_epochs=10, + train_split=None, + ) + + X = X.astype(np.float32) + X_test = X_test.astype(np.float32) + + # without dict + estimator.fit(X, y, sample_domain=sample_domain) + + estimator.predict(X_test, sample_domain=sample_domain_test, allow_source=True) + estimator.predict_proba(X, sample_domain=sample_domain, allow_source=True) + + scorer = MaNoScorer(threshold=-1) + scorer(estimator, X, y, sample_domain) + print(scorer.chosen_normalization.lower()) + assert ( + scorer.chosen_normalization.lower() == "softmax" + ), "the wrong normalization was chosen" + + +def test_mano_taylor(da_dataset): + X, y, sample_domain = da_dataset.pack_train(as_sources=["s"], as_targets=["t"]) + X_test, y_test, sample_domain_test = da_dataset.pack_test(as_targets=["t"]) + + estimator = DeepCoral( + ToyModule2D(proba=True), + reg=1, + layer_name="dropout", + batch_size=10, + max_epochs=10, + train_split=None, + ) + + X = X.astype(np.float32) + X_test = X_test.astype(np.float32) + + # without dict + estimator.fit(X, y, sample_domain=sample_domain) + + estimator.predict(X_test, sample_domain=sample_domain_test, allow_source=True) + estimator.predict_proba(X, sample_domain=sample_domain, allow_source=True) + + scorer = MaNoScorer(threshold=float("inf")) + scorer(estimator, X, y, sample_domain) + assert ( + scorer.chosen_normalization.lower() == "taylor" + ), "the wrong normalization was chosen" + + +def test_mano_output_range(da_dataset): + X, y, sample_domain = da_dataset.pack_train(as_sources=["s"], as_targets=["t"]) + X_test, y_test, sample_domain_test = da_dataset.pack_test(as_targets=["t"]) + + estimator = DeepCoral( + ToyModule2D(proba=True), + reg=1, + layer_name="dropout", + batch_size=10, + max_epochs=10, + train_split=None, + ) + + X = X.astype(np.float32) + X_test = X_test.astype(np.float32) + + # without dict + estimator.fit(X, y, sample_domain=sample_domain) + + estimator.predict(X_test, sample_domain=sample_domain_test, allow_source=True) + estimator.predict_proba(X, sample_domain=sample_domain, allow_source=True) + + scorer = MaNoScorer(threshold=float("inf")) + score = scorer(estimator, X, y, sample_domain) + assert (scorer._sign * score >= 0) and ( + scorer._sign * score <= 1 + ), "The output range should be [-1, 0] or [0, 1]." From 93879f4c64e69a1b9ddcb1f49fdc4b23b11b0476 Mon Sep 17 00:00:00 2001 From: ambroiseodt Date: Fri, 21 Feb 2025 11:40:36 +0100 Subject: [PATCH 11/11] Add output range test --- skada/tests/test_scorer.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/skada/tests/test_scorer.py b/skada/tests/test_scorer.py index 611c760c..7ff53d91 100644 --- a/skada/tests/test_scorer.py +++ b/skada/tests/test_scorer.py @@ -382,6 +382,13 @@ def test_mano_scorer(da_dataset): with pytest.raises(ValueError): MaNoScorer(p=-1) + # Test correct output range + scorer = MaNoScorer() + score_mean = scorer._score(estimator, X, y, sample_domain=sample_domain) + assert (scorer._sign * score_mean >= 0) and ( + scorer._sign * score_mean <= 1 + ), "The output range should be [-1, 0] or [0, 1]." + def test_mano_scorer_regression(da_reg_dataset): X, y, sample_domain = da_reg_dataset.pack(as_sources=["s"], as_targets=["t"])