From a3eb9ff9230d9d892d826a356c284e93f3dfb5e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gy=C3=B6rgy=20Kov=C3=A1cs?= Date: Sun, 1 Sep 2024 13:21:08 +0200 Subject: [PATCH] now done --- mlscorecheck/auc/__init__.py | 1 + mlscorecheck/auc/_acc_aggregated.py | 546 ++++-------------------- mlscorecheck/auc/_acc_single.py | 68 +-- mlscorecheck/auc/_auc_aggregated.py | 30 +- mlscorecheck/auc/_auc_single.py | 75 ++-- mlscorecheck/auc/_max_acc_aggregated.py | 443 +++++++++++++++++++ mlscorecheck/auc/_utils.py | 2 +- tests/auc/test_acc_aggregated.py | 153 +++---- tests/auc/test_acc_single.py | 8 + tests/auc/test_auc_aggregated.py | 32 ++ tests/auc/test_auc_single.py | 30 ++ tests/auc/test_max_acc_aggregated.py | 188 ++++++++ 12 files changed, 889 insertions(+), 687 deletions(-) create mode 100644 mlscorecheck/auc/_max_acc_aggregated.py create mode 100644 tests/auc/test_max_acc_aggregated.py diff --git a/mlscorecheck/auc/__init__.py b/mlscorecheck/auc/__init__.py index 06930ae..67718e6 100644 --- a/mlscorecheck/auc/__init__.py +++ b/mlscorecheck/auc/__init__.py @@ -7,3 +7,4 @@ from ._acc_single import * from ._auc_aggregated import * from ._acc_aggregated import * +from ._max_acc_aggregated import * diff --git a/mlscorecheck/auc/_acc_aggregated.py b/mlscorecheck/auc/_acc_aggregated.py index caa142f..bc42278 100644 --- a/mlscorecheck/auc/_acc_aggregated.py +++ b/mlscorecheck/auc/_acc_aggregated.py @@ -11,7 +11,7 @@ from ._utils import prepare_intervals, translate_folding -from ._acc_single import acc_min, acc_max, macc_min +from ._acc_single import acc_min, acc_max from ._auc_aggregated import R, check_cvxopt __all__ = [ @@ -22,13 +22,9 @@ "acc_from_aggregated", "acc_lower_from_aggregated", "acc_upper_from_aggregated", - "macc_min_aggregated", - "max_acc_from_aggregated", - "max_acc_lower_from_aggregated", - "max_acc_upper_from_aggregated" + "FAccRMax" ] -RES = 1e-10 def acc_min_aggregated( auc: float, ps: np.array, ns: np.array, return_solutions: bool = False @@ -184,7 +180,7 @@ def __init__(self, ps: np.array, ns: np.array): self.maxs = np.array([max(p, n) for p, n in zip(ps, ns)]) self.mins = np.array([min(p, n) for p, n in zip(ps, ns)]) self.weights = self.mins / (ps + ns) - self.lower_bounds = 0.5 + 1.0/(ps*ns) + self.lower_bounds = 0.5 + 1.0 / (ps * ns) def __call__(self, x: matrix = None, z: matrix = None): """ @@ -217,7 +213,7 @@ def __call__(self, x: matrix = None, z: matrix = None): z (cvxopt.matrix | None): a weight vector Returns: - (n, matrix): the number of non-linear constraints and a feasible + (int, matrix): the number of non-linear constraints and a feasible point when x is None and z is None (matrix, matrix): the objective value at x, and the gradient at x when x is not None but z is None @@ -229,10 +225,12 @@ def __call__(self, x: matrix = None, z: matrix = None): # The number of non-linear constraints and one point of # the feasible region return (0, matrix(1.0, (self.k, 1))) - - if np.any(np.array(x)[:, 0] <= self.lower_bounds) or np.any(np.array(x) > 1.0): + + if np.any( + np.array(x)[:, 0] <= self.lower_bounds + ): # or np.any(np.array(x) > 1.0): return None, None - #print('error:', np.array(x)) + # print('error:', np.array(x)) # if x is not None: # the function to be evaluated @@ -295,14 +293,14 @@ def acc_rmax_solve( k = ps.shape[0] - lower_bounds = 0.5 + 1.0/(ps*ns) - #upper_bounds = np.repeat(1.0 - np.min(1 / ((ps + 1) * (ns + 1))), k) + lower_bounds = 0.5 + 1.0 / (ps * ns) + # upper_bounds = np.repeat(1.0 - np.min(1 / ((ps + 1) * (ns + 1))), k) A = np.repeat(1.0 / k, k).reshape(-1, 1).T # pylint: disable=invalid-name b = np.array([avg_auc]).astype(float) G = np.vstack([np.eye(k), -np.eye(k)]).astype(float) # pylint: disable=invalid-name h = np.hstack([np.repeat(1.0, k), -lower_bounds]) - #h = np.hstack([upper_bounds, -lower_bounds]) + # h = np.hstack([upper_bounds, -lower_bounds]) G = matrix(G) # pylint: disable=invalid-name h = matrix(h) @@ -311,6 +309,7 @@ def acc_rmax_solve( actual = solvers.options.get("show_progress", None) solvers.options["show_progress"] = False + # solvers.options["maxiters"] = 200 res = cp(F, G, h, A=A, b=b) @@ -328,9 +327,57 @@ def acc_rmax_solve( return results -def adjust_lower_bound(bound): - #return np.ceil(bound * RES) / RES - return bound +def reduce_rmax_edge_case( + auc: float, ps: np.array, ns: np.array, eps: float, lower_bounds: np.array +): + """ + Reduces and solves the edge case problem + + Args: + auc (float): the AUC value + ps (np.array): the numbers of positives + ns (np.array): the numbers of negatives + eps (float): the epsilon + lower_bounds (np.array): the lower bounds + + Returns: + float, np.array, np.array: the result, the aucs and the lower bounds + """ + n_1s = 0 + auc_p = auc + + sorting = np.argsort(lower_bounds) + rev_sorting = np.zeros(len(ps), dtype=int) + rev_sorting[sorting] = np.arange(len(ps)) + + lower_bounds_p = lower_bounds[sorting] + ps_p = ps[sorting] + ns_p = ns[sorting] + + while len(ps_p) > 0 and auc_p <= np.mean(lower_bounds_p) + eps: + n_1s += 1 + ps_p = ps_p[1:] + ns_p = ns_p[1:] + lower_bounds_p = lower_bounds_p[1:] + if len(ps_p) > 0: + auc_p = (auc_p * (len(ps_p) + 1) - 0.5) / len(ps_p) + + if len(ps_p) > 0: + results, (aucs, ps_p, ns_p, lower_bounds_p, _) = acc_rmax_solve( + ps_p, ns_p, auc_p, return_solutions=True + ) + + aucs = np.hstack([np.repeat(0.5, n_1s), aucs])[rev_sorting] + lower_bounds = np.hstack([np.repeat(0.5, n_1s), lower_bounds_p])[rev_sorting] + + results = acc_rmax_evaluate(ps[sorting], ns[sorting], aucs) + else: + # no entries left + # ideally, we should never arrive here + results = np.mean([max(p, n) / (p + n) for p, n in zip(ps, ns)]) + aucs = np.repeat(0.5, len(ps)) + + return results, aucs, lower_bounds def acc_rmax_aggregated( @@ -363,13 +410,13 @@ def acc_rmax_aggregated( k = len(ps) - EPS = np.min(1.0/(ps*ns)) + eps = np.min(1.0 / (ps * ns)) - lower_bounds = 0.5 + 1.0/(ps*ns) + lower_bounds = 0.5 + 1.0 / (ps * ns) upper_bounds = np.repeat(1.0, k) if auc == 0.5: - results = np.mean([max(p, n)/(p + n) for p, n in zip(ps, ns)]) + results = np.mean([max(p, n) / (p + n) for p, n in zip(ps, ns)]) if return_solutions: results = results, ( np.repeat(0.5, len(ps)), @@ -379,285 +426,19 @@ def acc_rmax_aggregated( np.repeat(1.0, len(ps)), ) return results - elif auc <= np.mean(lower_bounds) + EPS: + if auc <= np.mean(lower_bounds) + eps: # sorting the items by the lower_bounds, and eliminating # the smallest ones until the resulting problem becomes feasible - n_1s = 0 - auc_p = auc - - sorting = np.argsort(lower_bounds) - rev_sorting = np.zeros(len(ps), dtype=int) - rev_sorting[sorting] = np.arange(len(ps)) - - upper_bounds_p = upper_bounds[sorting] - lower_bounds_p = lower_bounds[sorting] - ps_p = ps[sorting] - ns_p = ns[sorting] - - while len(ps_p) > 0 and auc_p <= np.mean(lower_bounds_p) + EPS: - n_1s += 1 - ps_p = ps_p[1:] - ns_p = ns_p[1:] - lower_bounds_p = lower_bounds_p[1:] - if len(ps_p) > 0: - auc_p = (auc_p * (len(ps_p) + 1) - 0.5) / len(ps_p) - - if len(ps_p) > 0: - results, (aucs, lower_bounds_p, _) = acc_rmax_solve(ps_p, ns_p, auc_p, return_solutions=True) - - aucs = np.hstack([np.repeat(0.5, n_1s), aucs])[rev_sorting] - lower_bounds = np.hstack([np.repeat(0.5, n_1s), lower_bounds_p])[rev_sorting] - results = acc_rmax_evaluate(ps[sorting], ns[sorting], aucs) - - else: - # no entries left - # ideally, we should never arrive here - results = np.mean([max(p, n)/(p + n) for p, n in zip(ps, ns)]) - aucs = np.repeat(0.5, len(ps)) - - if return_solutions: - return results, (aucs, lower_bounds, upper_bounds) - return results - else: - return acc_rmax_solve(ps, ns, auc, return_solutions) - - -class FMAccMin: # pylint: disable=too-few-public-methods - """ - Implements the convex programming objective for the maximum accuracy - minimization. - """ - - def __init__(self, ps: np.array, ns: np.array): - """ - The constructor of the object - - Args: - ps (np.array): the number of positive samples - ns (np.array): the number of negative samples - """ - self.ps = ps - self.ns = ns - self.k = len(ps) - self.maxs = np.array([max(p, n) for p, n in zip(ps, ns)]) - self.mins = np.array([min(p, n) for p, n in zip(ps, ns)]) - self.weights = np.sqrt(2 * (ps * ns)) / (ps + ns) - self.lower_bounds = 1.0 - np.array( - [min(p, n) / (2 * max(p, n)) for p, n in zip(ps, ns)] + results, aucs, lower_bounds = reduce_rmax_edge_case( + auc, ps, ns, eps, lower_bounds ) - self.upper_bounds = np.repeat(1.0 - np.min(1 / ((ps + 1) * (ns + 1))), self.k) - - def __call__(self, x: matrix = None, z: matrix = None): - """ - The call method according to the specification in cvxopt - - Evaluates the function originating from the objective - - f_0(auc) = dfrac{1}{k} sum limits_{i=1}^{k}macc_min(auc_i, p_i, n_i), - - f_0(auc) = 1/k sum (1 - sqrt(2*p_i*n_i*(1 - auc_i))/(p_i + n_i)) - - by removing the multiplier 1/k and the constants parts. - - Also: - auc = sp.Symbol('auc') - f = sp.sqrt(1 - auc) - sp.diff(f, auc), sp.diff(sp.diff(f, auc), auc) - - (-1/(2*sqrt(1 - auc)), -1/(4*(1 - auc)**(3/2))) - - Args: - x (cvxopt.matrix | None): a vector in the parameter space - z (cvxopt.matrix | None): a weight vector - - Returns: - (n, matrix): the number of non-linear constraints and a feasible - point when x is None and z is None - (matrix, matrix): the objective value at x, and the gradient - at x when x is not None but z is None - (matrix, matrix, matrx): the objective value at x, the gradient - at x and the weighted sum of the Hessian of the objective and - all non-linear constraints with the weights z if z is not None - """ - if x is None and z is None: - return (0, matrix(self.lower_bounds, (self.k, 1))) - # return (0, matrix(np.repeat(1.0, self.k), (self.k, 1))) - - if np.any(np.array(x)[:, 0] > self.upper_bounds) or np.any(np.array(x)[:, 0] < self.lower_bounds): - return None, None - - # if x is not None: - f = matrix(-np.sum(np.clip(np.sqrt(1 - x), 0.0, 2.0) * self.weights.reshape(-1, 1))) - df = matrix(1.0 / (2 * np.clip(np.sqrt(1 - x), 0.0, 2.0)) * self.weights.reshape(-1, 1)).T - - if z is None: - return (f, df) - - hess = np.diag(1.0 / (4 * np.clip(np.array(1 - x), 0.0, 2.0)[:, 0] ** (3 / 2)) * self.weights) - - hess = matrix(z[0] * hess) - - return (f, df, hess) - - -def macc_min_evaluate(ps: np.array, ns: np.array, aucs: np.array): - """ - Evaluates a particular macc_min configuration - - Args: - ps (np.array): the number of positive samples - ns (np.array): the number of negative samples - aucs (np.array): the AUCs - - Returns: - float: the average accuracy - """ - return np.mean([macc_min(auc, p, n) for auc, p, n in zip(aucs, ps, ns)]) - - -def macc_min_solve( - ps: np.array, ns: np.array, avg_auc: float, return_solutions: bool = False -): - """ - Solves the maximum accuracy minimum curves problem - - Args: - ps (np.array): the number of positive samples - ns (np.array): the number of negative samples - avg_auc (np.array): the average AUC - return_solutions (bool): whether to return the solutions and - further details - - Returns: - float | (float, np.array, np.array, np.array): the mean accuracy, - or the mean accuracy, the auc parameters, lower bounds and upper - bounds - - Raises: - ValueError: when no optimal solution is found - """ - F = FMAccMin(ps, ns) # pylint: disable=invalid-name - - k = ps.shape[0] - - lower_bounds = 1.0 - np.array([min(p, n) / (2 * max(p, n)) for p, n in zip(ps, ns)]) - upper_bounds = np.repeat(1.0 - np.min(1 / ((ps + 1) * (ns + 1))), k) - #print(upper_bounds) - - A = np.repeat(1.0 / k, k).reshape(-1, 1).T # pylint: disable=invalid-name - b = np.array([avg_auc]).astype(float) - G = np.vstack([np.eye(k), -np.eye(k)]).astype(float) # pylint: disable=invalid-name - h = np.hstack([upper_bounds, -lower_bounds]) - - G = matrix(G) # pylint: disable=invalid-name - h = matrix(h) - A = matrix(A) # pylint: disable=invalid-name - b = matrix(b) - - actual = solvers.options.get("show_progress", None) - solvers.options["show_progress"] = False - - results = cp(F, G, h, A=A, b=b) - - solvers.options["show_progress"] = actual - - check_cvxopt(results, "macc_min_aggregated") - - aucs = np.array(results["x"])[:, 0] - - results = macc_min_evaluate(ps, ns, aucs) - - if return_solutions: - results = results, (aucs, lower_bounds, upper_bounds) - - return results - - -def macc_min_aggregated( - auc: float, ps: np.array, ns: np.array, return_solutions: bool = False -): - """ - The minimum for the maximum average accuracy from average AUC - - Args: - auc (float): the average accuracy - ps (np.array): the number of positive samples - ns (np.array): the number of negative samples - return_solutions (bool): whether to return the solutions to the - underlying optimization problem - - Returns: - float | (float, (np.array, np.array, np.array)): - the acc or the acc and the following details: the AUC values for - the individual underlying curves, the lower bounds and the upper bounds - - Raises: - ValueError: when the auc is less then the desired lower bound or no - optimal solution is found - """ - ps = np.array(ps) - ns = np.array(ns) - - EPS = np.min(1.0/(ps*ns)) - - k = len(ps) - - lower_bounds = 1.0 - np.array([min(p, n) / (2 * max(p, n)) for p, n in zip(ps, ns)]) - - if auc < np.mean(lower_bounds): - raise ValueError("auc too small (macc_min_aggregated)") - - upper_bounds = np.repeat(1.0 - np.min(1 / ((ps + 1) * (ns + 1))), k) - - if auc == 1.0: # or auc >= np.mean(upper_bounds): - # the gradient would go to infinity in this case - results = 1.0 if return_solutions: - results = results, ( - np.repeat(1.0, len(ps)), - lower_bounds, - np.repeat(1.0, len(ps)), - ) + return results, (aucs, lower_bounds, upper_bounds) return results - elif auc >= np.mean(upper_bounds) - EPS: - n_1s = 0 - auc_p = auc - - sorting = np.argsort(upper_bounds) - rev_sorting = np.zeros(len(ps), dtype=int) - rev_sorting[sorting] = np.arange(len(ps)) - - upper_bounds_p = upper_bounds[sorting] - ps_p = ps[sorting] - ns_p = ns[sorting] - - while len(ps_p) > 0 and auc_p >= np.mean(upper_bounds_p) - EPS: - n_1s += 1 - ps_p = ps_p[:-1] - ns_p = ns_p[:-1] - upper_bounds_p = upper_bounds_p[:-1] - if len(ps_p) > 0: - auc_p = (auc_p * (len(ps_p) + 1) - 1) / len(ps_p) - - if len(ps_p) > 0: - results, (aucs, _, upper_bounds) = macc_min_solve(ps_p, ns_p, auc_p, return_solutions=True) - aucs = np.hstack([aucs, np.repeat(1.0, n_1s)]) - upper_bounds = np.hstack([upper_bounds, np.repeat(1.0, n_1s)]) - results = macc_min_evaluate(ps[sorting], ns[sorting], aucs) - if return_solutions: - return results, (aucs[rev_sorting], lower_bounds, upper_bounds[rev_sorting]) - else: - return results - else: - results = 1.0 - if return_solutions: - return results, (np.repeat(1.0, len(ps)), lower_bounds, upper_bounds) - return results - - return macc_min_solve(ps, ns, auc, return_solutions) + return acc_rmax_solve(ps, ns, auc, return_solutions) def acc_lower_from_aggregated( @@ -667,10 +448,10 @@ def acc_lower_from_aggregated( ps: int = None, ns: int = None, folding: dict = None, - lower: str = "min" + lower: str = "min", ) -> tuple: """ - This function applies the lower bound estimation schemes to estimate + This function applies the lower bound estimation schemes to estimate acc from scores Args: @@ -720,10 +501,10 @@ def acc_upper_from_aggregated( ps: int = None, ns: int = None, folding: dict = None, - upper: str = "max" + upper: str = "max", ) -> tuple: """ - This function applies the upper bound estimation schemes to estimate + This function applies the upper bound estimation schemes to estimate acc from scores Args: @@ -777,7 +558,7 @@ def acc_from_aggregated( upper: str = "max", ) -> tuple: """ - This function applies the estimation schemes to estimate + This function applies the estimation schemes to estimate acc from scores Args: @@ -801,180 +582,11 @@ def acc_from_aggregated( """ lower0 = acc_lower_from_aggregated( - scores=scores, - eps=eps, - ps=ps, - ns=ns, - folding=folding, - lower=lower + scores=scores, eps=eps, ps=ps, ns=ns, folding=folding, lower=lower ) upper0 = acc_upper_from_aggregated( - scores=scores, - eps=eps, - ps=ps, - ns=ns, - folding=folding, - upper=upper - ) - - return (lower0, upper0) - - -def max_acc_lower_from_aggregated( - *, - scores: dict, - eps: float, - ps: int = None, - ns: int = None, - folding: dict = None, - lower: str = "min" -) -> tuple: - """ - This function applies the lower bound estimation schemes to estimate - maximum accuracy from scores - - Args: - scores (dict): the reported scores - eps (float): the numerical uncertainty - p (int): the number of positive samples - n (int): the number of negative samples - folding (dict): description of a folding, alternative to specifying - ps and ns, contains the keys 'p', 'n', 'n_repeats', - 'n_folds', 'folding' (currently 'stratified_sklearn' - supported for 'folding') - lower (str): 'min' - - Returns: - float: the lower bound for the maximum accuracy - - Raises: - ValueError: when no optimal solution is found, or the parameters - violate the expectations of the estimation scheme - """ - - intervals = prepare_intervals(scores, eps) - - if "auc" not in intervals: - raise ValueError("auc must be specified") - - if (ps is not None or ns is not None) and folding is not None: - raise ValueError("specify either (ps and ns) or folding") - - if ps is None and ns is None and folding is not None: - ps, ns = translate_folding(folding) - - if lower == "min": - lower0 = macc_min_aggregated(intervals["auc"][0], ps, ns) - else: - raise ValueError(f"unsupported lower bound {lower}") - - return lower0 - - -def max_acc_upper_from_aggregated( - *, - scores: dict, - eps: float, - ps: int = None, - ns: int = None, - folding: dict = None, - upper: str = "max", -) -> tuple: - """ - This function applies the upper bound estimation schemes to estimate - maximum accuracy from scores - - Args: - scores (dict): the reported scores - eps (float): the numerical uncertainty - p (int): the number of positive samples - n (int): the number of negative samples - folding (dict): description of a folding, alternative to specifying - ps and ns, contains the keys 'p', 'n', 'n_repeats', - 'n_folds', 'folding' (currently 'stratified_sklearn' - supported for 'folding') - upper (str): 'max'/'rmax' - the type of upper bound - - Returns: - float: the upper bound for the maximum accuracy - - Raises: - ValueError: when no optimal solution is found, or the parameters - violate the expectations of the estimation scheme - """ - - intervals = prepare_intervals(scores, eps) - - if "auc" not in intervals: - raise ValueError("auc must be specified") - - if (ps is not None or ns is not None) and folding is not None: - raise ValueError("specify either (ps and ns) or folding") - - if ps is None and ns is None and folding is not None: - ps, ns = translate_folding(folding) - - if upper == "max": - upper0 = acc_max_aggregated(intervals["auc"][1], ps, ns) - elif upper == "rmax": - upper0 = acc_rmax_aggregated(intervals["auc"][1], ps, ns) - else: - raise ValueError(f"unsupported upper bound {upper}") - - return upper0 - - -def max_acc_from_aggregated( - *, - scores: dict, - eps: float, - ps: int = None, - ns: int = None, - folding: dict = None, - lower: str = "min", - upper: str = "max", -) -> tuple: - """ - This function applies the estimation schemes to estimate - maximum acc from scores - - Args: - scores (dict): the reported scores - eps (float): the numerical uncertainty - p (int): the number of positive samples - n (int): the number of negative samples - folding (dict): description of a folding, alternative to specifying - ps and ns, contains the keys 'p', 'n', 'n_repeats', - 'n_folds', 'folding' (currently 'stratified_sklearn' - supported for 'folding') - lower (str): 'min' - upper (str): 'max'/'rmax' - the type of upper bound - - Returns: - tuple(float, float): the interval for the accuracy - - Raises: - ValueError: when no optimal solution is found, or the parameters - violate the expectations of the estimation scheme - """ - - lower0 = max_acc_lower_from_aggregated( - scores=scores, - eps=eps, - ps=ps, - ns=ns, - folding=folding, - lower=lower - ) - - upper0 = max_acc_upper_from_aggregated( - scores=scores, - eps=eps, - ps=ps, - ns=ns, - folding=folding, - upper=upper + scores=scores, eps=eps, ps=ps, ns=ns, folding=folding, upper=upper ) return (lower0, upper0) diff --git a/mlscorecheck/auc/_acc_single.py b/mlscorecheck/auc/_acc_single.py index 4ca3746..4be850e 100644 --- a/mlscorecheck/auc/_acc_single.py +++ b/mlscorecheck/auc/_acc_single.py @@ -112,16 +112,9 @@ def macc_min(auc, p, n): return max(p, n) / (p + n) -def acc_lower_from( - *, - scores: dict, - eps: float, - p: int, - n: int, - lower: str = "min" -): +def acc_lower_from(*, scores: dict, eps: float, p: int, n: int, lower: str = "min"): """ - This function applies the lower bound estimation schemes to estimate + This function applies the lower bound estimation schemes to estimate acc from scores Args: @@ -150,20 +143,13 @@ def acc_lower_from( lower0 = acc_rmin(intervals["auc"][0], p, n) else: raise ValueError(f"unsupported lower bound {lower}") - + return lower0 -def acc_upper_from( - *, - scores: dict, - eps: float, - p: int, - n: int, - upper: str = "max" -): +def acc_upper_from(*, scores: dict, eps: float, p: int, n: int, upper: str = "max"): """ - This function applies the lower bound estimation schemes to estimate + This function applies the lower bound estimation schemes to estimate acc from scores Args: @@ -192,7 +178,7 @@ def acc_upper_from( upper0 = acc_rmax(intervals["auc"][1], p, n) else: raise ValueError(f"unsupported upper bound {upper}") - + return upper0 @@ -218,27 +204,13 @@ def acc_from( or the scores are inconsistent """ - lower0 = acc_lower_from( - scores=scores, - eps=eps, - p=p, - n=n, - lower=lower - ) - upper0 = acc_upper_from( - scores=scores, - eps=eps, - p=p, - n=n, - upper=upper - ) + lower0 = acc_lower_from(scores=scores, eps=eps, p=p, n=n, lower=lower) + upper0 = acc_upper_from(scores=scores, eps=eps, p=p, n=n, upper=upper) return (lower0, upper0) -def max_acc_lower_from( - *, scores: dict, eps: float, p: int, n: int, lower: str = "min" -): +def max_acc_lower_from(*, scores: dict, eps: float, p: int, n: int, lower: str = "min"): """ This function applies the estimation schemes to estimate maximum accuracy from scores @@ -271,9 +243,7 @@ def max_acc_lower_from( return lower0 -def max_acc_upper_from( - *, scores: dict, eps: float, p: int, n: int, upper: str = "min" -): +def max_acc_upper_from(*, scores: dict, eps: float, p: int, n: int, upper: str = "min"): """ This function applies the estimation schemes to estimate maximum accuracy from scores @@ -331,20 +301,8 @@ def max_acc_from( or the scores are inconsistent """ - lower0 = max_acc_lower_from( - scores=scores, - eps=eps, - p=p, - n=n, - lower=lower - ) - - upper0 = max_acc_upper_from( - scores=scores, - eps=eps, - p=p, - n=n, - upper=upper - ) + lower0 = max_acc_lower_from(scores=scores, eps=eps, p=p, n=n, lower=lower) + + upper0 = max_acc_upper_from(scores=scores, eps=eps, p=p, n=n, upper=upper) return (lower0, upper0) diff --git a/mlscorecheck/auc/_auc_aggregated.py b/mlscorecheck/auc/_auc_aggregated.py index dce0a22..32e0b1e 100644 --- a/mlscorecheck/auc/_auc_aggregated.py +++ b/mlscorecheck/auc/_auc_aggregated.py @@ -36,6 +36,8 @@ "estimate_tpr_interval", "estimate_fpr_interval", "augment_intervals_aggregated", + "check_applicability_lower_aggregated", + "check_applicability_upper_aggregated", ] @@ -729,9 +731,7 @@ def auc_armin_aggregated( return results -def check_applicability_lower_aggregated( - intervals: dict, lower: str, ps: int, ns: int -): +def check_applicability_lower_aggregated(intervals: dict, lower: str, ps: int, ns: int): """ Checks the applicability of the methods @@ -756,9 +756,7 @@ def check_applicability_lower_aggregated( raise ValueError("acc must be specified") -def check_applicability_upper_aggregated( - intervals: dict, upper: str, ps: int, ns: int -): +def check_applicability_upper_aggregated(intervals: dict, upper: str, ps: int, ns: int): """ Checks the applicability of the methods @@ -794,7 +792,7 @@ def auc_lower_from_aggregated( lower: str = "min", ): """ - This function applies the lower bound estimation schemes to estimate + This function applies the lower bound estimation schemes to estimate AUC from scores Args: @@ -858,7 +856,7 @@ def auc_upper_from_aggregated( upper: str = "min", ): """ - This function applies the upper bound estimation schemes to estimate + This function applies the upper bound estimation schemes to estimate AUC from scores Args: @@ -947,23 +945,11 @@ def auc_from_aggregated( """ lower0 = auc_lower_from_aggregated( - scores=scores, - eps=eps, - k=k, - ps=ps, - ns=ns, - folding=folding, - lower=lower + scores=scores, eps=eps, k=k, ps=ps, ns=ns, folding=folding, lower=lower ) upper0 = auc_upper_from_aggregated( - scores=scores, - eps=eps, - k=k, - ps=ps, - ns=ns, - folding=folding, - upper=upper + scores=scores, eps=eps, k=k, ps=ps, ns=ns, folding=folding, upper=upper ) return (lower0, upper0) diff --git a/mlscorecheck/auc/_auc_single.py b/mlscorecheck/auc/_auc_single.py index 1ab1d75..945fbc6 100644 --- a/mlscorecheck/auc/_auc_single.py +++ b/mlscorecheck/auc/_auc_single.py @@ -26,6 +26,8 @@ "auc_amin", "auc_armin", "auc_amax", + "check_lower_applicability", + "check_upper_applicability", ] @@ -388,16 +390,15 @@ def check_lower_applicability(intervals: dict, lower: str, p: int, n: int): ValueError: when the methods are not applicable with the specified scores """ - if lower in ["min", "rmin", "grmin"]: - if "fpr" not in intervals or "tpr" not in intervals: - raise ValueError("fpr, tpr or their complements must be specified") - if lower in ["grmin", "amin", "armin"]: - if p is None or n is None: - raise ValueError("p and n must be specified") - if lower in ["amin", "armin"]: - if "acc" not in intervals: - raise ValueError("acc must be specified") - + if lower in ["min", "rmin", "grmin"] and ( + "fpr" not in intervals or "tpr" not in intervals + ): + raise ValueError("fpr, tpr or their complements must be specified") + if lower in ["grmin", "amin", "armin"] and (p is None or n is None): + raise ValueError("p and n must be specified") + if lower in ["amin", "armin"] and ("acc" not in intervals): + raise ValueError("acc must be specified") + def check_upper_applicability(intervals: dict, upper: str, p: int, n: int): """ @@ -413,27 +414,19 @@ def check_upper_applicability(intervals: dict, upper: str, p: int, n: int): ValueError: when the methods are not applicable with the specified scores """ - if upper in ["max"]: - if "fpr" not in intervals or "tpr" not in intervals: - raise ValueError("fpr, tpr or their complements must be specified") - if upper in ["amax", "maxa"]: - if p is None or n is None: - raise ValueError("p and n must be specified") - if upper in ["amax", "maxa"]: - if "acc" not in intervals: - raise ValueError("acc must be specified") + if upper in ["max"] and ("fpr" not in intervals or "tpr" not in intervals): + raise ValueError("fpr, tpr or their complements must be specified") + if upper in ["amax", "maxa"] and (p is None or n is None): + raise ValueError("p and n must be specified") + if upper in ["amax", "maxa"] and "acc" not in intervals: + raise ValueError("acc must be specified") def auc_lower_from( - *, - scores: dict, - eps: float, - p: int = None, - n: int = None, - lower: str = "min" + *, scores: dict, eps: float, p: int = None, n: int = None, lower: str = "min" ): """ - This function applies the lower bound estimation schemes to estimate + This function applies the lower bound estimation schemes to estimate AUC from scores Args: @@ -472,20 +465,15 @@ def auc_lower_from( lower0 = auc_armin(intervals["acc"][0], p, n) else: raise ValueError(f"unsupported lower bound {lower}") - + return lower0 def auc_upper_from( - *, - scores: dict, - eps: float, - p: int = None, - n: int = None, - upper: str = "max" + *, scores: dict, eps: float, p: int = None, n: int = None, upper: str = "max" ): """ - This function applies the lower bound estimation schemes to estimate + This function applies the lower bound estimation schemes to estimate AUC from scores Args: @@ -503,7 +491,7 @@ def auc_upper_from( ValueError: when the parameters are not suitable for the estimation methods or the scores are inconsistent """ - + scores = translate_scores(scores) intervals = prepare_intervals(scores, eps) @@ -523,6 +511,7 @@ def auc_upper_from( return upper0 + def auc_from( *, scores: dict, @@ -553,20 +542,8 @@ def auc_from( or the scores are inconsistent """ - lower0 = auc_lower_from( - scores=scores, - eps=eps, - p=p, - n=n, - lower=lower - ) + lower0 = auc_lower_from(scores=scores, eps=eps, p=p, n=n, lower=lower) - upper0 = auc_upper_from( - scores=scores, - eps=eps, - p=p, - n=n, - upper=upper - ) + upper0 = auc_upper_from(scores=scores, eps=eps, p=p, n=n, upper=upper) return (lower0, upper0) diff --git a/mlscorecheck/auc/_max_acc_aggregated.py b/mlscorecheck/auc/_max_acc_aggregated.py new file mode 100644 index 0000000..259a28f --- /dev/null +++ b/mlscorecheck/auc/_max_acc_aggregated.py @@ -0,0 +1,443 @@ +""" +This module implements the maximum accuracy estimations in the aggregated case +""" + +import numpy as np + +from cvxopt import matrix +from cvxopt.solvers import cp +from cvxopt import solvers + +from ._utils import prepare_intervals, translate_folding + +from ._acc_single import macc_min +from ._auc_aggregated import check_cvxopt +from ._acc_aggregated import acc_max_aggregated, acc_rmax_aggregated + +__all__ = [ + "macc_min_aggregated", + "max_acc_from_aggregated", + "max_acc_lower_from_aggregated", + "max_acc_upper_from_aggregated", + "FMAccMin", +] + + +class FMAccMin: # pylint: disable=too-few-public-methods + """ + Implements the convex programming objective for the maximum accuracy + minimization. + """ + + def __init__(self, ps: np.array, ns: np.array): + """ + The constructor of the object + + Args: + ps (np.array): the number of positive samples + ns (np.array): the number of negative samples + """ + self.ps = ps + self.ns = ns + self.k = len(ps) + self.weights = np.sqrt(2 * (ps * ns)) / (ps + ns) + self.lower_bounds = 1.0 - np.array( + [min(p, n) / (2 * max(p, n)) for p, n in zip(ps, ns)] + ) + self.upper_bounds = np.repeat(1.0 - np.min(1 / ((ps + 1) * (ns + 1))), self.k) + + def __call__(self, x: matrix = None, z: matrix = None): + """ + The call method according to the specification in cvxopt + + Evaluates the function originating from the objective + + f_0(auc) = dfrac{1}{k} sum limits_{i=1}^{k}macc_min(auc_i, p_i, n_i), + + f_0(auc) = 1/k sum (1 - sqrt(2*p_i*n_i*(1 - auc_i))/(p_i + n_i)) + + by removing the multiplier 1/k and the constants parts. + + Also: + auc = sp.Symbol('auc') + f = sp.sqrt(1 - auc) + sp.diff(f, auc), sp.diff(sp.diff(f, auc), auc) + + (-1/(2*sqrt(1 - auc)), -1/(4*(1 - auc)**(3/2))) + + Args: + x (cvxopt.matrix | None): a vector in the parameter space + z (cvxopt.matrix | None): a weight vector + + Returns: + (int, matrix): the number of non-linear constraints and a feasible + point when x is None and z is None + (matrix, matrix): the objective value at x, and the gradient + at x when x is not None but z is None + (matrix, matrix, matrx): the objective value at x, the gradient + at x and the weighted sum of the Hessian of the objective and + all non-linear constraints with the weights z if z is not None + """ + if x is None and z is None: + return (0, matrix(self.lower_bounds, (self.k, 1))) + # return (0, matrix(np.repeat(1.0, self.k), (self.k, 1))) + + if np.any( + np.array(x)[:, 0] > self.upper_bounds + ): # or np.any(np.array(x)[:, 0] < self.lower_bounds): + return None, None + + # if x is not None: + f = matrix( + -np.sum(np.clip(np.sqrt(1 - x), 0.0, 2.0) * self.weights.reshape(-1, 1)) + ) + df = matrix( + 1.0 / (2 * np.clip(np.sqrt(1 - x), 0.0, 2.0)) * self.weights.reshape(-1, 1) + ).T + + if z is None: + return (f, df) + + hess = np.diag( + 1.0 + / (4 * np.clip(np.array(1 - x), 0.0, 2.0)[:, 0] ** (3 / 2)) + * self.weights + ) + + hess = matrix(z[0] * hess) + + return (f, df, hess) + + +def macc_min_evaluate(ps: np.array, ns: np.array, aucs: np.array): + """ + Evaluates a particular macc_min configuration + + Args: + ps (np.array): the number of positive samples + ns (np.array): the number of negative samples + aucs (np.array): the AUCs + + Returns: + float: the average accuracy + """ + return np.mean([macc_min(auc, p, n) for auc, p, n in zip(aucs, ps, ns)]) + + +def macc_min_solve( + ps: np.array, ns: np.array, avg_auc: float, return_solutions: bool = False +): + """ + Solves the maximum accuracy minimum curves problem + + Args: + ps (np.array): the number of positive samples + ns (np.array): the number of negative samples + avg_auc (np.array): the average AUC + return_solutions (bool): whether to return the solutions and + further details + + Returns: + float | (float, np.array, np.array, np.array): the mean accuracy, + or the mean accuracy, the auc parameters, lower bounds and upper + bounds + + Raises: + ValueError: when no optimal solution is found + """ + F = FMAccMin(ps, ns) # pylint: disable=invalid-name + + k = ps.shape[0] + + lower_bounds = 1.0 - np.array([min(p, n) / (2 * max(p, n)) for p, n in zip(ps, ns)]) + upper_bounds = np.repeat(1.0 - np.min(1 / ((ps + 1) * (ns + 1))), k) + # print(upper_bounds) + + A = np.repeat(1.0 / k, k).reshape(-1, 1).T # pylint: disable=invalid-name + b = np.array([avg_auc]).astype(float) + G = np.vstack([np.eye(k), -np.eye(k)]).astype(float) # pylint: disable=invalid-name + h = np.hstack([upper_bounds, -lower_bounds]) + + G = matrix(G) # pylint: disable=invalid-name + h = matrix(h) + A = matrix(A) # pylint: disable=invalid-name + b = matrix(b) + + actual = solvers.options.get("show_progress", None) + solvers.options["show_progress"] = False + + results = cp(F, G, h, A=A, b=b) + + solvers.options["show_progress"] = actual + + check_cvxopt(results, "macc_min_aggregated") + + aucs = np.array(results["x"])[:, 0] + + results = macc_min_evaluate(ps, ns, aucs) + + if return_solutions: + results = results, (aucs, lower_bounds, upper_bounds) + + return results + + +def reduce_macc_min_edge_case( + auc: float, ps: np.array, ns: np.array, eps: float, upper_bounds: np.array +): + """ + Solves the reduced macc min problem + + Args: + auc (float): the AUC value + ps (np.array): the numbers of positives + ns (np.array): the numbers of negatives + eps (float): the epsilon + upper_bounds (np.array): the upper bounds + + Returns: + float, np.array, np.array: the result, the aucs and the upper bounds + """ + n_1s = 0 + auc_p = auc + + sorting = np.argsort(upper_bounds) + rev_sorting = np.zeros(len(ps), dtype=int) + rev_sorting[sorting] = np.arange(len(ps)) + + upper_bounds_p = upper_bounds[sorting] + ps_p = ps[sorting] + ns_p = ns[sorting] + + while len(ps_p) > 0 and auc_p >= np.mean(upper_bounds_p) - eps: + n_1s += 1 + ps_p = ps_p[:-1] + ns_p = ns_p[:-1] + upper_bounds_p = upper_bounds_p[:-1] + if len(ps_p) > 0: + auc_p = (auc_p * (len(ps_p) + 1) - 1) / len(ps_p) + + if len(ps_p) > 0: + results, (aucs, _, upper_bounds) = macc_min_solve( + ps_p, ns_p, auc_p, return_solutions=True + ) + + aucs = np.hstack([aucs, np.repeat(1.0, n_1s)]) + upper_bounds = np.hstack([upper_bounds, np.repeat(1.0, n_1s)]) + results = macc_min_evaluate(ps[sorting], ns[sorting], aucs) + + aucs = aucs[rev_sorting] + upper_bounds = upper_bounds[rev_sorting] + else: + results = 1.0 + aucs = np.repeat(1.0, len(ps)) + + return results, aucs, upper_bounds + + +def macc_min_aggregated( + auc: float, ps: np.array, ns: np.array, return_solutions: bool = False +): + """ + The minimum for the maximum average accuracy from average AUC + + Args: + auc (float): the average accuracy + ps (np.array): the number of positive samples + ns (np.array): the number of negative samples + return_solutions (bool): whether to return the solutions to the + underlying optimization problem + + Returns: + float | (float, (np.array, np.array, np.array)): + the acc or the acc and the following details: the AUC values for + the individual underlying curves, the lower bounds and the upper bounds + + Raises: + ValueError: when the auc is less then the desired lower bound or no + optimal solution is found + """ + ps = np.array(ps) + ns = np.array(ns) + + eps = np.min(1.0 / (ps * ns)) + + k = len(ps) + + lower_bounds = 1.0 - np.array([min(p, n) / (2 * max(p, n)) for p, n in zip(ps, ns)]) + + if auc < np.mean(lower_bounds): + raise ValueError("auc too small (macc_min_aggregated)") + + upper_bounds = np.repeat(1.0 - np.min(1 / ((ps + 1) * (ns + 1))), k) + + if auc == 1.0: # or auc >= np.mean(upper_bounds): + # the gradient would go to infinity in this case + results = 1.0 + + if return_solutions: + results = results, ( + np.repeat(1.0, len(ps)), + lower_bounds, + np.repeat(1.0, len(ps)), + ) + return results + if auc >= np.mean(upper_bounds) - eps: + results, aucs, upper_bounds = reduce_macc_min_edge_case( + auc, ps, ns, eps, upper_bounds + ) + + if return_solutions: + return results, (aucs, lower_bounds, upper_bounds) + return results + + return macc_min_solve(ps, ns, auc, return_solutions) + + +def max_acc_lower_from_aggregated( + *, + scores: dict, + eps: float, + ps: int = None, + ns: int = None, + folding: dict = None, + lower: str = "min", +) -> tuple: + """ + This function applies the lower bound estimation schemes to estimate + maximum accuracy from scores + + Args: + scores (dict): the reported scores + eps (float): the numerical uncertainty + p (int): the number of positive samples + n (int): the number of negative samples + folding (dict): description of a folding, alternative to specifying + ps and ns, contains the keys 'p', 'n', 'n_repeats', + 'n_folds', 'folding' (currently 'stratified_sklearn' + supported for 'folding') + lower (str): 'min' + + Returns: + float: the lower bound for the maximum accuracy + + Raises: + ValueError: when no optimal solution is found, or the parameters + violate the expectations of the estimation scheme + """ + + intervals = prepare_intervals(scores, eps) + + if "auc" not in intervals: + raise ValueError("auc must be specified") + + if (ps is not None or ns is not None) and folding is not None: + raise ValueError("specify either (ps and ns) or folding") + + if ps is None and ns is None and folding is not None: + ps, ns = translate_folding(folding) + + if lower == "min": + lower0 = macc_min_aggregated(intervals["auc"][0], ps, ns) + else: + raise ValueError(f"unsupported lower bound {lower}") + + return lower0 + + +def max_acc_upper_from_aggregated( + *, + scores: dict, + eps: float, + ps: int = None, + ns: int = None, + folding: dict = None, + upper: str = "max", +) -> tuple: + """ + This function applies the upper bound estimation schemes to estimate + maximum accuracy from scores + + Args: + scores (dict): the reported scores + eps (float): the numerical uncertainty + p (int): the number of positive samples + n (int): the number of negative samples + folding (dict): description of a folding, alternative to specifying + ps and ns, contains the keys 'p', 'n', 'n_repeats', + 'n_folds', 'folding' (currently 'stratified_sklearn' + supported for 'folding') + upper (str): 'max'/'rmax' - the type of upper bound + + Returns: + float: the upper bound for the maximum accuracy + + Raises: + ValueError: when no optimal solution is found, or the parameters + violate the expectations of the estimation scheme + """ + + intervals = prepare_intervals(scores, eps) + + if "auc" not in intervals: + raise ValueError("auc must be specified") + + if (ps is not None or ns is not None) and folding is not None: + raise ValueError("specify either (ps and ns) or folding") + + if ps is None and ns is None and folding is not None: + ps, ns = translate_folding(folding) + + if upper == "max": + upper0 = acc_max_aggregated(intervals["auc"][1], ps, ns) + elif upper == "rmax": + upper0 = acc_rmax_aggregated(intervals["auc"][1], ps, ns) + else: + raise ValueError(f"unsupported upper bound {upper}") + + return upper0 + + +def max_acc_from_aggregated( + *, + scores: dict, + eps: float, + ps: int = None, + ns: int = None, + folding: dict = None, + lower: str = "min", + upper: str = "max", +) -> tuple: + """ + This function applies the estimation schemes to estimate + maximum acc from scores + + Args: + scores (dict): the reported scores + eps (float): the numerical uncertainty + p (int): the number of positive samples + n (int): the number of negative samples + folding (dict): description of a folding, alternative to specifying + ps and ns, contains the keys 'p', 'n', 'n_repeats', + 'n_folds', 'folding' (currently 'stratified_sklearn' + supported for 'folding') + lower (str): 'min' + upper (str): 'max'/'rmax' - the type of upper bound + + Returns: + tuple(float, float): the interval for the accuracy + + Raises: + ValueError: when no optimal solution is found, or the parameters + violate the expectations of the estimation scheme + """ + + lower0 = max_acc_lower_from_aggregated( + scores=scores, eps=eps, ps=ps, ns=ns, folding=folding, lower=lower + ) + + upper0 = max_acc_upper_from_aggregated( + scores=scores, eps=eps, ps=ps, ns=ns, folding=folding, upper=upper + ) + + return (lower0, upper0) diff --git a/mlscorecheck/auc/_utils.py b/mlscorecheck/auc/_utils.py index a9f6dda..7235acc 100644 --- a/mlscorecheck/auc/_utils.py +++ b/mlscorecheck/auc/_utils.py @@ -153,7 +153,7 @@ def check_cvxopt(results, message): Raises: ValueError: when the solution is not optimal """ - return 0.0 + if results["status"] != "optimal": raise ValueError( "no optimal solution found for the configuration " + f"({message})" diff --git a/tests/auc/test_acc_aggregated.py b/tests/auc/test_acc_aggregated.py index a374715..6bae926 100644 --- a/tests/auc/test_acc_aggregated.py +++ b/tests/auc/test_acc_aggregated.py @@ -11,15 +11,14 @@ acc_max, acc_rmin, acc_rmax, - macc_min, acc_min_aggregated, acc_rmin_aggregated, acc_max_aggregated, acc_rmax_aggregated, - macc_min_aggregated, perturb_solutions, acc_from_aggregated, - max_acc_from_aggregated, + acc_upper_from_aggregated, + FAccRMax, ) random_seeds = list(range(100)) @@ -147,46 +146,61 @@ def test_acc_rmax_exception(): acc_rmax_aggregated(0.3, [], []) -@pytest.mark.parametrize("conf", acc_auc_confs) -@pytest.mark.parametrize("random_seed", random_seeds) -def test_macc_min_aggregated(conf, random_seed): +def test_acc_rmax_edge(): """ - Testing if perturbation cant find smaller objective - - Args: - conf (dict): the configuration - random_seed (int): the random seed + Testing the edge cases for acc_rmax """ + np.testing.assert_almost_equal( + acc_rmax_aggregated( + auc=0.5, ps=np.array([10, 10]), ns=np.array([20, 20]), return_solutions=True + )[0], + 2 / 3, + ) - auc = conf["auc"] - ps = conf["ps"] - ns = conf["ns"] - - lower_bounds = 1.0 - np.array([min(p, n) / (2 * max(p, n)) for p, n in zip(ps, ns)]) - if auc < np.mean(lower_bounds): - with pytest.raises(ValueError): - acc, (aucs, lower, upper) = macc_min_aggregated(auc, ps, ns, True) - return - - acc, (aucs, lower, upper) = macc_min_aggregated(auc, ps, ns, True) - acc = np.round(acc, 8) - - tmp = perturb_solutions(aucs, lower, upper, random_seed) - - acc_tmp = np.mean([macc_min(acc, p, n) for acc, p, n in zip(tmp, ps, ns)]) - acc_tmp = np.round(acc_tmp, 8) + assert ( + acc_rmax_aggregated( + auc=0.5001, + ps=np.array([10, 10]), + ns=np.array([20, 20]), + return_solutions=True, + ) + is not None + ) - assert acc <= acc_tmp + assert ( + acc_rmax_aggregated( + auc=0.501, + ps=np.array([10, 10]), + ns=np.array([20, 20]), + return_solutions=True, + ) + is not None + ) + assert ( + acc_rmax_aggregated( + auc=0.51, + ps=np.array([10, 10]), + ns=np.array([20, 20]), + return_solutions=True, + ) + is not None + ) -def test_macc_min_aggregated_extreme(): - """ - Testing the edge case of macc_min_aggregated - """ + assert ( + acc_rmax_aggregated(auc=0.5001, ps=np.array([10, 10]), ns=np.array([20, 20])) + is not None + ) - acc, (_, _, _) = macc_min_aggregated(1.0, [10, 10], [20, 20], True) + assert ( + acc_rmax_aggregated(auc=0.501, ps=np.array([10, 10]), ns=np.array([20, 20])) + is not None + ) - assert acc == 1.0 + assert ( + acc_rmax_aggregated(auc=0.51, ps=np.array([10, 10]), ns=np.array([20, 20])) + is not None + ) def test_acc_from_aggregated(): @@ -200,6 +214,9 @@ def test_acc_from_aggregated(): with pytest.raises(ValueError): acc_from_aggregated(scores={}, eps=1e-4, ps=ps, ns=ns) + with pytest.raises(ValueError): + acc_upper_from_aggregated(scores={}, eps=1e-4, ps=ps, ns=ns) + for lower in ["min", "rmin"]: for upper in ["max", "rmax"]: tmp = acc_from_aggregated( @@ -213,33 +230,14 @@ def test_acc_from_aggregated(): with pytest.raises(ValueError): acc_from_aggregated(scores={"auc": 0.9}, eps=1e-4, ps=ps, ns=ns, upper="dummy") - -def test_max_acc_from_aggregated(): - """ - Testing the max_acc_from functionality - """ - - ps = [60, 70, 80] - ns = [80, 90, 80] - - with pytest.raises(ValueError): - max_acc_from_aggregated(scores={}, eps=1e-4, ps=ps, ns=ns) - - for lower in ["min"]: - for upper in ["max", "rmax"]: - tmp = max_acc_from_aggregated( - scores={"auc": 0.9}, eps=1e-4, ps=ps, ns=ns, lower=lower, upper=upper - ) - assert tmp[0] <= tmp[1] - with pytest.raises(ValueError): - max_acc_from_aggregated( - scores={"auc": 0.9}, eps=1e-4, ps=ps, ns=ns, lower="dummy" + acc_from_aggregated( + scores={"auc": 0.9}, eps=1e-4, ps=ps, ns=ns, upper="dummy", folding={} ) with pytest.raises(ValueError): - max_acc_from_aggregated( - scores={"auc": 0.9}, eps=1e-4, ps=ps, ns=ns, upper="dummy" + acc_upper_from_aggregated( + scores={"auc": 0.9}, eps=1e-4, ps=ps, ns=ns, upper="dummy", folding={} ) @@ -284,42 +282,11 @@ def test_acc_from_aggregated_error(): ) -def test_max_acc_from_aggregated_folding(): +def test_faccrmax(): """ - Testing the max_acc_from_aggregated with folding + Testing some functionalities from FAccRMax """ - result = max_acc_from_aggregated( - scores={"auc": 0.9}, - eps=0.01, - folding={ - "p": 20, - "n": 80, - "n_repeats": 1, - "n_folds": 5, - "folding": "stratified_sklearn", - }, - ) - - assert result is not None - - -def test_max_acc_from_aggregated_error(): - """ - Testing the max_acc_from_aggregated throwing exception - """ + obj = FAccRMax(np.array([10, 10]), np.array([20, 20])) - with pytest.raises(ValueError): - max_acc_from_aggregated( - scores={"acc": 0.9}, - eps=0.01, - ps=[10, 20], - ns=[20, 30], - folding={ - "p": 20, - "n": 80, - "n_repeats": 1, - "n_folds": 5, - "folding": "stratified_sklearn", - }, - ) + assert obj(np.array([[-1], [-1]])) == (None, None) diff --git a/tests/auc/test_acc_single.py b/tests/auc/test_acc_single.py index 9274996..ed62023 100644 --- a/tests/auc/test_acc_single.py +++ b/tests/auc/test_acc_single.py @@ -14,6 +14,8 @@ macc_min, acc_from, max_acc_from, + acc_upper_from, + max_acc_upper_from, ) auc_scenarios = [{"auc": 0.8, "p": 20, "n": 60}, {"auc": 0.4, "p": 50, "n": 51}] @@ -106,6 +108,9 @@ def test_acc_from(): with pytest.raises(ValueError): acc_from(scores={}, eps=1e-4, p=50, n=100) + with pytest.raises(ValueError): + acc_upper_from(scores={}, eps=1e-4, p=50, n=100) + for lower in ["min", "rmin"]: for upper in ["max", "rmax"]: tmp = acc_from( @@ -128,6 +133,9 @@ def test_max_acc_from(): with pytest.raises(ValueError): max_acc_from(scores={}, eps=1e-4, p=50, n=100) + with pytest.raises(ValueError): + max_acc_upper_from(scores={}, eps=1e-4, p=50, n=100) + for lower in ["min"]: for upper in ["max", "rmax"]: tmp = max_acc_from( diff --git a/tests/auc/test_auc_aggregated.py b/tests/auc/test_auc_aggregated.py index 5614e02..3cc74d6 100644 --- a/tests/auc/test_auc_aggregated.py +++ b/tests/auc/test_auc_aggregated.py @@ -27,6 +27,8 @@ estimate_tpr_interval, estimate_fpr_interval, augment_intervals_aggregated, + check_applicability_upper_aggregated, + auc_upper_from_aggregated, ) random_seeds = list(range(100)) @@ -335,11 +337,21 @@ def test_auc_from_aggregated(): with pytest.raises(ValueError): auc_from_aggregated(scores={"tpr": 0.9}, eps=0.01, k=5) + with pytest.raises(ValueError): + check_applicability_upper_aggregated( + intervals={"tpr": 0.9}, upper="max", ps=None, ns=None + ) + with pytest.raises(ValueError): auc_from_aggregated( scores={"tpr": 0.9, "fpr": 0.1}, eps=0.01, k=5, lower="amin" ) + with pytest.raises(ValueError): + check_applicability_upper_aggregated( + intervals={"tpr": 0.9, "fpr": 0.1}, upper="amax", ps=None, ns=None + ) + with pytest.raises(ValueError): auc_from_aggregated( scores={"tpr": 0.9}, @@ -351,6 +363,11 @@ def test_auc_from_aggregated(): ns=ns, ) + with pytest.raises(ValueError): + check_applicability_upper_aggregated( + intervals={"tpr": 0.9, "fpr": 0.1}, upper="amax", ps=[], ns=[] + ) + for lower in ["min", "rmin", "amin", "armin"]: for upper in ["max", "amax", "maxa"]: tmp = auc_from_aggregated( @@ -424,3 +441,18 @@ def test_auc_from_aggregated_error(): "folding": "stratified_sklearn", }, ) + + with pytest.raises(ValueError): + auc_upper_from_aggregated( + scores={"tpr": 0.9, "fpr": 0.1}, + eps=0.01, + ps=[10, 20], + ns=[20, 30], + folding={ + "p": 20, + "n": 80, + "n_repeats": 1, + "n_folds": 5, + "folding": "stratified_sklearn", + }, + ) diff --git a/tests/auc/test_auc_single.py b/tests/auc/test_auc_single.py index f6c8ceb..a11e8dc 100644 --- a/tests/auc/test_auc_single.py +++ b/tests/auc/test_auc_single.py @@ -23,6 +23,8 @@ integrate_roc_curve, augment_intervals, auc_from, + check_lower_applicability, + check_upper_applicability, ) scenarios = [ @@ -244,6 +246,34 @@ def test_augment_intervals(): ) +def test_applicabilities(): + """ + Testing the check applicability functionalities + """ + + with pytest.raises(ValueError): + check_lower_applicability(intervals={"tpr": (0, 1)}, lower="min", p=5, n=10) + + with pytest.raises(ValueError): + check_lower_applicability( + intervals={"tpr": (0, 1), "fpr": (0, 1)}, lower="grmin", p=5, n=None + ) + + with pytest.raises(ValueError): + check_lower_applicability(intervals={"tpr": (0, 1)}, lower="amin", p=5, n=10) + + with pytest.raises(ValueError): + check_upper_applicability(intervals={"tpr": (0, 1)}, upper="max", p=5, n=10) + + with pytest.raises(ValueError): + check_upper_applicability( + intervals={"tpr": (0, 1), "fpr": (0, 1)}, upper="amax", p=5, n=None + ) + + with pytest.raises(ValueError): + check_upper_applicability(intervals={"tpr": (0, 1)}, upper="amax", p=5, n=10) + + def test_auc_from(): """ Testing the auc_from function diff --git a/tests/auc/test_max_acc_aggregated.py b/tests/auc/test_max_acc_aggregated.py new file mode 100644 index 0000000..9d2a58e --- /dev/null +++ b/tests/auc/test_max_acc_aggregated.py @@ -0,0 +1,188 @@ +""" +Testing the aggregated maximum accuracy related estimations +""" + +import pytest + +import numpy as np + +from mlscorecheck.auc import ( + macc_min, + macc_min_aggregated, + perturb_solutions, + max_acc_from_aggregated, + max_acc_upper_from_aggregated, + FMAccMin, +) + +random_seeds = list(range(100)) + +acc_auc_confs = [ + {"auc": 0.8, "ps": [10, 10, 10], "ns": [19, 19, 20]}, + {"auc": 0.6, "ps": [10, 10, 10], "ns": [19, 19, 20]}, + {"auc": 0.4, "ps": [10, 10, 10], "ns": [19, 19, 20]}, + {"auc": 0.8, "ps": [10, 11, 12, 13, 14], "ns": [19, 18, 17, 16, 15]}, + {"auc": 0.6, "ps": [10, 11, 12, 13, 14], "ns": [19, 18, 17, 16, 15]}, +] + + +@pytest.mark.parametrize("conf", acc_auc_confs) +@pytest.mark.parametrize("random_seed", random_seeds) +def test_macc_min_aggregated(conf, random_seed): + """ + Testing if perturbation cant find smaller objective + + Args: + conf (dict): the configuration + random_seed (int): the random seed + """ + + auc = conf["auc"] + ps = conf["ps"] + ns = conf["ns"] + + lower_bounds = 1.0 - np.array([min(p, n) / (2 * max(p, n)) for p, n in zip(ps, ns)]) + if auc < np.mean(lower_bounds): + with pytest.raises(ValueError): + acc, (aucs, lower, upper) = macc_min_aggregated(auc, ps, ns, True) + return + + acc, (aucs, lower, upper) = macc_min_aggregated(auc, ps, ns, True) + acc = np.round(acc, 8) + + tmp = perturb_solutions(aucs, lower, upper, random_seed) + + acc_tmp = np.mean([macc_min(acc, p, n) for acc, p, n in zip(tmp, ps, ns)]) + acc_tmp = np.round(acc_tmp, 8) + + assert acc <= acc_tmp + + +def test_macc_min_aggregated_extreme(): + """ + Testing the edge case of macc_min_aggregated + """ + + ps = [10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10] + ns = [20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20] + + acc, (_, _, _) = macc_min_aggregated(1.0, ps, ns, True) + + assert acc == 1.0 + + acc, (_, _, _) = macc_min_aggregated(0.999, ps, ns, True) + + assert acc is not None + + acc = macc_min_aggregated(0.999, ps, ns, False) + + assert acc is not None + + acc = macc_min_aggregated(0.999, ps, ns, False) + + assert acc is not None + + acc, (_, _, _) = macc_min_aggregated(0.99999, ps, ns, True) + + assert acc is not None + + acc = macc_min_aggregated(0.99999, ps, ns, False) + + assert acc is not None + + acc = macc_min_aggregated(0.99999, ps, ns, False) + + assert acc is not None + + +def test_max_acc_from_aggregated(): + """ + Testing the max_acc_from functionality + """ + + ps = [60, 70, 80] + ns = [80, 90, 80] + + with pytest.raises(ValueError): + max_acc_from_aggregated(scores={}, eps=1e-4, ps=ps, ns=ns) + + with pytest.raises(ValueError): + max_acc_upper_from_aggregated(scores={}, eps=1e-4, ps=ps, ns=ns) + + for lower in ["min"]: + for upper in ["max", "rmax"]: + tmp = max_acc_from_aggregated( + scores={"auc": 0.9}, eps=1e-4, ps=ps, ns=ns, lower=lower, upper=upper + ) + assert tmp[0] <= tmp[1] + + with pytest.raises(ValueError): + max_acc_from_aggregated( + scores={"auc": 0.9}, eps=1e-4, ps=ps, ns=ns, lower="dummy" + ) + + with pytest.raises(ValueError): + max_acc_from_aggregated( + scores={"auc": 0.9}, eps=1e-4, ps=ps, ns=ns, upper="dummy" + ) + + with pytest.raises(ValueError): + max_acc_from_aggregated( + scores={"auc": 0.9}, eps=1e-4, ps=ps, ns=ns, upper="dummy", folding={} + ) + + with pytest.raises(ValueError): + max_acc_upper_from_aggregated( + scores={"auc": 0.9}, eps=1e-4, ps=ps, ns=ns, upper="dummy", folding={} + ) + + +def test_max_acc_from_aggregated_folding(): + """ + Testing the max_acc_from_aggregated with folding + """ + + result = max_acc_from_aggregated( + scores={"auc": 0.9}, + eps=0.01, + folding={ + "p": 20, + "n": 80, + "n_repeats": 1, + "n_folds": 5, + "folding": "stratified_sklearn", + }, + ) + + assert result is not None + + +def test_max_acc_from_aggregated_error(): + """ + Testing the max_acc_from_aggregated throwing exception + """ + + with pytest.raises(ValueError): + max_acc_from_aggregated( + scores={"acc": 0.9}, + eps=0.01, + ps=[10, 20], + ns=[20, 30], + folding={ + "p": 20, + "n": 80, + "n_repeats": 1, + "n_folds": 5, + "folding": "stratified_sklearn", + }, + ) + + +def test_fmaccmin(): + """ + Testing some functionalities from FAccRMax + """ + + obj = FMAccMin(np.array([10, 10]), np.array([20, 20])) + + assert obj(np.array([[2], [2]])) == (None, None)