diff --git a/.gitignore b/.gitignore index 1150045..4d54fcd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,141 @@ -__pycache__ poetry.lock -.ipynb_checkpoints* testing_notebooks outs slurm -.vscode \ No newline at end of file + +# DS_Store +.DS_Store + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# vscode +.vscode/settings.json +.vscode diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..45ea023 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,50 @@ +fail_fast: false +default_language_version: + python: python3 +default_stages: + - commit + - push +minimum_pre_commit_version: 2.16.0 +repos: + - repo: https://github.com/psf/black + rev: "23.3.0" + hooks: + - id: black + - repo: https://github.com/asottile/blacken-docs + rev: 1.13.0 + hooks: + - id: blacken-docs + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v3.0.0-alpha.6 + hooks: + - id: prettier + # Newer versions of node don't work on systems that have an older version of GLIBC + # (in particular Ubuntu 18.04 and Centos 7) + # EOL of Centos 7 is in 2024-06, we can probably get rid of this then. + # See https://github.com/scverse/cookiecutter-scverse/issues/143 and + # https://github.com/jupyterlab/jupyterlab/issues/12675 + language_version: "17.9.1" + - repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.261 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: detect-private-key + - id: check-ast + - id: end-of-file-fixer + - id: mixed-line-ending + args: [--fix=lf] + - id: trailing-whitespace + - id: check-case-conflict + - repo: local + hooks: + - id: forbid-to-commit + name: Don't commit rej files + entry: | + Cannot commit .rej files. These indicate merge conflicts that arise during automated template updates. + Fix the merge conflicts manually and remove the .rej files. + language: fail + files: '.*\.rej$' \ No newline at end of file diff --git a/README.rst b/README.rst deleted file mode 100644 index e69de29..0000000 diff --git a/balanced_clustering/__init__.py b/balanced_clustering/__init__.py index 334f5b8..c87863b 100644 --- a/balanced_clustering/__init__.py +++ b/balanced_clustering/__init__.py @@ -1,5 +1,11 @@ -__version__ = "0.1.0" +# https://github.com/python-poetry/poetry/pull/2366#issuecomment-652418094 +# https://github.com/python-poetry/poetry/issues/144#issuecomment-623927302 +import importlib.metadata as importlib_metadata + +package_name = "balanced-clustering" +__version__ = importlib_metadata.version(package_name) + from .ari import balanced_adjusted_rand_index from .ami import balanced_adjusted_mutual_info from .vmeasure import balanced_homogeneity, balanced_completeness, balanced_v_measure -from .return_metrics import return_metrics \ No newline at end of file +from .return_metrics import return_metrics diff --git a/balanced_clustering/ami.py b/balanced_clustering/ami.py index 27feb5c..ea94492 100644 --- a/balanced_clustering/ami.py +++ b/balanced_clustering/ami.py @@ -84,17 +84,13 @@ def balanced_adjusted_mutual_info( ) # Recalculate labels_true and labels_pred if reweigh is True to # factor in the reweighting based on the true class frequencies. - # These won't preserve order but this is fine since entropy is + # These won't preserve order but this is fine since entropy is # invariant to order if reweigh is True: - true_sums = np.squeeze(np.asarray(sp.csc_matrix.sum(contingency, axis = 1))) - pred_sums = np.squeeze(np.asarray(sp.csc_matrix.sum(contingency, axis = 0))) - labels_true = np.repeat( - np.arange(len(true_sums)), true_sums - ) - labels_pred = np.repeat( - np.arange(len(pred_sums)), pred_sums - ) + true_sums = np.squeeze(np.asarray(sp.csc_matrix.sum(contingency, axis=1))) + pred_sums = np.squeeze(np.asarray(sp.csc_matrix.sum(contingency, axis=0))) + labels_true = np.repeat(np.arange(len(true_sums)), true_sums) + labels_pred = np.repeat(np.arange(len(pred_sums)), pred_sums) contingency = contingency.astype(np.float64) # Calculate the MI for the two clusterings mi = mutual_info_score(labels_true, labels_pred, contingency=contingency) diff --git a/balanced_clustering/return_metrics.py b/balanced_clustering/return_metrics.py index 12ad38b..a3623b6 100644 --- a/balanced_clustering/return_metrics.py +++ b/balanced_clustering/return_metrics.py @@ -1,15 +1,21 @@ -from sklearn.metrics import adjusted_rand_score, adjusted_mutual_info_score, \ - homogeneity_score, completeness_score, v_measure_score +from sklearn.metrics import ( + adjusted_rand_score, + adjusted_mutual_info_score, + homogeneity_score, + completeness_score, + v_measure_score, +) from .ari import balanced_adjusted_rand_index from .ami import balanced_adjusted_mutual_info from .vmeasure import balanced_homogeneity, balanced_completeness, balanced_v_measure -def return_metrics(class_arr, cluster_arr, print_metrics = True): - ''' - Compare imbalanced and balanced ARI, AMI, homogeneity, completeness, and + +def return_metrics(class_arr, cluster_arr, print_metrics=True): + """ + Compare imbalanced and balanced ARI, AMI, homogeneity, completeness, and V-measure scores. - + Parameters ---------- class_arr : int array-like of shape (n_samples,) @@ -19,7 +25,7 @@ def return_metrics(class_arr, cluster_arr, print_metrics = True): print_metrics : bool, default=True If True, print the scores. If False, return the scores. - + Returns ------- ari_imbalanced : float @@ -42,9 +48,9 @@ def return_metrics(class_arr, cluster_arr, print_metrics = True): The imbalanced V-measure score. v_measure_balanced : float The balanced V-measure score. - ''' - - # Determine the imbalanced (base) metric scores + """ + + # Determine the imbalanced (base) metric scores ari_imbalanced = adjusted_rand_score(class_arr, cluster_arr) ami_imbalanced = adjusted_mutual_info_score(class_arr, cluster_arr) homog_imbalanced = homogeneity_score(class_arr, cluster_arr) @@ -57,31 +63,50 @@ def return_metrics(class_arr, cluster_arr, print_metrics = True): homog_balanced = balanced_homogeneity(class_arr, cluster_arr) complete_balanced = balanced_completeness(class_arr, cluster_arr) v_measure_balanced = balanced_v_measure(class_arr, cluster_arr) - + # If print is True, print the scores if print_metrics: print( - "ARI imbalanced: " + str(round(ari_imbalanced, 4)) + " " + - "ARI balanced: " + str(round(ari_balanced, 4)) + "ARI imbalanced: " + + str(round(ari_imbalanced, 4)) + + " " + + "ARI balanced: " + + str(round(ari_balanced, 4)) ) print( - "AMI imbalanced: " + str(round(ami_imbalanced, 4)) + " " + - "AMI balanced: " + str(round(ami_balanced, 4)) + "AMI imbalanced: " + + str(round(ami_imbalanced, 4)) + + " " + + "AMI balanced: " + + str(round(ami_balanced, 4)) ) print( - "Homogeneity imbalanced: " + str(round(homog_imbalanced, 4)) + " " + - "Homogeneity balanced: " + str(round(homog_balanced, 4)) + "Homogeneity imbalanced: " + + str(round(homog_imbalanced, 4)) + + " " + + "Homogeneity balanced: " + + str(round(homog_balanced, 4)) ) print( - "Completeness imbalanced: " + str(round(complete_imbalanced, 4)) + " " + - "Completeness balanced : " + str(round(complete_balanced, 4)) + "Completeness imbalanced: " + + str(round(complete_imbalanced, 4)) + + " " + + "Completeness balanced : " + + str(round(complete_balanced, 4)) ) print( - "V-measure imbalanced: " + str(round(v_measure_imbalanced, 4)) + " " + - "V-measure balanced: " + str(round(v_measure_balanced, 4)) + "V-measure imbalanced: " + + str(round(v_measure_imbalanced, 4)) + + " " + + "V-measure balanced: " + + str(round(v_measure_balanced, 4)) ) - + # Return paired balanced imbalanced scores - return (ari_imbalanced, ari_balanced), (ami_imbalanced, ami_balanced), \ - (homog_imbalanced, homog_balanced), (complete_imbalanced, complete_balanced), \ - (v_measure_imbalanced, v_measure_balanced) \ No newline at end of file + return ( + (ari_imbalanced, ari_balanced), + (ami_imbalanced, ami_balanced), + (homog_imbalanced, homog_balanced), + (complete_imbalanced, complete_balanced), + (v_measure_imbalanced, v_measure_balanced), + ) diff --git a/balanced_clustering/utils/__init__.py b/balanced_clustering/utils/__init__.py index ffde74c..a0c5687 100644 --- a/balanced_clustering/utils/__init__.py +++ b/balanced_clustering/utils/__init__.py @@ -1,8 +1,6 @@ -import pyximport import numpy -pyximport.install(setup_args={"include_dirs": numpy.get_include()}, reload_support=True) -from ._emi_cython import expected_mutual_information +from ._emi import expected_mutual_information from .contingency import pair_confusion_matrix, contingency_matrix from .checks import check_clusterings from .mi import mutual_info_score, entropy diff --git a/balanced_clustering/utils/_emi.py b/balanced_clustering/utils/_emi.py new file mode 100644 index 0000000..926e954 --- /dev/null +++ b/balanced_clustering/utils/_emi.py @@ -0,0 +1,78 @@ +# Authors: Robert Layton +# Corey Lynch +# License: BSD 3 clause + +import numpy as np +import numba +from scipy.sparse import spmatrix +from math import exp, lgamma + + +@numba.njit(fastmath=True, cache=True, parallel=True) +def _emi(a, b, N): + R = len(a) + C = len(b) + # There are three major terms to the EMI equation, which are multiplied to + # and then summed over varying nij values. + # While nijs[0] will never be used, having it simplifies the indexing. + nijs = np.arange(0.0, float(max(np.max(a), np.max(b)) + 1)) + nijs[0] = 1 # Stops divide by zero warnings. As its not used, no issue. + # term1 is nij / N + term1 = nijs / N + # term2 is log((N*nij) / (a * b)) == log(N * nij) - log(a * b) + log_a = np.log(a) + log_b = np.log(b) + # term2 uses log(N * nij) = log(N) + log(nij) + log_Nnij = np.log(N) + np.log(nijs) + # term3 is large, and involved many factorials. Calculate these in log + # space to stop overflows. + gln_a = [] + gln_Na = [] + for ai in a: + gln_a.append(lgamma(ai + 1)) + gln_Na.append(lgamma(N - ai + 1)) + gln_b = [] + gln_Nb = [] + for bi in b: + gln_b.append(lgamma(bi + 1)) + gln_Nb.append(lgamma(N - bi + 1)) + gln_N = lgamma(N + 1) + gln_nij = [lgamma(nijs_i + 1) for nijs_i in nijs] + # start and end values for nij terms for each summation. + # start = np.array([[v - N + w for w in b] for v in a]) + start = np.zeros((R, C)) + end = np.zeros((R, C)) + for r in range(R): + for c in range(C): + start[r, c] = a[r] + b[c] - N + end[r, c] = min(a[r], b[c]) + 1 + start = np.maximum(start, 1) + # emi itself is a summation over the various values. + emi = 0.0 + for i in range(R): + for j in range(C): + for nij in range(start[i, j], end[i, j]): + term2 = log_Nnij[nij] - log_a[i] - log_b[j] + # Numerators are positive, denominators are negative. + gln = ( + gln_a[i] + + gln_b[j] + + gln_Na[i] + + gln_Nb[j] + - gln_N + - gln_nij[nij] + - lgamma(a[i] - nij + 1) + - lgamma(b[j] - nij + 1) + - lgamma(N - a[i] - b[j] + nij + 1) + ) + term3 = exp(gln) + emi += term1[nij] * term2 * term3 + return emi + + +def expected_mutual_information(contingency: spmatrix, n_samples: int): + """Calculate the expected mutual information for two labelings.""" + N = n_samples + a = np.ravel(contingency.sum(axis=1).astype(np.int32, copy=False)) + b = np.ravel(contingency.sum(axis=0).astype(np.int32, copy=False)) + return _emi(a, b, N) diff --git a/balanced_clustering/utils/_emi_cython.pyx b/balanced_clustering/utils/_emi_cython.pyx deleted file mode 100644 index ae403b5..0000000 --- a/balanced_clustering/utils/_emi_cython.pyx +++ /dev/null @@ -1,65 +0,0 @@ -# Authors: Robert Layton -# Corey Lynch -# License: BSD 3 clause - -from libc.math cimport exp, lgamma -from scipy.special import gammaln -import numpy as np -cimport numpy as np -cimport cython - -np.import_array() -ctypedef np.float64_t DOUBLE - -def expected_mutual_information(contingency, int n_samples): - """Calculate the expected mutual information for two labelings.""" - cdef int R, C - cdef DOUBLE N, gln_N, emi, term2, term3, gln - cdef np.ndarray[DOUBLE] gln_a, gln_b, gln_Na, gln_Nb, gln_nij, log_Nnij - cdef np.ndarray[DOUBLE] nijs, term1 - cdef np.ndarray[DOUBLE] log_a, log_b - cdef np.ndarray[np.int32_t] a, b - #cdef np.ndarray[int, ndim=2] start, end - R, C = contingency.shape - N = n_samples - a = np.ravel(contingency.sum(axis=1).astype(np.int32, copy=False)) - b = np.ravel(contingency.sum(axis=0).astype(np.int32, copy=False)) - # There are three major terms to the EMI equation, which are multiplied to - # and then summed over varying nij values. - # While nijs[0] will never be used, having it simplifies the indexing. - nijs = np.arange(0, max(np.max(a), np.max(b)) + 1, dtype='float') - nijs[0] = 1 # Stops divide by zero warnings. As its not used, no issue. - # term1 is nij / N - term1 = nijs / N - # term2 is log((N*nij) / (a * b)) == log(N * nij) - log(a * b) - log_a = np.log(a) - log_b = np.log(b) - # term2 uses log(N * nij) = log(N) + log(nij) - log_Nnij = np.log(N) + np.log(nijs) - # term3 is large, and involved many factorials. Calculate these in log - # space to stop overflows. - gln_a = gammaln(a + 1) - gln_b = gammaln(b + 1) - gln_Na = gammaln(N - a + 1) - gln_Nb = gammaln(N - b + 1) - gln_N = gammaln(N + 1) - gln_nij = gammaln(nijs + 1) - # start and end values for nij terms for each summation. - start = np.array([[v - N + w for w in b] for v in a], dtype='int') - start = np.maximum(start, 1) - end = np.minimum(np.resize(a, (C, R)).T, np.resize(b, (R, C))) + 1 - # emi itself is a summation over the various values. - emi = 0.0 - cdef Py_ssize_t i, j, nij - for i in range(R): - for j in range(C): - for nij in range(start[i,j], end[i,j]): - term2 = log_Nnij[nij] - log_a[i] - log_b[j] - # Numerators are positive, denominators are negative. - gln = (gln_a[i] + gln_b[j] + gln_Na[i] + gln_Nb[j] - - gln_N - gln_nij[nij] - lgamma(a[i] - nij + 1) - - lgamma(b[j] - nij + 1) - - lgamma(N - a[i] - b[j] + nij + 1)) - term3 = exp(gln) - emi += (term1[nij] * term2 * term3) - return emi \ No newline at end of file diff --git a/balanced_clustering/utils/checks.py b/balanced_clustering/utils/checks.py index 72ba989..fb4f5e4 100644 --- a/balanced_clustering/utils/checks.py +++ b/balanced_clustering/utils/checks.py @@ -491,7 +491,6 @@ def check_array( ensure_min_features=1, estimator=None, ): - """Input validation on an array, list, sparse matrix or similar. By default, the input is checked to be a non-empty 2D array containing only finite values. If the dtype of the array is object, attempt diff --git a/balanced_clustering/vmeasure.py b/balanced_clustering/vmeasure.py index f8fe2bc..44786ba 100644 --- a/balanced_clustering/vmeasure.py +++ b/balanced_clustering/vmeasure.py @@ -58,7 +58,7 @@ def balanced_homogeneity_completeness_v_measure( if len(labels_true) == 0: return 1.0, 1.0, 1.0 - + contingency = contingency_matrix( labels_true, labels_pred, reweigh=reweigh, sparse=True ) @@ -66,17 +66,13 @@ def balanced_homogeneity_completeness_v_measure( # Recalculate labels_true and labels_pred if reweigh is True to # factor in the reweighting based on the true class frequencies. - # These won't preserve order but this is fine since entropy is + # These won't preserve order but this is fine since entropy is # invariant to order if reweigh is True: - true_sums = np.squeeze(np.asarray(sp.csc_matrix.sum(contingency, axis = 1))) - pred_sums = np.squeeze(np.asarray(sp.csc_matrix.sum(contingency, axis = 0))) - labels_true = np.repeat( - np.arange(len(true_sums)), true_sums - ) - labels_pred = np.repeat( - np.arange(len(pred_sums)), pred_sums - ) + true_sums = np.squeeze(np.asarray(sp.csc_matrix.sum(contingency, axis=1))) + pred_sums = np.squeeze(np.asarray(sp.csc_matrix.sum(contingency, axis=0))) + labels_true = np.repeat(np.arange(len(true_sums)), true_sums) + labels_pred = np.repeat(np.arange(len(pred_sums)), pred_sums) entropy_C = entropy(labels_true) entropy_K = entropy(labels_pred) diff --git a/dist/imbalanced-clustering-0.1.0.tar.gz b/dist/imbalanced-clustering-0.1.0.tar.gz deleted file mode 100644 index e35c9c0..0000000 Binary files a/dist/imbalanced-clustering-0.1.0.tar.gz and /dev/null differ diff --git a/dist/imbalanced_clustering-0.1.0-py3-none-any.whl b/dist/imbalanced_clustering-0.1.0-py3-none-any.whl deleted file mode 100644 index e34731d..0000000 Binary files a/dist/imbalanced_clustering-0.1.0-py3-none-any.whl and /dev/null differ diff --git a/experiments/01_imbalanced_metric_expectation.py b/experiments/01_imbalanced_metric_expectation.py index 7dc4fe7..a7918b6 100644 --- a/experiments/01_imbalanced_metric_expectation.py +++ b/experiments/01_imbalanced_metric_expectation.py @@ -3,18 +3,28 @@ import pandas as pd import matplotlib.pyplot as plt import seaborn as sns -from sklearn.metrics import adjusted_rand_score, adjusted_mutual_info_score, \ - homogeneity_score, completeness_score, v_measure_score +from sklearn.metrics import ( + adjusted_rand_score, + adjusted_mutual_info_score, + homogeneity_score, + completeness_score, + v_measure_score, +) from sklearn.cluster import KMeans -from balanced_clustering import balanced_adjusted_rand_index, \ - balanced_adjusted_mutual_info, balanced_completeness, \ - balanced_homogeneity, balanced_v_measure +from balanced_clustering import ( + balanced_adjusted_rand_index, + balanced_adjusted_mutual_info, + balanced_completeness, + balanced_homogeneity, + balanced_v_measure, +) + # Define function to return all the relevant metrics - both balanced # and imbalanced def return_metrics(class_arr, cluster_arr): - # Determine the imbalanced (base) metric scores + # Determine the imbalanced (base) metric scores ari_imbalanced = adjusted_rand_score(class_arr, cluster_arr) ami_imbalanced = adjusted_mutual_info_score(class_arr, cluster_arr) homog_imbalanced = homogeneity_score(class_arr, cluster_arr) @@ -27,23 +37,28 @@ def return_metrics(class_arr, cluster_arr): homog_balanced = balanced_homogeneity(class_arr, cluster_arr) complete_balanced = balanced_completeness(class_arr, cluster_arr) v_measure_balanced = balanced_v_measure(class_arr, cluster_arr) - + # Return paired balanced imbalance scores - return (ari_imbalanced, ari_balanced), (ami_imbalanced, ami_balanced), \ - (homog_imbalanced, homog_balanced), (complete_imbalanced, complete_balanced), \ - (v_measure_imbalanced, v_measure_balanced) - + return ( + (ari_imbalanced, ari_balanced), + (ami_imbalanced, ami_balanced), + (homog_imbalanced, homog_balanced), + (complete_imbalanced, complete_balanced), + (v_measure_imbalanced, v_measure_balanced), + ) + + # Define function for generating completely random data from uniform -# distributions, clustering and scoring results +# distributions, clustering and scoring results def random_data(num_classes, num_clusters, min_class_size, max_class_size): # Ensure number of classes and clusters is greater than 1 assert num_classes > 1 and num_clusters > 1 - + # Sample class sizes from uniform distribution class_size_samples = [ np.random.randint(min_class_size, max_class_size) for i in range(num_classes) ] - + # Iterate over each class and generate random data x_1_all = [] x_2_all = [] @@ -51,48 +66,51 @@ def random_data(num_classes, num_clusters, min_class_size, max_class_size): for i in range(num_classes): # Sample class values from uniform distribution x = np.random.uniform(0, 100, size=(class_size_samples[i], 2)) - # Append values to list + # Append values to list x_1_all.append(x[:, 0]) - x_2_all.append(x[:, 1]) - class_vals.append(i * np.ones(class_size_samples[i], dtype = "int")) - - # Concatenate all class values into single dataframe - class_df = pd.DataFrame({ - "x": np.concatenate(x_1_all), - "y": np.concatenate(x_2_all), - "class": np.concatenate(class_vals) - }) - + x_2_all.append(x[:, 1]) + class_vals.append(i * np.ones(class_size_samples[i], dtype="int")) + + # Concatenate all class values into single dataframe + class_df = pd.DataFrame( + { + "x": np.concatenate(x_1_all), + "y": np.concatenate(x_2_all), + "class": np.concatenate(class_vals), + } + ) + # Perform clustering with given number of clusters class_arr = np.array(class_df.iloc[:, 0:2]) - kmeans_res = KMeans(n_clusters = num_clusters).fit_predict(X = class_arr) + kmeans_res = KMeans(n_clusters=num_clusters).fit_predict(X=class_arr) class_df["kmeans"] = kmeans_res - - # Get all metrics and return + + # Get all metrics and return aris, amis, homogs, completes, v_measures = return_metrics( - class_arr = class_df["class"].__array__(), - cluster_arr = class_df["kmeans"].__array__() + class_arr=class_df["class"].__array__(), + cluster_arr=class_df["kmeans"].__array__(), ) return aris, amis, homogs, completes, v_measures - + + # Define function for generating perfectly separated isotropic Gaussian -# distributions +# distributions def separated_gaussians(num_classes, num_clusters, min_class_size, max_class_size): # Ensure number of classes and clusters is greater than 1 assert num_classes > 1 and num_clusters > 1 - + # Sample class sizes from uniform distribution class_size_samples = [ np.random.randint(min_class_size, max_class_size) for size in range(num_classes) ] - + # Sample centers for Gaussian distributions gaussian_center_interval = np.arange(0, 1000, 10) gaussian_centers = [ - np.random.choice(gaussian_center_interval, num_classes, replace = False) \ - for center in range(num_classes) + np.random.choice(gaussian_center_interval, num_classes, replace=False) + for center in range(num_classes) ] - + # Sample values from 0.1 stdev Gauassian distributions x_1_all = [] x_2_all = [] @@ -100,112 +118,113 @@ def separated_gaussians(num_classes, num_clusters, min_class_size, max_class_siz for i in range(num_classes): # Sample class values from Gaussian distribution x = np.random.normal( - loc = gaussian_centers[i][0], - scale = 0.1, - size=(class_size_samples[i], 2) + loc=gaussian_centers[i][0], scale=0.1, size=(class_size_samples[i], 2) ) - # Append values to list + # Append values to list x_1_all.append(x[:, 0]) - x_2_all.append(x[:, 1]) - class_vals.append(i * np.ones(class_size_samples[i], dtype = "int")) - - # Concatenate all class values into single dataframe - class_df = pd.DataFrame({ - "x": np.concatenate(x_1_all), - "y": np.concatenate(x_2_all), - "class": np.concatenate(class_vals) - }) - + x_2_all.append(x[:, 1]) + class_vals.append(i * np.ones(class_size_samples[i], dtype="int")) + + # Concatenate all class values into single dataframe + class_df = pd.DataFrame( + { + "x": np.concatenate(x_1_all), + "y": np.concatenate(x_2_all), + "class": np.concatenate(class_vals), + } + ) + # Perform clustering with given number of clusters class_arr = np.array(class_df.iloc[:, 0:2]) - kmeans_res = KMeans(n_clusters = num_clusters).fit_predict(X = class_arr) + kmeans_res = KMeans(n_clusters=num_clusters).fit_predict(X=class_arr) class_df["kmeans"] = kmeans_res - - # Get all metrics and return + + # Get all metrics and return aris, amis, homogs, completes, v_measures = return_metrics( - class_arr = class_df["class"], cluster_arr = class_df["kmeans"] + class_arr=class_df["class"], cluster_arr=class_df["kmeans"] ) return aris, amis, homogs, completes, v_measures + # Define main expectation function -def main(num_trials = 1000): - # Define necessary values of importance +def main(num_trials=1000): + # Define necessary values of importance ari_random = [] ami_random = [] homog_random = [] complete_random = [] vmeasure_random = [] - + bal_ari_random = [] bal_ami_random = [] bal_homog_random = [] bal_complete_random = [] bal_vmeasure_random = [] - + ari_sep = [] ami_sep = [] homog_sep = [] complete_sep = [] vmeasure_sep = [] - + bal_ari_sep = [] bal_ami_sep = [] bal_homog_sep = [] bal_complete_sep = [] bal_vmeasure_sep = [] - + # Define ranges for key values class_size_range = [2, 3, 4, 5, 6, 7, 8, 9, 10] cluster_size_range = [2, 3, 4, 5, 6, 7, 8, 9, 10] - + # Iterate over given trials and generate random data for i in range(num_trials): # Sample class and cluster numbers num_classes = np.random.choice(class_size_range) num_clusters = np.random.choice(cluster_size_range) - - # Get scores from generating random data + + # Get scores from generating random data aris, amis, homogs, completes, v_measures = random_data( - num_classes = num_classes, - num_clusters = num_clusters, - min_class_size = 50, - max_class_size = 2000 + num_classes=num_classes, + num_clusters=num_clusters, + min_class_size=50, + max_class_size=2000, ) - + # Append scores for random data ari_random.append(aris[0]) ami_random.append(amis[0]) homog_random.append(homogs[0]) complete_random.append(completes[0]) vmeasure_random.append(v_measures[0]) - + bal_ari_random.append(aris[1]) bal_ami_random.append(amis[1]) bal_homog_random.append(homogs[1]) bal_complete_random.append(completes[1]) bal_vmeasure_random.append(v_measures[1]) - + # Iterate over given trials and generate separated data for i in range(num_trials): # Sample class and cluster numbers (equal here to simulate perfect case) num_classes = np.random.choice(class_size_range) num_clusters = num_classes - - # Get scores from generating random data + + # Get scores from generating random data aris, amis, homogs, completes, v_measures = separated_gaussians( - num_classes = num_classes, - num_clusters = num_clusters, - min_class_size = 50, - max_class_size = 2000 + num_classes=num_classes, + num_clusters=num_clusters, + min_class_size=50, + max_class_size=2000, ) - + # Append scores for random data ari_sep.append(aris[0]) ami_sep.append(amis[0]) homog_sep.append(homogs[0]) complete_sep.append(completes[0]) vmeasure_sep.append(v_measures[0]) - + bal_ari_sep.append(aris[1]) bal_ami_sep.append(amis[1]) bal_homog_sep.append(homogs[1]) @@ -213,123 +232,125 @@ def main(num_trials = 1000): bal_vmeasure_sep.append(v_measures[1]) # Create dataframe of results and save - res_df = pd.DataFrame({ - "ari_random": ari_random, - "bal_ari_random": bal_ari_random, - "ami_random": ami_random, - "bal_ami_random": bal_ami_random, - "homog_random": homog_random, - "bal_homog_random": bal_homog_random, - "complete_random": complete_random, - "bal_complete_random": bal_complete_random, - "vmeasure_random": vmeasure_random, - "bal_vmeasure_random": bal_vmeasure_random, - "ari_sep": ari_sep, - "bal_ari_sep": bal_ari_sep, - "ami_sep": ami_sep, - "bal_ami_sep": bal_ami_sep, - "homog_sep": homog_sep, - "bal_homog_sep": bal_homog_sep, - "complete_sep": complete_sep, - "bal_complete_sep": bal_complete_sep, - "vmeasure_sep": vmeasure_sep, - "bal_vmeasure_sep": bal_vmeasure_sep, - "trial_num": np.arange(num_trials) - }) - res_df.to_csv("../outs/01_expectation_results_full.tsv", sep = "\t") - + res_df = pd.DataFrame( + { + "ari_random": ari_random, + "bal_ari_random": bal_ari_random, + "ami_random": ami_random, + "bal_ami_random": bal_ami_random, + "homog_random": homog_random, + "bal_homog_random": bal_homog_random, + "complete_random": complete_random, + "bal_complete_random": bal_complete_random, + "vmeasure_random": vmeasure_random, + "bal_vmeasure_random": bal_vmeasure_random, + "ari_sep": ari_sep, + "bal_ari_sep": bal_ari_sep, + "ami_sep": ami_sep, + "bal_ami_sep": bal_ami_sep, + "homog_sep": homog_sep, + "bal_homog_sep": bal_homog_sep, + "complete_sep": complete_sep, + "bal_complete_sep": bal_complete_sep, + "vmeasure_sep": vmeasure_sep, + "bal_vmeasure_sep": bal_vmeasure_sep, + "trial_num": np.arange(num_trials), + } + ) + res_df.to_csv("../outs/01_expectation_results_full.tsv", sep="\t") + # Get the mean and standard deviation of each metric ari_random_mean = np.mean(ari_random) ari_random_stdev = np.std(ari_random) - + ami_random_mean = np.mean(ami_random) ami_random_stdev = np.std(ami_random) - + homog_random_mean = np.mean(homog_random) homog_random_stdev = np.std(homog_random) - + complete_random_mean = np.mean(complete_random) complete_random_stdev = np.std(complete_random) - + vmeasure_random_mean = np.mean(vmeasure_random) vmeasure_random_stdev = np.std(vmeasure_random) - + bal_ari_random_mean = np.mean(bal_ari_random) bal_ari_random_stdev = np.std(bal_ari_random) - + bal_ami_random_mean = np.mean(bal_ami_random) bal_ami_random_stdev = np.std(bal_ami_random) - + bal_homog_random_mean = np.mean(bal_homog_random) bal_homog_random_stdev = np.std(bal_homog_random) - + bal_complete_random_mean = np.mean(bal_complete_random) bal_complete_random_stdev = np.std(bal_complete_random) - + bal_vmeasure_random_mean = np.mean(bal_vmeasure_random) bal_vmeasure_random_stdev = np.std(bal_vmeasure_random) - + ari_sep_mean = np.mean(ari_sep) ari_sep_stdev = np.std(ari_sep) - + ami_sep_mean = np.mean(ami_sep) ami_sep_stdev = np.std(ami_sep) - + homog_sep_mean = np.mean(homog_sep) homog_sep_stdev = np.std(homog_sep) - + complete_sep_mean = np.mean(complete_sep) complete_sep_stdev = np.std(complete_sep) - + vmeasure_sep_mean = np.mean(vmeasure_sep) vmeasure_sep_stdev = np.std(vmeasure_sep) - + bal_ari_sep_mean = np.mean(bal_ari_sep) bal_ari_sep_stdev = np.std(bal_ari_sep) - + bal_ami_sep_mean = np.mean(bal_ami_sep) bal_ami_sep_stdev = np.std(bal_ami_sep) - + bal_homog_sep_mean = np.mean(bal_homog_sep) bal_homog_sep_stdev = np.std(bal_homog_sep) - + bal_complete_sep_mean = np.mean(bal_complete_sep) bal_complete_sep_stdev = np.std(bal_complete_sep) - + bal_vmeasure_sep_mean = np.mean(bal_vmeasure_sep) bal_vmeasure_sep_stdev = np.std(bal_vmeasure_sep) - - # Assert that all means are close to expected value - assert np.isclose(ari_random_mean, 0, atol = 0.01) - assert np.isclose(ami_random_mean, 0, atol = 0.01) - assert np.isclose(homog_random_mean, 0, atol = 0.01) - assert np.isclose(complete_random_mean, 0, atol = 0.01) - assert np.isclose(vmeasure_random_mean, 0, atol = 0.01) - - assert np.isclose(bal_ari_random_mean, 0, atol = 0.01) - assert np.isclose(bal_ami_random_mean, 0, atol = 0.01) - assert np.isclose(bal_homog_random_mean, 0, atol = 0.01) - assert np.isclose(bal_complete_random_mean, 0, atol = 0.01) - assert np.isclose(bal_vmeasure_random_mean, 0, atol = 0.01) - + + # Assert that all means are close to expected value + assert np.isclose(ari_random_mean, 0, atol=0.01) + assert np.isclose(ami_random_mean, 0, atol=0.01) + assert np.isclose(homog_random_mean, 0, atol=0.01) + assert np.isclose(complete_random_mean, 0, atol=0.01) + assert np.isclose(vmeasure_random_mean, 0, atol=0.01) + + assert np.isclose(bal_ari_random_mean, 0, atol=0.01) + assert np.isclose(bal_ami_random_mean, 0, atol=0.01) + assert np.isclose(bal_homog_random_mean, 0, atol=0.01) + assert np.isclose(bal_complete_random_mean, 0, atol=0.01) + assert np.isclose(bal_vmeasure_random_mean, 0, atol=0.01) + print("Random clustering asserts passed") - - assert np.isclose(ari_sep_mean, 1, atol = 0.01) - assert np.isclose(ami_sep_mean, 1, atol = 0.01) - assert np.isclose(homog_sep_mean, 1, atol = 0.01) - assert np.isclose(complete_sep_mean, 1, atol = 0.01) - assert np.isclose(vmeasure_sep_mean, 1, atol = 0.01) - - assert np.isclose(bal_ari_sep_mean, 1, atol = 0.01) - assert np.isclose(bal_ami_sep_mean, 1, atol = 0.01) - assert np.isclose(bal_homog_sep_mean, 1, atol = 0.01) - assert np.isclose(bal_complete_sep_mean, 1, atol = 0.01) - assert np.isclose(bal_vmeasure_sep_mean, 1, atol = 0.01) - + + assert np.isclose(ari_sep_mean, 1, atol=0.01) + assert np.isclose(ami_sep_mean, 1, atol=0.01) + assert np.isclose(homog_sep_mean, 1, atol=0.01) + assert np.isclose(complete_sep_mean, 1, atol=0.01) + assert np.isclose(vmeasure_sep_mean, 1, atol=0.01) + + assert np.isclose(bal_ari_sep_mean, 1, atol=0.01) + assert np.isclose(bal_ami_sep_mean, 1, atol=0.01) + assert np.isclose(bal_homog_sep_mean, 1, atol=0.01) + assert np.isclose(bal_complete_sep_mean, 1, atol=0.01) + assert np.isclose(bal_vmeasure_sep_mean, 1, atol=0.01) + print("Separated clustering asserts passed") - + # Create dataframe of mean and stdev results for each metric - # and save + # and save res_df_mean_stdev = pd.DataFrame( { "ari_random_mean": ari_random_mean, @@ -371,12 +392,13 @@ def main(num_trials = 1000): "bal_complete_sep_mean": bal_complete_sep_mean, "bal_complete_sep_stdev": bal_complete_sep_stdev, "bal_vmeasure_sep_mean": bal_vmeasure_sep_mean, - "bal_vmeasure_sep_stdev": bal_vmeasure_sep_stdev + "bal_vmeasure_sep_stdev": bal_vmeasure_sep_stdev, }, - index = [0] + index=[0], ) - res_df_mean_stdev.to_csv("../outs/01_expectation_results_mean_std.tsv", sep = "\t") - + res_df_mean_stdev.to_csv("../outs/01_expectation_results_mean_std.tsv", sep="\t") + + if __name__ == "__main__": - # Run main script - main(num_trials=1000) \ No newline at end of file + # Run main script + main(num_trials=1000) diff --git a/notebooks/01_imbalanced_metric_demo.ipynb b/notebooks/01_imbalanced_metric_demo.ipynb index 988c9bb..0aa1216 100644 --- a/notebooks/01_imbalanced_metric_demo.ipynb +++ b/notebooks/01_imbalanced_metric_demo.ipynb @@ -151,6 +151,9 @@ } ], "source": [ + "%pip install --quiet matplotlib\n", + "%pip install --quiet seaborn\n", + "\n", "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", diff --git a/notebooks/02_imbalanced_metric_interpolation_tests.ipynb b/notebooks/02_imbalanced_metric_interpolation_tests.ipynb index 018dd12..a224d5e 100644 --- a/notebooks/02_imbalanced_metric_interpolation_tests.ipynb +++ b/notebooks/02_imbalanced_metric_interpolation_tests.ipynb @@ -11,6 +11,17 @@ "fixed cluster/class sizes. " ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Pip install the necessary packages \n", + "%pip install --quiet matplotlib\n", + "%pip install --quiet seaborn" + ] + }, { "cell_type": "code", "execution_count": 1, diff --git a/pyproject.toml b/pyproject.toml index c68c2fb..37c4497 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,26 +1,22 @@ [tool.poetry] -name = "imbalanced-clustering" +name = "balanced-clustering" version = "0.1.0" description = "Clustering metrics for imbalanced datasets" authors = ["Hassaan Maan "] [tool.poetry.dependencies] -python = ">=3.8,<=3.10.0" -numpy = "^1.22.2" -scipy = "^1.8.0" -scikit-learn = "^1.0.2" -pandas = "^1.4.1" -Cython = "^0.29.28" -jupyter = "^1.0.0" -ipykernel = "^6.9.1" -jupyterlab = "^3.2.9" -jupyterlab-latex = "^3.1.0" -seaborn = "^0.11.2" +python = ">=3.8" +numpy = ">=1.13.3" +scipy = ">=1.4" +scikit-learn = ">=0.22" +pandas = ">=1.0.0" +numba = ">=0.41.0" [tool.poetry.dev-dependencies] -pytest = "^5.2" -black = "^22.1.0" -flake8 = "^3.8.3" +pytest = ">=5.2" +black = ">=22.1.0" +flake8 = ">=3.8.3" +pre-commit = ">=3.0.0" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/tests/test_imbalanced_clustering.py b/tests/test_balanced_clustering.py similarity index 100% rename from tests/test_imbalanced_clustering.py rename to tests/test_balanced_clustering.py diff --git a/tests/test_metrics.py b/tests/test_metrics.py index d8525e4..06e0c4b 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -9,7 +9,6 @@ balanced_adjusted_rand_index, balanced_adjusted_mutual_info, balanced_homogeneity, - balanced_completeness, balanced_v_measure, ) @@ -17,6 +16,7 @@ random.seed(42) np.random.seed(42) + # Fixture for loading gaussian blobs of 3 classes with 1 minority class @pytest.fixture def three_classes_one_small(): @@ -133,7 +133,8 @@ def test_bal_ari_2_class_balanced(two_classes_balanced): def test_bal_ari_3_class_mixed_imbalanced(three_classes_mixed_imbalanced): - # Perform k-means clustering on three classes with mixed sizes and imbalanced/overlapping + # Perform k-means clustering on three classes with mixed sizes and + # imbalanced/overlapping class_cluster_df = k_means_df(three_classes_mixed_imbalanced, n_clusters=3) # Calculated balanced and imbalanced ARI @@ -181,7 +182,8 @@ def test_bal_ami_2_class_balanced(two_classes_balanced): def test_bal_ami_3_class_mixed_imbalanced(three_classes_mixed_imbalanced): - # Perform k-means clustering on three classes with mixed sizes and imbalanced/overlapping + # Perform k-means clustering on three classes with mixed sizes and + # imbalanced/overlapping class_cluster_df = k_means_df(three_classes_mixed_imbalanced, n_clusters=3) # Calculated balanced and imbalanced AMI @@ -212,22 +214,6 @@ def test_bal_homogeneity_3_class_1_small(three_classes_one_small): assert bal_homog < imbal_homog -def test_bal_completeness_3_class_1_small(three_classes_one_small): - # Perform k-means clustering on three classes with one minority class - class_cluster_df = k_means_df(three_classes_one_small, n_clusters=2) - - # Calculated balanced and imbalanced completeness - bal_comp = balanced_completeness( - class_cluster_df["cluster"], class_cluster_df["kmeans"], reweigh=True - ) - imbal_comp = balanced_completeness( - class_cluster_df["cluster"], class_cluster_df["kmeans"], reweigh=False - ) - - # Ensure that the balanced completeness is lower than the imbalanced completeness - assert bal_comp < imbal_comp - - def test_bal_v_measure_3_class_1_small(three_classes_one_small): # Perform k-means clustering on three classes with one minority class class_cluster_df = k_means_df(three_classes_one_small, n_clusters=2)