+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Clusters of entry stations over time
+
+
+
+
+
+ Select Sankey options :
+
+
+ node width =
+
+
+
+ node padding =
+
+
+
+ width =
+
+
+
+ height =
+
+
+
+
+ tick font size =
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Select entry cluster
+
+
+
+
+
+ Clusters of entry stations
+
+
+
+ Clusters of exit stations
+
+
+
+ 
+
+
+
+
+ 
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
diff --git a/clean_package.py b/clean_package.py
new file mode 100644
index 0000000..e89d767
--- /dev/null
+++ b/clean_package.py
@@ -0,0 +1,53 @@
+"""Function to clean all temporary files from the repo"""
+
+import shutil
+import os
+
+
+DEBUG_DIR = './dcblockmodels/model_debug_output/'
+NOTEBOOK_DIR = './notebooks/'
+
+
+def delete_dir(dirpath):
+ if os.path.exists(dirpath) and os.path.isdir(dirpath):
+ shutil.rmtree(dirpath)
+
+
+def delete_file(filename):
+ if os.path.exists(filename):
+ os.remove(filename)
+
+
+def clean():
+ if os.path.exists(DEBUG_DIR):
+ for file in os.listdir(DEBUG_DIR):
+ shutil.rmtree(DEBUG_DIR + file)
+ delete_dir(DEBUG_DIR)
+
+ for file in os.listdir(NOTEBOOK_DIR):
+ ext = file.split('.')[-1]
+ if ext in ['npy', 'npz']:
+ os.remove(NOTEBOOK_DIR + file)
+
+ delete_dir('./datasets/classic')
+ delete_dir('./saved_models')
+
+ delete_file('sparsebm.log')
+ delete_file(f'{NOTEBOOK_DIR}sparsebm.log')
+ delete_file('./dcblockmodels/sparsebm.log')
+
+ delete_dir('.ipynb_checkpoints')
+ delete_dir(f'{NOTEBOOK_DIR}/.ipynb_checkpoints')
+
+ delete_dir('./dcblockmodels/tests/.pytest_cache')
+ delete_dir('./dcblockmodels/models/.pytest_cache')
+ delete_dir('.pytest_cache')
+
+ delete_dir('./dcblockmodels/__pycache__')
+ delete_dir('./dcblockmodels/models/__pycache__')
+ delete_dir('./dcblockmodels/models/utils/__pycache__')
+ delete_dir('./dcblockmodels/tests/__pycache__')
+
+
+if __name__ == '__main__':
+ clean()
diff --git a/dcblockmodels/__init__.py b/dcblockmodels/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/dcblockmodels/data.py b/dcblockmodels/data.py
new file mode 100644
index 0000000..1e8dc82
--- /dev/null
+++ b/dcblockmodels/data.py
@@ -0,0 +1,407 @@
+"""Methods to sample data from static or dynamic LBM/SBM"""
+
+import numpy as np
+
+
+def sample_edge(mu_it, nu_it, gamma_tkl, block_sparsity_matrix_tkl):
+ """
+ Sample a given edge with paramters mu_it, nu_it, gamma_tkl
+ in a Poisson distribution, while keeping a sparsity of block_sparsity_matrix_tkl
+ (defined in [0, 1))
+ """
+ if np.random.rand() < block_sparsity_matrix_tkl:
+ return 0
+ return np.random.poisson(lam=mu_it * nu_it * gamma_tkl)
+
+
+def generate_data(
+ T,
+ model_type,
+ dimensions,
+ n_clusters,
+ prior_init,
+ prior_trans,
+ gamma,
+ with_margins,
+ margins,
+ self_loops,
+ directed,
+ noise_level,
+ with_absent_nodes,
+ absent_nodes,
+ dtype,
+ block_sparsity_matrix):
+ """
+ noise_level is the std of gaussian noise
+ block_sparsity_matrix is 1 - connectivity_matrix
+ in the binary case
+ """
+ assert model_type in ['SBM', 'LBM']
+ assert with_margins in [False, True]
+ assert dtype in ['int32', 'int64']
+
+ # parses the inputs
+ N = dimensions['N']
+ Kz = n_clusters['Kz']
+ alpha = prior_init['alpha']
+ if model_type == 'LBM':
+ beta = prior_init['beta']
+ D = dimensions['D']
+ Kw = n_clusters['Kw']
+ elif model_type == 'SBM':
+ D = N
+ Kw = Kz
+ W = None # for consistency
+
+ print('Data characteristics: ')
+ print(f' - T, N, D = {T, N, D}')
+ if T == 1:
+ print(' - Static ' + model_type)
+ else:
+ print(' - Dynamic ' + model_type)
+
+ if T > 1:
+ pi = prior_trans['pi']
+ if model_type == 'LBM':
+ rho = prior_trans['rho']
+
+ if not directed:
+ assert model_type == 'SBM'
+ print(' - Undirected')
+ else:
+ if model_type == 'SBM':
+ print(' - Directed')
+
+ if not self_loops:
+ assert model_type == 'SBM' or N == D
+ print(' - Without self-loops')
+ else:
+ print(' - With self-loops')
+
+ if block_sparsity_matrix is not None:
+ assert block_sparsity_matrix.shape == gamma.shape
+ else:
+ block_sparsity_matrix = np.zeros_like(gamma, dtype=dtype)
+
+ # the margins are either generated in generate_margins()
+ # or we generate constant margins equal to 1
+ if with_margins:
+ print(' - With margins')
+ mu = margins['mu']
+ nu = margins['nu']
+ else:
+ print(' - Without margins')
+ if T > 1:
+ mu = np.ones((T, N), dtype=dtype)
+ if directed:
+ nu = np.ones((T, D), dtype=dtype)
+ else:
+ mu = np.ones((N), dtype=dtype)
+ if directed:
+ nu = np.ones((D), dtype=dtype)
+
+ if with_absent_nodes and T > 1:
+ absent_row_nodes = absent_nodes['absent_row_nodes']
+ res_row = {t: 0 for t in range(T)}
+ for t, i in absent_row_nodes:
+ res_row[t] += 1
+ print(f' - absent row nodes {res_row}')
+ if directed:
+ absent_col_nodes = absent_nodes['absent_col_nodes']
+ res_col = {t: 0 for t in range(T)}
+ for t, i in absent_col_nodes:
+ res_col[t] += 1
+ print(f' - absent col nodes {res_col}')
+
+ # converts the static parameters to
+ # dynamic ones for genericity
+ if gamma.ndim == 2 and T > 1:
+ gamma = np.stack([gamma for _ in range(T)])
+
+ if mu.ndim == 1 and T > 1:
+ mu = np.stack([mu for _ in range(T)])
+
+ if directed:
+ if nu.ndim == 1 and T > 1:
+ nu = np.stack([nu for _ in range(T)])
+
+ if block_sparsity_matrix.ndim == 2 and T > 1:
+ block_sparsity_matrix = np.stack([block_sparsity_matrix for _ in range(T)])
+
+ # generates the latent processes
+ Z0 = np.random.choice(Kz, size=(N), p=alpha)
+
+ if model_type == 'LBM':
+ W0 = np.random.choice(Kw, size=(D), p=beta)
+
+ if T > 1:
+ Z = np.zeros((T, N), dtype=dtype)
+ Z[0] = Z0.copy()
+ for t in range(1, T):
+ for i in range(N):
+ k = Z[t - 1, i]
+ Z[t, i] = np.random.choice(Kz, p=pi[k, :])
+
+ if model_type == 'LBM':
+ W = np.zeros((T, D), dtype=dtype)
+ W[0] = W0.copy()
+ for t in range(1, T):
+ for j in range(D):
+ k = W[t - 1, j]
+ W[t, j] = np.random.choice(Kw, p=rho[k, :])
+ else:
+ Z = Z0
+ if model_type == 'LBM':
+ W = W0
+
+ # Generate the data matrix with the latent processes
+ if T > 1: # pylint: disable=R1702
+ X = np.zeros((T, N, D), dtype=dtype)
+ if not directed:
+ # we have model_type == SBM
+ for t in range(T):
+ for i in range(N):
+ for j in range(i + 1):
+ k, l = Z[t, i], Z[t, j]
+ val = sample_edge(
+ mu[t, i], mu[t, j],
+ gamma[t, k, l],
+ block_sparsity_matrix[t, k, l]
+ )
+ X[t, i, j] = val
+ X[t, j, i] = val
+ else:
+ for t in range(T):
+ for i in range(N):
+ for j in range(D):
+ if model_type == 'LBM':
+ k, l = Z[t, i], W[t, j]
+ else:
+ k, l = Z[t, i], Z[t, j]
+ val = sample_edge(
+ mu[t, i], nu[t, j],
+ gamma[t, k, l],
+ block_sparsity_matrix[t, k, l]
+ )
+ X[t, i, j] = val
+ else:
+ X = np.zeros((N, D), dtype=dtype)
+ if not directed:
+ for i in range(N):
+ for j in range(i + 1):
+ k, l = Z[i], Z[j]
+ val = sample_edge(
+ mu[i], mu[j],
+ gamma[k, l],
+ block_sparsity_matrix[k, l]
+ )
+ X[i, j] = val
+ X[j, i] = val
+ else:
+ for i in range(N):
+ for j in range(D):
+ if model_type == 'LBM':
+ k, l = Z[i], W[j]
+ else:
+ k, l = Z[i], Z[j]
+ val = sample_edge(
+ mu[i], nu[j],
+ gamma[k, l],
+ block_sparsity_matrix[k, l]
+ )
+ X[i, j] = val
+
+ if not self_loops:
+ if T > 1:
+ for t in range(T):
+ np.fill_diagonal(X[t], 0)
+ else:
+ np.fill_diagonal(X, 0)
+
+ # adds noise to the resulting matrix
+ noise = np.random.normal(loc=0., scale=noise_level, size=X.shape).astype(dtype)
+
+ if not self_loops:
+ if T == 1:
+ np.fill_diagonal(noise, 0)
+ else:
+ for t in range(T):
+ np.fill_diagonal(noise[t], 0)
+
+ np.add(X, noise, out=X)
+ np.clip(X, a_min=0, a_max=None, out=X)
+
+ if with_absent_nodes and T > 1:
+ for t, i in absent_row_nodes:
+ X[t, i, :] = 0
+ if not (directed and model_type == 'SBM'):
+ Z[t, i] = -1
+ if directed:
+ for t, j in absent_col_nodes:
+ X[t, :, j] = 0
+ if model_type == 'SBM':
+ # see below
+ pass
+ elif model_type == 'LBM':
+ W[t, j] = -1
+
+ # in the directed case with SBM and different row and col absent nodes a node is
+ # considered absent in Z if it is absent in the row AND the col
+ if model_type == 'SBM':
+ for t, i in absent_row_nodes:
+ if (t, i) in absent_col_nodes:
+ Z[t, i] = -1
+
+ if T == 1:
+ sparsity = (X == 0).sum() / (N * D)
+ print(f' - Sparsity {100 * sparsity: .2f}%')
+ else:
+ print(' - Sparsity', end=' ')
+ for t in range(T):
+ sparsity = (X[t] == 0).sum() / (N * D)
+ print(f't = {t} {100 * sparsity: .2f}%', end='|')
+
+ if not with_absent_nodes:
+ dynamic_offset = int(T > 1)
+ if (X.sum(dynamic_offset) == 0).any() or (X.sum(dynamic_offset + 1) == 0).any():
+ print(
+ 'Warning : with_absent_nodes is False but there are '
+ 'nodes with a zero in- or out-degree. They will be '
+ 'classified to cluster -1, which is different from'
+ 'their true cluster. The model parameters are probably '
+ 'too low'
+ )
+ return X, Z, W
+
+
+def AR1_process_margins(x0, a, sigma2, T):
+ """
+ given x0, array of margins at time t=0, generates
+ margins with T samples for each margin
+ x[t+1] = N(a * x[t] + c, sigma2)
+ where c is adapted so that the margins are
+ increasing over time
+ """
+ max_trials = 5000
+ n_trials = 0
+
+ def sample(x0, a, sigma2, T):
+ c = .1 + x0 * (1. - a)
+ # c = 0.
+ res = np.zeros(shape=(T, x0.shape[0]), dtype='float32')
+ res[0] = x0
+ for t in range(1, T):
+ mu = a * res[t - 1] + c
+ cov = sigma2 * np.eye(x0.shape[0])
+ res[t] = np.random.multivariate_normal(mu, cov, 1)[0]
+ return res
+
+ res = np.full((T, x0.shape[0]), -1.)
+ while (res <= 0.).any():
+ res = sample(x0, a, sigma2, T)
+ n_trials += 1
+ if n_trials == max_trials:
+ raise Exception('Could not sample all non negative values from AR1(x0, a, sigma2)')
+ return res
+
+
+def generate_margins(
+ T, N, D, constant_margins, start, stop, step,
+ directed, order_power_law,
+ ar_margins=None, a_ar=None, sigma2_ar=None
+):
+ """
+ Generates margins : arrays of shape (T, N) or (N)
+ values x sampled in np.arange(start, stop, step)
+ with probability propto x ^ order_power_law
+ """
+ assert order_power_law < 0
+ if D is None:
+ D = N
+ nax = np.newaxis
+
+ if constant_margins:
+ print('Constant margins')
+ else:
+ print('Variable margins')
+
+ values_margins = np.arange(start, stop, step)
+ proba_distrib = values_margins ** order_power_law
+ np.divide(proba_distrib, proba_distrib.sum(), out=proba_distrib)
+ if T == 1 or constant_margins or ar_margins:
+ mu = np.random.choice(
+ a=values_margins,
+ size=(N),
+ p=proba_distrib
+ )
+ if directed:
+ nu = np.random.choice(
+ a=values_margins,
+ size=(D),
+ p=proba_distrib
+ )
+ else:
+ mu = np.random.choice(
+ a=values_margins,
+ size=(T, N),
+ p=proba_distrib
+ )
+ if directed:
+ nu = np.random.choice(
+ a=values_margins,
+ size=(T, D),
+ p=proba_distrib
+ )
+ if T > 1 and constant_margins:
+ mu = np.concatenate([mu[nax, :] for t in range(T)], axis=0)
+ if directed:
+ nu = np.concatenate([nu[nax, :] for t in range(T)], axis=0)
+
+ if T > 1 and ar_margins:
+ mu = AR1_process_margins(mu, a_ar, sigma2_ar, T)
+ if directed:
+ nu = AR1_process_margins(nu, a_ar, sigma2_ar, T)
+
+ if directed:
+ return mu, nu
+ return mu, None
+
+
+def generate_diag_transition_matrix(Kzw, val_diag):
+ """
+ val_diag : the proba of transition to the same state
+ """
+ val_off_diag = (1. - val_diag) / (Kzw - 1)
+ return (val_diag - val_off_diag) * np.eye((Kzw)) + val_off_diag
+
+
+def generate_initial_proportions(Kzw, alpha_dirichlet):
+ """
+ generates a np.array of shape (Kzw, ) from a dirichlet
+ distribution with constant parameter alpha_dirichlet
+ """
+ return np.random.dirichlet(np.full(Kzw, alpha_dirichlet), size=1)[0]
+
+
+def sample_absent_nodes(T, ND, min_proba_t=None, max_proba_t=None, proba_absent=None):
+ """
+ Samples the time and node index of absent nodes.
+ If min_proba_t and max_proba_t are given, samples for each
+ time step t a probability p_t from Unif(min_proba_t, max_proba_t),
+ then samples the number n_abs_t of absent nodes at time t
+ from Binom(ND, p_t), then samples n_abs_t nodes without
+ replacement.
+ If min_proba_t=None, max_proba_t=None and proba_absent is not
+ None, then all p_t = proba_absent
+ """
+ if min_proba_t is not None and max_proba_t is not None:
+ assert proba_absent is None
+ proba_absent = np.random.uniform(low=min_proba_t, high=max_proba_t, size=T)
+
+ absent = []
+ n_absent_t = np.random.binomial(ND, proba_absent, size=(T))
+ for t, n_abs in enumerate(n_absent_t):
+ i_absent_t = np.sort(np.random.choice(ND, size=n_abs, replace=False))
+ absent += list(zip([t] * n_abs, list(i_absent_t)))
+
+ return absent
diff --git a/dcblockmodels/metrics.py b/dcblockmodels/metrics.py
new file mode 100644
index 0000000..37b7344
--- /dev/null
+++ b/dcblockmodels/metrics.py
@@ -0,0 +1,473 @@
+"""
+Methods to measure the quality of a partition of the data
+or to measure the class separability given the ground truth
+"""
+
+import warnings
+
+import numpy as np
+import prince
+import pandas as pd
+
+from sklearn.metrics import (adjusted_rand_score,
+ normalized_mutual_info_score,
+ confusion_matrix)
+from scipy.optimize import linear_sum_assignment
+from sparsebm import CARI
+
+
+def AFD(X, Z, n_factors=5):
+ """
+ Discriminative Factorial Analysis
+ https://marie-chavent.perso.math.cnrs.fr/wp-content/uploads/2013/10/AFD.pdf
+ """
+ assert X.ndim == 2
+
+ n, d = X.shape
+ K = np.unique(Z).shape[0]
+
+ X_centered = X - X.mean(axis=0)
+ V = 1 / n * X_centered.T @ X_centered
+
+ B = np.zeros((d, d))
+ for k in range(K):
+ ind_k = np.where(Z == k)[0]
+ if ind_k.shape[0] == 0:
+ continue
+ X_k = X_centered[ind_k, :]
+ group_gravity = X_k.mean(axis=0)
+ B += ind_k.shape[0] * np.outer(group_gravity, group_gravity)
+
+ B = B / n
+
+ # just to check that V = W + B
+ # W = np.zeros((d, d))
+ # for k in range(K):
+ # X_k = X_centered[np.where(Z == k)[0], :]
+ # X_g = X_k - X_k.mean(axis=0)
+ # for i in range(X_g.shape[0]):
+ # W += np.outer(X_g[i], X_g[i])
+ #
+ # W = W / n
+
+ _eigen_values, eigen_vectors = np.linalg.eigh(np.linalg.inv(V) @ B)
+ eigen_vectors = eigen_vectors[:, ::-1]
+ eigen_vectors = eigen_vectors[:, :n_factors]
+
+ discriminant_power = np.zeros((n_factors))
+ for k in range(n_factors):
+ u = eigen_vectors[:, k]
+ discriminant_power[k] = (u.T.dot(B).dot(u) / u.T.dot(V).dot(u))
+
+ return eigen_vectors, discriminant_power
+
+
+def AFD_CA_linear_separation(X, Z, W, n_components, absent_row_nodes=None, absent_col_nodes=None):
+ """
+ Level of linear separability of the classes after projection onto R^N using correspondence
+ analysis
+ """
+ warnings.filterwarnings("ignore", category=FutureWarning)
+
+ assert X.ndim == 2
+ N, D = X.shape
+ present_row = np.setdiff1d(np.arange(N), absent_row_nodes)
+ present_col = np.setdiff1d(np.arange(D), absent_col_nodes)
+ X_ = X[np.ix_(present_row, present_col)]
+ X_ = X_ + 1e-10
+ # N_, D_ = X_.shape
+
+ if N == D and not np.array_equal(present_row, present_col):
+ print('Special case: SBM with different absent row and col nodes.')
+
+ # for directed SBM with different absent
+ # row and col nodes, X_ is not square
+ # and W_ is taken from Z with absent col nodes
+ Z_ = Z[present_row]
+ if W is not None:
+ W_ = W[present_col]
+ else:
+ W_ = Z[present_col]
+
+ ca = prince.CA(
+ n_components=n_components,
+ n_iter=30,
+ copy=True,
+ check_input=True,
+ engine='auto',
+ random_state=42
+ )
+ df = pd.DataFrame(X_)
+ ca = ca.fit(df)
+ row_factor_score = ca.row_coordinates(df).values
+ col_factor_score = ca.column_coordinates(df).values
+
+ lambdas_row = AFD(row_factor_score, Z_, n_factors=n_components)[1]
+ lambdas_col = AFD(col_factor_score, W_, n_factors=n_components)[1]
+ return lambdas_row, lambdas_col
+
+
+def sort_partitions(criteria, partitions, n_first):
+ """
+ Sorts multiple partitions based on their log likelihoods
+ and returns the n_first
+ criteria: list of lists, criteria[k]: list of
+ log likelihoods at each iteration for initialization k
+ of the model
+ partitions: list of lists: len(partitions) = 1 for dSBM/SBM
+ and len(partitions) = 2 for dLBM/LBM.
+ partitions[0][k] is the row partition obtained at the kth
+ initialization of the model.
+ """
+ assert n_first <= len(criteria)
+ assert len(partitions[0]) == len(criteria)
+
+ sorted_partitions = sorted(
+ zip(criteria, *(part for part in partitions)),
+ key=lambda x: x[0][-1],
+ reverse=True
+ )
+ return [x[1:] for x in sorted_partitions[:n_first]]
+
+
+def always_absent_nodes(ZW):
+ """
+ ZW : where absent nodes have already
+ been put in cluster -1
+ """
+ return np.where((ZW == -1).all(axis=0))[0]
+
+
+def cmat_clustering(cmat):
+ """
+ input : np.array shape (n,n) : a confusion matrix
+ returns : np.array shape (n,n) : a confusion matrix for
+ clustering, with the permutation leading to the
+ maximal diagonal
+ """
+ indexes = linear_sum_assignment(-cmat)
+ return cmat[:, indexes[1]]
+
+
+def accuracy(cmat):
+ """
+ Accuracy of a confusion matrix (diagonal sum over whole matrix sum)
+ """
+ return np.trace(cmat) / cmat.sum()
+
+
+def get_metrics(Z, W, Z_true, W_true, absent_nodes=None):
+ """
+ If absent_nodes is not None : returns the metrics only
+ considering present nodes, i.e. nodes in
+ cluster K do not bias the performance metrics.
+ We nevertheless consider appearing nodes
+ and the metrics are given for {always_present_nodes + appearing_nodes}
+ """
+ def clustering_accuracy(p1, p2):
+ if len(p1) == 0:
+ return np.nan
+ return accuracy(cmat_clustering(confusion_matrix(p1, p2)))
+
+ def ari_(p1, p2):
+ if len(p1) == 0:
+ return np.nan
+ return adjusted_rand_score(p1, p2)
+
+ def nmi_(p1, p2):
+ if len(p1) == 0:
+ return np.nan
+ return normalized_mutual_info_score(p1, p2, average_method='arithmetic')
+
+ if Z.ndim == 1:
+ T = 1
+ elif Z.ndim == 2:
+ T = Z.shape[0]
+ else:
+ raise ValueError
+
+ res_dic = {}
+
+ if absent_nodes is not None:
+ absent_row_nodes = absent_nodes.absent_row_nodes
+ absent_col_nodes = absent_nodes.absent_col_nodes
+ appearing_row_nodes = absent_nodes.appearing_row_nodes
+ appearing_col_nodes = absent_nodes.appearing_col_nodes
+ else:
+ absent_row_nodes, absent_col_nodes = None, None
+ appearing_row_nodes, appearing_col_nodes = None, None
+
+ if Z is not None and W is not None and Z_true is not None and W_true is not None:
+ if T == 1:
+ if Z is not None and W is not None:
+ cari = CARI(Z_true, W_true, Z, W)
+ res_dic['cari'] = cari
+ else:
+ Z_without_absent = partition_without_absent(Z, absent_row_nodes)
+ Z_true_without_absent = partition_without_absent(Z_true, absent_row_nodes)
+ W_without_absent = partition_without_absent(W, absent_col_nodes)
+ W_true_without_absent = partition_without_absent(W_true, absent_col_nodes)
+ caris = np.array([
+ CARI(
+ Z_true_without_absent[t],
+ W_true_without_absent[t],
+ Z_without_absent[t],
+ W_without_absent[t]
+ )
+ for t in range(T)])
+ cari_avg = np.nanmean(caris)
+
+ cari_f = CARI(
+ np.concatenate([Z_true_without_absent[t] for t in range(T)]),
+ np.concatenate([W_true_without_absent[t] for t in range(T)]),
+ np.concatenate([Z_without_absent[t] for t in range(T)]),
+ np.concatenate([W_without_absent[t] for t in range(T)])
+ )
+ res_dic['cari_f_without_absent'] = cari_f
+ res_dic['cari_avg_without_absent'] = cari_avg
+ res_dic['caris_without_absent'] = caris
+
+ list_dims = []
+ if Z is not None and Z_true is not None:
+ list_dims.append(('Z', Z, Z_true, absent_row_nodes, appearing_row_nodes))
+ if W is not None and W_true is not None:
+ list_dims.append(('W', W, W_true, absent_col_nodes, appearing_col_nodes))
+
+ for name, ZW, ZW_true, absent_nodes_, appearing_nodes in list_dims:
+ if ZW is not None:
+ assert len(ZW) == len(ZW_true)
+
+ if T == 1:
+ ari = ari_(ZW_true, ZW)
+ nmi = nmi_(ZW_true, ZW)
+ acc = clustering_accuracy(ZW_true, ZW)
+
+ res_dic['ari_' + name] = ari
+ res_dic['nmi_' + name] = nmi
+ res_dic['acc_' + name] = acc
+ else:
+ ZW_without_absent = partition_without_absent(ZW, absent_nodes_)
+ ZW_true_without_absent = partition_without_absent(ZW_true, absent_nodes_)
+
+ ZW_appearing = partition_apearing_nodes(ZW, appearing_nodes)
+ ZW_true_appearing = partition_apearing_nodes(ZW_true, appearing_nodes)
+
+ flat_ZW_without_absent = np.concatenate(
+ [ZW_without_absent[t] for t in range(T)]
+ )
+ flat_ZW_true_without_absent = np.concatenate(
+ [ZW_true_without_absent[t] for t in range(T)]
+ )
+ flat_ZW_appearing = np.concatenate(
+ [ZW_appearing[t - 1] for t in range(1, T)]
+ )
+ flat_ZW_true_appearing = np.concatenate(
+ [ZW_true_appearing[t - 1] for t in range(1, T)]
+ )
+ aris_without_absent = np.array([
+ ari_(ZW_true_without_absent[t], ZW_without_absent[t])
+ for t in range(T)
+ ])
+ nmis_without_absent = np.array([
+ nmi_(ZW_true_without_absent[t], ZW_without_absent[t])
+ for t in range(T)
+ ])
+ accs_without_absent = np.array([
+ clustering_accuracy(ZW_true_without_absent[t], ZW_without_absent[t])
+ for t in range(T)
+ ])
+ ari_avg_without_absent = np.nanmean(aris_without_absent)
+ nmi_avg_without_absent = np.nanmean(nmis_without_absent)
+ acc_avg_without_absent = np.nanmean(accs_without_absent)
+
+ ari_f_without_absent = ari_(flat_ZW_true_without_absent, flat_ZW_without_absent)
+ nmi_f_without_absent = nmi_(flat_ZW_true_without_absent, flat_ZW_without_absent)
+ acc_f_without_absent = clustering_accuracy(
+ flat_ZW_true_without_absent,
+ flat_ZW_without_absent
+ )
+ res_dic['aris_without_absent_' + name] = aris_without_absent
+ res_dic['nmis_without_absent_' + name] = nmis_without_absent
+ res_dic['accs_without_absent_' + name] = accs_without_absent
+ res_dic['ari_avg_without_absent_' + name] = ari_avg_without_absent
+ res_dic['nmi_avg_without_absent_' + name] = nmi_avg_without_absent
+ res_dic['acc_avg_without_absent_' + name] = acc_avg_without_absent
+ res_dic['ari_f_without_absent_' + name] = ari_f_without_absent
+ res_dic['nmi_f_without_absent_' + name] = nmi_f_without_absent
+ res_dic['acc_f_without_absent_' + name] = acc_f_without_absent
+
+ if absent_nodes_ is not None:
+ aris_appearing = np.array([
+ ari_(ZW_true_appearing[t - 1], ZW_appearing[t - 1])
+ for t in range(1, T)
+ ])
+ nmis_appearing = np.array([
+ nmi_(ZW_true_appearing[t - 1], ZW_appearing[t - 1])
+ for t in range(1, T)
+ ])
+ accs_appearing = np.array([
+ clustering_accuracy(ZW_true_appearing[t - 1], ZW_appearing[t - 1])
+ for t in range(1, T)
+ ])
+ ari_avg_appearing = np.nanmean(aris_appearing)
+ nmi_avg_appearing = np.nanmean(nmis_appearing)
+ acc_avg_appearing = np.nanmean(accs_appearing)
+
+ ari_f_appearing = ari_(flat_ZW_true_appearing, flat_ZW_appearing)
+ nmi_f_appearing = nmi_(flat_ZW_true_appearing, flat_ZW_appearing)
+ acc_f_appearing = clustering_accuracy(
+ flat_ZW_true_appearing, flat_ZW_appearing)
+
+ res_dic['aris_appearing_' + name] = aris_appearing
+ res_dic['nmis_appearing_' + name] = nmis_appearing
+ res_dic['accs_appearing_' + name] = accs_appearing
+ res_dic['ari_avg_appearing_' + name] = ari_avg_appearing
+ res_dic['nmi_avg_appearing_' + name] = nmi_avg_appearing
+ res_dic['acc_avg_appearing_' + name] = acc_avg_appearing
+ res_dic['ari_f_appearing_' + name] = ari_f_appearing
+ res_dic['nmi_f_appearing_' + name] = nmi_f_appearing
+ res_dic['acc_f_appearing_' + name] = acc_f_appearing
+
+ return res_dic
+
+
+def print_metrics(Z, W, Z_true, W_true, absent_nodes=None, print_each_timestep=False):
+ """Print all metrics in terminal"""
+ if Z.ndim == 1:
+ T = 1
+ elif Z.ndim == 2:
+ T, _ = Z.shape
+ else:
+ raise ValueError
+
+ metrics_dic = get_metrics(Z, W, Z_true, W_true, absent_nodes)
+
+ if T > 1:
+ if 'cari_f_without_absent' in metrics_dic:
+ print(f'global CARI : {(100 * metrics_dic["cari_f_without_absent"]):.2f}')
+ print(f'local AVG CARI : {(100 * metrics_dic["cari_avg_without_absent"]):.2f}')
+ if print_each_timestep:
+ print('\n\nAt each timestep: ')
+ for t in range(T):
+ print(f't = {t} CARI : {100 * metrics_dic["caris_without_absent"][t]:.2f}')
+ print('\n')
+ else:
+ if 'cari' in metrics_dic:
+ print(f'CARI : {100 * metrics_dic["cari"]:.2f}\n')
+
+ if absent_nodes is not None:
+ absent_row_nodes = absent_nodes.absent_row_nodes
+ absent_col_nodes = absent_nodes.absent_col_nodes
+ n_absent_row = absent_nodes.n_absent_row_tot
+ n_absent_col = absent_nodes.n_absent_col_tot
+ else:
+ absent_row_nodes, absent_col_nodes = None, None
+ n_absent_row, n_absent_col = None, None
+
+ list_dims = []
+ if Z is not None and Z_true is not None:
+ list_dims.append(('Z', absent_row_nodes, n_absent_row))
+ if W is not None and W_true is not None:
+ list_dims.append(('W', absent_col_nodes, n_absent_col))
+
+ for name, absent_nodes_, n_absent in list_dims:
+ print(' ' + name, end='\n\n')
+
+ if absent_nodes_ is not None:
+ print_for_absent_nodes = n_absent > 0
+ else:
+ print_for_absent_nodes = False
+
+ if T == 1:
+ print(
+ f'ARI : {100 * metrics_dic["ari_" + name]:.2f}, '
+ f'NMI : {100 * metrics_dic["nmi_" + name]:.2f}, '
+ f'ACC : {100 * metrics_dic["acc_" + name]:.2f}'
+ )
+ else:
+ print('Without absent nodes:')
+ print(f'local AVG '
+ f'ARI : {100 * metrics_dic["ari_avg_without_absent_" + name]:.2f}, '
+ f'NMI : {100 * metrics_dic["nmi_avg_without_absent_" + name]:.2f}, '
+ f'ACC : {100 * metrics_dic["acc_avg_without_absent_" + name]:.2f}'
+ )
+ print('global '
+ f'ARI : {100 * metrics_dic["ari_f_without_absent_" + name]:.2f}, '
+ f'NMI : {100 * metrics_dic["nmi_f_without_absent_" + name]:.2f}, '
+ f'ACC : {100 * metrics_dic["acc_f_without_absent_" + name]:.2f}'
+ )
+ if print_for_absent_nodes:
+ print()
+ print('Appearing nodes: ')
+ print('local AVG '
+ f'ARI : {100 * metrics_dic["ari_avg_appearing_" + name]:.2f}, '
+ f'NMI : {100 * metrics_dic["nmi_avg_appearing_" + name]:.2f}, '
+ f'ACC : {100 * metrics_dic["acc_avg_appearing_" + name]:.2f}'
+ )
+ print('global '
+ f'ARI : {100 * metrics_dic["ari_f_appearing_" + name]:.2f}, '
+ f'NMI : {100 * metrics_dic["nmi_f_appearing_" + name]:.2f}, '
+ f'ACC : {100 * metrics_dic["acc_f_appearing_" + name]:.2f}'
+ )
+
+ if print_each_timestep:
+ print('\n\nAt each timestep: ')
+ print('Without absent nodes:')
+ for t in range(T):
+ print(f't = {t} '
+ f'ARI : {100 * metrics_dic["aris_without_absent_" + name][t]:.2f}, '
+ f'NMI : {100 * metrics_dic["nmis_without_absent_" + name][t]:.2f}, '
+ f'ACC : {100 * metrics_dic["accs_without_absent_" + name][t]:.2f}'
+ )
+ print('\n')
+
+ if print_for_absent_nodes:
+ print('Appearing nodes: ')
+ for t in range(1, T):
+ print(f't = {t} '
+ f'ARI : {100 * metrics_dic["aris_appearing_" + name][t - 1]:.2f}, '
+ f'NMI : {100 * metrics_dic["nmis_appearing_" + name][t - 1]:.2f}, '
+ f'ACC : {100 * metrics_dic["accs_appearing_" + name][t - 1]:.2f}'
+ )
+ print('\n')
+
+
+def partition_apearing_nodes(ZW, appearing_nodes):
+ """
+ returns a list of arrays containing the elements of
+ the rows of ZW that correspond to absent nodes
+ """
+ if appearing_nodes is not None:
+ return [ZW[t][appearing_nodes[t]] for t in range(1, len(ZW))]
+
+ return [ZW[t] for t in range(1, len(ZW))]
+
+
+def partition_without_absent(ZW, absent_nodes):
+ """
+ returns the partition Z with absent nodes removed
+ as a list of T ndarray of different sizes
+ Z : (T, N)
+ """
+ if absent_nodes is not None:
+ return [np.delete(ZW[t], absent_nodes[t]) for t in range(ZW.shape[0])]
+
+ return [ZW[t] for t in range(ZW.shape[0])]
+
+
+def get_prop_clusters(ZW):
+ """
+ returns a np.array of size T x Kzw
+ containing the proportion of points in
+ each cluster at each time step
+ """
+ T = len(ZW)
+ flat_ZW = np.concatenate([ZW[t] for t in range(T)])
+ Kzw = np.unique(flat_ZW).shape[0]
+
+ res = np.zeros((T, Kzw))
+ for t in range(T):
+ for k in range(Kzw):
+ res[t, k] = (ZW[t] == k).sum()
+ res = res / res.sum(axis=1, keepdims=True)
+ return res
diff --git a/dcblockmodels/models/__init__.py b/dcblockmodels/models/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/dcblockmodels/models/blockmodel.py b/dcblockmodels/models/blockmodel.py
new file mode 100644
index 0000000..18d047e
--- /dev/null
+++ b/dcblockmodels/models/blockmodel.py
@@ -0,0 +1,367 @@
+"""Base class for the Block Models"""
+
+import time
+import os
+import sys
+import warnings
+import pickle
+
+import numpy as np
+import scipy as sp
+from numba import NumbaPendingDeprecationWarning
+from sklearn.base import BaseEstimator
+from sklearn.utils import check_random_state
+
+np.seterr(all='raise')
+
+# FIXME Probably a way to remove warnings? (problem in the whole program)
+sys.stderr = open(os.devnull, "w") # pylint: disable=R1732,W1514
+
+warnings.filterwarnings("ignore", category=NumbaPendingDeprecationWarning)
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+
+
+class BlockModel(BaseEstimator):
+ """Base class for the dynamic and semi-supervised LBM"""
+
+ model_parameters = [] # filled in subclasses
+
+ # The default blockmodel parameters
+ # overwritten by the params given in dic blockmodel_params
+ # in the construction of BlockModel
+ n_iter_min = 5
+ n_init_clustering_consensus = 100
+ loc_random_margins = 1e-8 # mean and std used for the initialization
+ scale_random_margins = 1e-3 # of the margins if self.type_init_margins == 'random'
+
+ # checking the inputs
+ MODEL_TYPES = [
+ 'with_margins', # static & dynamic
+ 'without_margins' # static & dynamic
+ ]
+ EM_TYPES = ['CEM', 'VEM']
+ INITS_TYPES = [
+ 'random',
+ 'given',
+ 'kmeans',
+ 'skmeans'
+ ]
+ TYPES_INIT_MARGINS = ['ones', 'X.', 'random', 'given']
+ DTYPES = ['float32', 'float64']
+ VERBOSE_LEVELS = [0, 1, 2]
+
+ def __init__(
+ self,
+ Kz,
+ Kw,
+ init_type,
+ em_type,
+ n_init,
+ n_init_clustering,
+ node_perturbation_rate,
+ model_type,
+ type_init_margins,
+ min_float,
+ min_gamma,
+ min_proba_Z,
+ min_proba_W,
+ min_margin,
+ min_proba_mixture_proportions,
+ threshold_absent_nodes,
+ dtype,
+ random_state,
+ max_iter,
+ tol_iter,
+ n_jobs,
+ verbose,
+ blockmodel_params,
+ model_id,
+ debug_list,
+ debug_output
+ ):
+ super().__init__()
+
+ for arg in ['Kz', 'Kw', 'init_type', 'em_type', 'n_init', 'model_type', 'min_gamma']:
+ if arg is None:
+ raise (f'Argument {arg} for class {type(self).__name__} must be initialized '
+ 'explicitly. See documentation for possible values.')
+
+ assert Kz > 1
+ assert Kw > 1
+
+ assert init_type in self.INITS_TYPES
+ assert em_type in self.EM_TYPES
+ assert model_type in self.MODEL_TYPES
+
+ assert dtype in self.DTYPES
+ assert verbose in self.VERBOSE_LEVELS
+ assert type_init_margins in self.TYPES_INIT_MARGINS
+
+ assert min_float > 0.
+ assert min_gamma >= 0.
+ assert 0. < min_proba_mixture_proportions < 1.
+
+ self.Kz = Kz
+ self.Kw = Kw
+
+ self.init_type = init_type
+ self.em_type = em_type
+ self.n_init = n_init
+ self.n_init_clustering = n_init_clustering
+ self.node_perturbation_rate = node_perturbation_rate
+ self.type_init_margins = type_init_margins
+
+ self.min_float = min_float
+ self.min_gamma = min_gamma
+ self.min_proba_Z = min_proba_Z
+ self.min_proba_W = min_proba_W
+ self.min_margin = min_margin
+ self.min_proba_mixture_proportions = min_proba_mixture_proportions
+ self.threshold_absent_nodes = threshold_absent_nodes
+
+ self.dtype = dtype
+ self.random_state = check_random_state(random_state)
+
+ self.max_iter = max_iter
+ self.tol_iter = tol_iter
+
+ self.n_jobs = n_jobs
+ self.verbose = verbose
+
+ self.blockmodel_params = blockmodel_params
+ self._set_blockmodel_params(blockmodel_params)
+
+ self.model_id = int(time.time()) if model_id is None else model_id
+ self.debug_list = debug_list if debug_list is not None else []
+ self.debug_output = debug_output
+
+ self.cluster_perturbation_rate = None
+
+ self.N, self.D, self.T = None, None, None
+ self.X = None
+ self.W, self.Z = None, None
+ self.alpha, self.beta = None, None
+ self.gamma, self.mu, self.nu = None, None, None
+ self.log_alpha, self.log_beta, self.log_pi, self.log_rho = None, None, None, None
+ self.Xi_, self.X_j = None, None
+
+ self.qz, self.qw = None, None
+ self.current_init_W, self.current_init_Z = None, None
+ self.global_init_W, self.global_init_Z = None, None
+
+ self.best_parameters, self.best_criterion = None, None
+
+ self.model_type = model_type
+
+ self.debug_counts = None
+ self.debug_path = None
+
+ self.fitted = False
+
+ self.all_row_partitions = None
+ self.all_col_partitions = None
+ self.all_iter_criterions = None
+ self.all_intermediate_iter_criterions = None
+ self.all_icls = None
+ self.all_regularizations = None
+ self.all_intermediate_regularizations = None
+
+ def _set_blockmodel_params(self, blockmodel_params):
+ """
+ overwrites the default params with given params in __init__
+ """
+ params_names = [
+ "min_proba_mixture_proportions",
+ "min_float_margins",
+ "n_iter_min",
+ "loc_random_margins",
+ "scale_random_margins",
+ ]
+ if blockmodel_params is not None:
+ for param_name, param_value in blockmodel_params.items():
+ if param_name in params_names:
+ setattr(self, param_name, param_value)
+
+ # ################### Verbose #################### #
+
+ def _print_verbose_msg_iter(self, n_iter):
+ if self.verbose == 1:
+ if (n_iter % 5 == 4) or (n_iter == self.max_iter - 1):
+ print(f' Iteration {n_iter + 1} on {self.max_iter}')
+ elif self.verbose == 2:
+ print(f' Iteration {n_iter + 1} on {self.max_iter}')
+
+ def _print_verbose_msg_init(self, n_init):
+ if self.verbose >= 1:
+ print()
+ print(f'*** Model {self.model_id}: '
+ f'initialization {n_init + 1} on {self.n_init} ***')
+
+ def _print_verbose_converged(self, n_iter_tot, ind_smoothing):
+ if self.verbose >= 1:
+ if ind_smoothing is None:
+ print(f' {n_iter_tot + 1} Iterations are enough')
+ else:
+ if ind_smoothing % 5 == 4:
+ print(f' smoothing step {ind_smoothing + 1} at iter {n_iter_tot + 1}')
+
+ def _print_verbose_smoothing(self, smoothing_schedule, n_iter_supp_smoothing):
+ if self.verbose >= 1:
+ n_steps = len(smoothing_schedule)
+ n_iter_supp = n_steps * n_iter_supp_smoothing
+ print(
+ f' Smoothing: {n_iter_supp} new iter max ({n_steps} steps, '
+ f'{n_iter_supp_smoothing} iter max per tau)'
+ )
+
+ # ################### Debug #################### #
+
+ def _init_debug(self):
+ """
+ creates appropriate directories where
+ the parameters we want to debug will be saved
+ """
+ self.debug_output.mkdir(exist_ok=True)
+
+ # FIXME Verify that the path is a Pathlib.path!
+
+ # FIXME And create directory dynamically!
+ def _unique_dir(base_directory, name_pattern):
+ c = 0
+ while True:
+ c += 1
+ directory = base_directory / name_pattern.format(c)
+ if not directory.is_dir():
+ return directory
+
+ # for each init, the number of times debug() was called
+ self.debug_counts = np.zeros((self.n_init), dtype='int')
+
+ # name_pattern = 'debug_model_{}'.format(self.model_id) + '_{:03d}'
+ # TODO verify behaviour
+ name_pattern = f'debug_model_{self.model_id}_{{:03d}}'
+ self.debug_path = _unique_dir(self.debug_output, name_pattern)
+
+ if len(self.debug_list) > 0:
+ os.mkdir(self.debug_path)
+ for debug_item in self.debug_list:
+ os.mkdir(self.debug_path / debug_item)
+
+ def _debug(self, init_nb):
+ """
+ e.g. debug_list = ['Z', 'gamma']
+ adds Z_0.npy in dir /Z and gamma_0.npy in /gamma
+ then adds Z_1.npy and gamma_1.npy
+ etc ...
+
+ call the method every time you want to log
+ the state of a given variable
+ """
+ for debug_item in self.debug_list:
+ item = getattr(self, debug_item, None)
+ item_id = f'{debug_item}_init_{init_nb}_{self.debug_counts[init_nb]}'
+ item_path = self.debug_path / debug_item / item_id
+ _save_item(item, item_path)
+
+ self.debug_counts[init_nb] += 1
+
+ def has_sparse_ZW(self):
+ """Check whether self.Z is either sparse or a list"""
+ return sp.sparse.issparse(self.Z) or isinstance(self.Z, list)
+
+ def get_debug(self, sublist=None):
+ """
+ returns a dictionnary containing the values of each
+ item in self.debug_list (e.g. debug_list = ['Z', 'gamma'])
+ at each iteration of the algorithm
+ if sublist is not None, only returns the elements that are
+ both in self.debug_list and sublist
+ """
+ if sublist is not None:
+ debug_list = [x for x in sublist if x in self.debug_list]
+ else:
+ debug_list = self.debug_list
+
+ res = {}
+ for debug_item in debug_list:
+ if debug_item in ['Z', 'W']:
+ ext = '.npz' if self.has_sparse_ZW() else '.npy'
+ else:
+ ext = '.npy'
+ res_item = [] # one list for a given item, e.g. gamma
+ for init_nb in range(self.n_init):
+ res_init = [] # one list for each initialization of an item
+ for debug_count in range(self.debug_counts[init_nb]):
+ item_id = (debug_item + f'_init_{init_nb}_{debug_count}' + ext)
+ item_path = self.debug_path / debug_item / item_id
+ item = _load_item(item_path)
+ res_init.append(item)
+ res_item.append(res_init)
+ res[debug_item] = res_item
+ return res
+
+ # ################### Model utils #################### #
+
+ def _set_best_parameters(self, criterion, cur_init, cur_iter):
+ if cur_init == 0 and cur_iter == 0:
+ self.best_parameters = [[] for _ in range(self.n_init)]
+
+ if cur_iter == 0:
+ self.best_criterion = criterion
+ self._set_new_best_params(cur_init)
+ else:
+ if criterion > self.best_criterion:
+ self.best_criterion = criterion
+ self._set_new_best_params(cur_init)
+
+ def _set_new_best_params(self, cur_init):
+ best_params = [self.best_criterion]
+ for param_name in self.model_parameters:
+ best_params.append(getattr(self, param_name))
+ self.best_parameters[cur_init] = best_params
+
+ def _write_best_parameters(self):
+ """
+ writes attribute best_parameters
+ [(criterion(init_i),
+ {'param_name': best_value_param_init_i, ...}), ...]
+ ordered by criterion
+ """
+ self.best_parameters = [
+ (self.best_parameters[i][0],
+ dict(zip(self.model_parameters, self.best_parameters[i][1:])))
+ for i in range(self.n_init)
+ ]
+ self.best_parameters = sorted(self.best_parameters, key=lambda x: x[0], reverse=True)
+
+ def save(self, path='.', modelname=None):
+ """
+ Saves the model at path path with the name modelname as a pickle file.
+ """
+ if modelname is None:
+ modelname = f'{type(self).__name__}_saved_at_{int(time.time())}'
+
+ full_path = path + '/' + modelname
+ os.makedirs(full_path, exist_ok=True)
+
+ model_path = full_path + '/' + 'model.pickle'
+ with open(model_path, 'wb') as f:
+ pickle.dump(self, f)
+ print('Model saved at : ' + model_path)
+
+
+def _save_item(item, path):
+ if isinstance(item, np.ndarray):
+ np.save(path.with_suffix('.npy'), item)
+ elif sp.sparse.issparse(item):
+ sp.sparse.save_npz(path.with_suffix('.npz'), item)
+ else:
+ raise TypeError
+
+
+def _load_item(path):
+ if path.suffix == '.npy':
+ return np.load(path, allow_pickle=True)
+ if path.suffix == '.npz':
+ return sp.sparse.load_npz(path)
+ raise TypeError
diff --git a/dcblockmodels/models/dlbm.py b/dcblockmodels/models/dlbm.py
new file mode 100644
index 0000000..006ad3c
--- /dev/null
+++ b/dcblockmodels/models/dlbm.py
@@ -0,0 +1,1217 @@
+"""Dynamic Latent Block Model"""
+
+import os
+import sys
+
+import warnings
+import numpy as np
+from numba import NumbaPendingDeprecationWarning
+
+from dcblockmodels import metrics
+from .blockmodel import BlockModel
+from .utils import (
+ init,
+ e_step,
+ m_step,
+ general,
+ consensus,
+ absent_nodes
+)
+
+sys.stderr = open(os.devnull, "w") # pylint: disable=R1732,W1514
+
+warnings.filterwarnings("ignore", category=NumbaPendingDeprecationWarning)
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+
+
+class dLBM(BlockModel):
+ """Dynamic Latent Block Model
+
+ Block Model General Parameters
+ ----------
+ Kz : int
+ Number of row clusters
+ Kw : int
+ Number of column clusters
+ init_type : {'random', 'kmeans', 'skmeans', 'given'}, optional,
+ by default 'skmeans'. The methods used for the initialization of
+ the algorithm. It is fit once for each side (rows and columns)
+ and gives two global initialization partitions. These partitions
+ are then used to initialization-specific partitions using the
+ parameters `node_perturbation_rate` and `cluster_perturbation_rate`:
+ - '`random'` randomly initialized clusters,
+ - `'kmeans'` clusters initialized using `sklearn.cluster.KMeans`,
+ - `'skmeans'` clusters initialized using `spherecluster.SphericalKMeans`,
+ - `'given'` given in the `fit()` method
+ em_type : {'VEM', 'CEM'}
+ The EM algorithm used
+ n_init : int
+ Number of initializations of the model. Each initialization
+ will start from different initial row and col partitions
+ and will converge to a partition and parameter set with
+ a corresponding complete-data log likelihood. One can then
+ select one of these partitions using the `best_partition()`
+ method.
+ n_init_clustering : int, optional
+ the number of initializations of the clustering algorithm
+ chosen in `'init_type'`, by default 100
+ node_perturbation_rate : float, optional
+ the fraction of nodes (row or cols) that are reassgined to
+ a random cluster at each new intitialization of the model, by default .1
+ model_type : {'with_margins', 'without_margins'}
+ The dynamic model used:
+ - `'with_margins'` : a model with dynamic margins
+ and static connectivity matrix gamma
+ - `'without_margins'` : a model with only a dynamic
+ connectivity matrix as presented in Matias & Miele
+ type_init_margins : {'ones', 'X.', 'random', 'given'}, optional
+ How the margins are initialized, by default 'ones':
+ - `'ones'` : mu[t, i] = nu[t_, j] = 1
+ - `'X.'` : mu = X.sum((2)) nu = X.sum((1))
+ - `'random'` : mu and nu sampled from normal distribution
+ followed by an absolute value.
+ - `'given'` : mu an nu are given in the `fit()` method
+ min_float : float, optional
+ The minimum float used to avoid numerical issues, by default 1e-15
+ min_gamma : float
+ The minimum value of connectivity parameter gamma
+ especially important in CEM to avoid empty clusters
+ min_proba_Z : float, optional
+ The probability (between 0 and 1) at which the variational probabilities
+ for the row memberships are clipped, by default .05
+ min_proba_W : float, optional
+ The probability (between 0 and 1) at which the variational probabilities
+ for the column memberships are clipped, by default .05
+ min_margin : float, optional
+ The value at which the margins are clipped, by default 1e-10
+ min_proba_mixture_proportions : float, optional
+ The probability (between 0 and 1) at which the mixture proportions
+ are clipped, by default .05
+ threshold_absent_nodes : int, optional
+ Row or column nodes that have, at a given time step, a degree below
+ `threshold_absent_nodes` are considered absent and do not contribute
+ to the observed data log likelihood, by default 0
+ This value can be set this to -1 to avoid considering 0-degree nodes as absent
+ # TODO Verify compatibility with the rest of the code
+ dtype : str, optional
+ The dtype of the floats used in the model, by default 'float32'
+ random_state : int | np.random.RandomState | None, optional
+ Creates a random state generator from a seed or uses the given
+ random state generator, by default None
+ max_iter : int, optional
+ The maximum number of EM iterations for a single
+ initialization of the model, by default 500
+ tol_iter : float, optional
+ The decrease ratio below which we consider the algorithm
+ has converged, by default 1e-5
+ n_jobs : int, optional
+ The number of jobs used in the initialization clustering algorithm
+ in sklearn or spherecluster, by default -1 i.e. all cores
+ verbose : {0, 1, 2}, optional
+ 0 is silent, 1 is normal verbose and 2 is very verbose, by default 1
+ blockmodel_params : [type], optional
+ A dictionnary of parameters can can overwrite the class parameters
+ if non empty or None, by default None. The parameters that can be changed
+ are the following:
+ - `n_iter_min`
+ - `n_init_clustering_consensus`
+ - `loc_random_margins`
+ - `scale_random_margins`
+ model_id : int | None, optional
+ The numerical id of the model, used for debugging purposes, by default None
+ debug_list : list, optional
+ a list of strings that correspond to the names of the model attributes
+ whose value we wish to keep track of during the iterations of the
+ EM algorithm, by default []. `debug_list` must be a sublist of
+ `self.model_parameters`.
+ debug_output : str, optional
+ The directory where the values of the parameters in `debug_list`
+ will be outputed in the form of .npy files, by default '.'
+
+ dLBM Specific Parameters
+ ----------
+ n_iter_supp_smoothing : int, optional
+ The maximum number of smoothing iterations for a given step of
+ the smoothing schedule, by default 5
+ parameter_smoothing : bool
+ Whether we apply parameter smoothing or not
+ smoothing_schedule : class SmoothingSchedule
+ Describes how the parameters will be smoothed during the
+ iterations of the EM algorithm
+ diag_pi_init : float, optional
+ The value of each entry of the diagonal of the transition
+ matrix pi at initialization, by default None which results
+ in pi being first estimated using a M-step
+ diag_rho_init : float, optional
+ The value of each entry of the diagonal of the transition
+ matrix pi at initialization, by default None which results
+ in rho being first estimated using a M-step
+ prior_diagonal_pi : float, optional
+ Used as an informative prior for the diagonal terms of pi,
+ and produces pseudocounts for intr-cluster transitions,
+ by default 0.
+ prior_diagonal_rho : float, optional
+ Used as an informative prior for the diagonal terms of rho,
+ and produces pseudocounts for intr-cluster transitions,
+ by default 0.
+ cluster_perturbation_rate : float, optional
+ The probability of applying a new permutation between two
+ clusters at a given time step in the intial_partition. This
+ sampling of two-cycles continues until False is sampled or
+ every cluster has been permutated once, by default .1
+ """
+
+ def __init__(
+ self,
+ # Global blockmodel arguments
+ Kz=None,
+ Kw=None,
+ init_type=None,
+ em_type=None,
+ n_init=None,
+ n_init_clustering=100,
+ node_perturbation_rate=.1,
+ model_type=None,
+ type_init_margins='ones',
+ min_float=1e-15,
+ min_gamma=None,
+ min_proba_Z=.05,
+ min_proba_W=.05,
+ min_margin=1e-10,
+ min_proba_mixture_proportions=.05,
+ threshold_absent_nodes=0,
+ dtype='float32',
+ random_state=None,
+ max_iter=500,
+ tol_iter=1e-5,
+ n_jobs=-1,
+ verbose=1,
+ blockmodel_params=None,
+ model_id=None,
+ debug_list=None,
+ debug_output='.',
+ # Specific DLBM arguments
+ n_iter_supp_smoothing=5,
+ parameter_smoothing=None,
+ smoothing_schedule=None,
+ diag_pi_init=None,
+ diag_rho_init=None,
+ prior_diagonal_pi=0.,
+ prior_diagonal_rho=0.,
+ cluster_perturbation_rate=.1,
+ ):
+ super().__init__(
+ Kz=Kz,
+ Kw=Kw,
+ init_type=init_type,
+ em_type=em_type,
+ n_init=n_init,
+ n_init_clustering=n_init_clustering,
+ node_perturbation_rate=node_perturbation_rate,
+ model_type=model_type,
+ type_init_margins=type_init_margins,
+ min_float=min_float,
+ min_gamma=min_gamma,
+ min_proba_Z=.05,
+ min_proba_W=.05,
+ min_margin=1e-10,
+ min_proba_mixture_proportions=min_proba_mixture_proportions,
+ threshold_absent_nodes=threshold_absent_nodes,
+ dtype=dtype,
+ random_state=random_state,
+ max_iter=max_iter,
+ tol_iter=tol_iter,
+ n_jobs=n_jobs,
+ verbose=verbose,
+ blockmodel_params=blockmodel_params,
+ model_id=model_id,
+ debug_list=debug_list,
+ debug_output=debug_output
+ )
+
+ for arg in ['parameter_smoothing', 'smoothing_schedule']:
+ if arg is None:
+ raise (f'Argument {arg} for class {type(self).__name__} must be initialized '
+ 'explicitly. See documentation for possible values.')
+
+ assert isinstance(parameter_smoothing, bool)
+ assert ((smoothing_schedule[1:] - smoothing_schedule[:-1]) > 0).all()
+ assert n_iter_supp_smoothing > 0
+
+ self.directed = True # for compatibility with dsbm
+
+ self.parameter_smoothing = parameter_smoothing
+ self.smoothing_schedule = smoothing_schedule if self.parameter_smoothing else []
+
+ self.diag_pi_init = diag_pi_init
+ self.diag_rho_init = diag_rho_init
+ self.prior_diagonal_pi = prior_diagonal_pi
+ self.prior_diagonal_rho = prior_diagonal_rho
+
+ self.n_iter_supp_smoothing = n_iter_supp_smoothing
+ self.cluster_perturbation_rate = cluster_perturbation_rate
+
+ self.pi, self.rho = None, None
+ self.pi_mask, self.rho_mask = None, None
+ self.prior_pi, self.prior_rho = None, None
+
+ # In the dynamic case, if there is margins, they are estimated
+ self.estimated_margins = (self.model_type == 'with_margins')
+ self.model_parameters = ['log_alpha', 'log_beta', 'log_pi', 'log_rho', 'gamma']
+ if self.model_type == 'with_margins':
+ self.model_parameters += ['mu', 'nu']
+
+ self.absent_nodes = None
+
+ self.tau = None
+
+ self.density_part_Lc = None
+
+ def fit(
+ self, X,
+ given_Z=None, given_W=None,
+ given_mu=None, given_nu=None
+ ):
+ """Fits the model to the given data
+
+ Parameters
+ ----------
+ X : np.ndarray with ndim == 3, first axis representing the
+ time, second reprenting the rows and third the columns |
+ list of scipy.sparse matrices, where each matrix is a snapshot
+ of the graph.
+ The discrete-time dynamic bi-partite graph to fit the data to.
+ given_Z : np.ndarray of shape (, n_rows) or shape (n_timesteps, n_rows)
+ such that each values of the array indicates the cluster of the row.
+ The values must be in {0, ..., Kz-1}, as returned by sklearn api
+ e.g. KMeans().fit(X).labels_
+ This parameter must not be None if the `init_type` parameter is
+ `'given'`, otherwise it will not be used. By default None
+ given_W : np.ndarray of shape (, n_cols) or shape (n_timesteps, n_cols)
+ such that each values of the array indicates the cluster of the row.
+ The values must be in {0, ..., Kz-1}, as returned by sklearn api
+ e.g. KMeans().fit(X).labels_
+ This parameter must not be None if the `init_type` parameter is
+ `'given'`, otherwise it will not be used. By default None
+ given_mu : np.ndarray of shape (n_timesteps, n_rows), optional
+ Gives an initial value for the margin parameter mu of the model.
+ This parameter must not be None if the `type_init_margins` parameter is
+ `'given'`, otherwise it will not be used. By default None
+ given_nu : np.ndarray of shape (n_timesteps, n_cols), optional
+ Gives an initial value for the margin parameter nu of the model.
+ This parameter must not be None if the `type_init_margins` parameter is
+ `'given'`, otherwise it will not be used. By default None
+
+ Returns
+ -------
+ self : object
+ Fitted estimator.
+ """
+ general.check_X(
+ X=X,
+ is_graph=False,
+ self_loops=True,
+ directed=True
+ )
+ self.X = X
+ self.T = len(self.X)
+ self.N, self.D = self.X[0].shape
+ self.absent_nodes = absent_nodes.AbsentNodes(
+ self.X,
+ self.threshold_absent_nodes,
+ 'LBM'
+ )
+ self._set_transition_masks()
+ self._set_global_init_partition(given_Z, given_W)
+ self._init_debug()
+ self._set_data_margins()
+
+ (all_iter_criterions,
+ all_intermediate_iter_criterions,
+ all_icls,
+ all_row_partitions,
+ all_col_partitions) = [], [], [], [], []
+
+ # each initialization of the model
+ for init_number in range(self.n_init):
+ self._print_verbose_msg_init(init_number)
+ self._set_current_init_partition()
+ self._init_q()
+ self._init_margins(given_mu, given_nu)
+ self.tau = 0. # the smoothing parameter, \in [0., 1.]
+ self._full_m_step()
+ self._debug(init_number)
+
+ old_iter_criterion = - np.finfo(np.float32).max
+ new_iter_criterion = - np.finfo(np.float32).max
+ iter_criterions, intermediate_iter_criterions = [], []
+ iter_number = 0
+
+ # each iteration of the model, before smoothing
+ for _ in range(self.max_iter):
+ self._print_verbose_msg_iter(iter_number)
+ self._debug(init_number)
+
+ # E + M steps
+ old_iter_criterion = new_iter_criterion
+ interm_criterion, new_iter_criterion = self._fit_single()
+
+ iter_criterions.append(new_iter_criterion)
+ intermediate_iter_criterions.append(interm_criterion)
+
+ self._set_best_parameters(new_iter_criterion, init_number, iter_number)
+ delta_iter = general.get_delta(old_iter_criterion, new_iter_criterion)
+
+ if (iter_number >= self.n_iter_min) and (delta_iter < self.tol_iter):
+ self._print_verbose_converged(iter_number, None)
+ break
+ iter_number += 1
+
+ self._print_verbose_smoothing(self.smoothing_schedule, self.n_iter_supp_smoothing)
+
+ # each iteration of the model, with smoothing
+ for ind_tau, tau in enumerate(self.smoothing_schedule):
+ self.tau = tau
+ for _ in range(self.n_iter_supp_smoothing):
+ self._debug(init_number)
+
+ # E + M steps
+ old_iter_criterion = new_iter_criterion
+ interm_criterion, new_iter_criterion = self._fit_single()
+
+ iter_criterions.append(new_iter_criterion)
+ intermediate_iter_criterions.append(interm_criterion)
+
+ self._set_best_parameters(new_iter_criterion, init_number, iter_number)
+ delta_iter = general.get_delta(old_iter_criterion, new_iter_criterion)
+
+ if delta_iter < self.tol_iter:
+ self._print_verbose_converged(iter_number, ind_tau)
+ break
+ iter_number += 1
+ if self.verbose >= 1:
+ print('Done')
+
+ all_row_partitions.append(self.Z.argmax(axis=2).copy())
+ all_col_partitions.append(self.W.argmax(axis=2).copy())
+
+ all_iter_criterions.append(iter_criterions)
+ all_intermediate_iter_criterions.append(intermediate_iter_criterions)
+ all_icls.append(self.icl())
+
+ self.all_row_partitions = all_row_partitions
+ self.all_col_partitions = all_col_partitions
+ self.all_iter_criterions = all_iter_criterions
+ self.all_intermediate_iter_criterions = all_intermediate_iter_criterions
+ self.all_icls = all_icls
+
+ self.fitted = True
+ self._set_mixture_proportions()
+ self._write_best_parameters()
+ return self
+
+ def _fit_single(self):
+ """
+ a single iteration of EM:
+ - computes the log density
+ - applies a row E-step
+ - applies an M-step on row parameters
+ - computes the log density
+ - applies a col E-step
+ - applies an M-step on col parameters
+ - returns a tuple containing the likelihoods
+ after row E+M step and after col E+M step
+ """
+ L1 = self._fit_single_one_sided(mode='row')
+ L2 = self._fit_single_one_sided(mode='col')
+ return L1, L2
+
+ def _fit_single_one_sided(self, mode):
+ """
+ a single iteration of EM :
+ - applies a row/col E-step
+ - applies an M-step on row/col parameters
+ - returns the complete data loglikelihood after row/col iteration
+ """
+ L = 0.
+ num_gamma, den_gamma, den_mu, den_nu = self._init_num_den_parameters()
+
+ for t in range(self.T):
+ # e step
+ X_red_t = self._e_step(t, mode)
+
+ # sufficient statistics for the m step at time t
+ (num_gamma[t], den_gamma[t],
+ den_mu[t], den_nu[t], Lc_t) = self._m_step_t(t, X_red_t, mode)
+
+ # local part of the loglikelihhod
+ L += Lc_t
+ self.density_part_Lc = L
+
+ # m step
+ self._set_parameters(num_gamma, den_gamma, den_mu, den_nu)
+ self._update_mixture_proportions(mode)
+ L += self._mixture_part_complete_data_loglikelihood()
+ L += self.entropy()
+ return L
+
+ def entropy(self):
+ """Compute entropy"""
+ if self.em_type == 'VEM':
+ Hz = m_step.entropy_dynamic(
+ self.Z,
+ self.qz,
+ self.absent_nodes.appearing_row_nodes,
+ self.min_float
+ )
+ Hw = m_step.entropy_dynamic(
+ self.W,
+ self.qw,
+ self.absent_nodes.appearing_col_nodes,
+ self.min_float
+ )
+ return Hz + Hw
+
+ return 0.
+
+ def _m_step_t(self, t, X_red_t, mode):
+ """
+ Fills the numerators and denominators of
+ the parameters of the model at a given time step t
+
+ We fill num_gamma, den_gamma, den_mu, den_nu
+ time step by time step. We then smooth theses quantities if needed.
+
+ Note that we keep time variying numerators and denominators
+ even if the parameter does not depend on time in the model
+ since we compute sufficient statistics at time t to
+ estimate gamma, mu and nu
+
+ Note that, contrary to the static case, the complete
+ data log likelihood is computed after the E step. In fact,
+ otherwise, we would have to compute num_gamma[t] and
+ den_gamma[t] for each timestep, to obtain gamma. Then,
+ we would have to compute Lc_t for each t, requiring to
+ recompute reduced matrices for each time step
+ """
+ if self.model_type == 'with_margins':
+ mu_t, nu_t = self.mu[t], self.nu[t]
+ elif self.model_type == 'without_margins':
+ mu_t, nu_t = None, None
+
+ # num_gamma_t = X_ZW
+ num_gamma_t, den_gamma_t = m_step.update_gamma(
+ mode, self.estimated_margins,
+ self.X[t], self.Z[t], self.W[t],
+ X_red_t, mu_t, nu_t,
+ self.dtype, self.model_type
+ )
+ gamma_no_smooth_t = m_step.get_gamma(
+ num_gamma_t,
+ den_gamma_t,
+ self.em_type,
+ self.min_float,
+ self.min_gamma
+ )
+ if self.model_type == 'with_margins':
+ den_mu_t = m_step.get_denominator_mu(
+ self.Z[t],
+ self.W[t],
+ self.nu[t],
+ gamma_no_smooth_t
+ )
+ mu_no_smooth_t = self.Xi_[t] / (den_mu_t + self.min_float)
+ mu_no_smooth_t[self.absent_nodes.absent_row_nodes[t]] = self.min_float
+ den_nu_t = m_step.get_denominator_nu(
+ self.Z[t],
+ self.W[t],
+ mu_no_smooth_t,
+ gamma_no_smooth_t
+ )
+
+ elif self.model_type == 'without_margins':
+ den_mu_t, den_nu_t = None, None
+
+ # in mode 'init', at initialization, there
+ # is no self.gamma yet. Ideally, we should first
+ # estimate gamma, and then compute Lc, but
+ # it is less expensive to compute an approximation
+ # of Lc using the local estimate of gamma
+ if mode == 'init':
+ gamma_t = gamma_no_smooth_t
+ else:
+ gamma_t = self.gamma if self.model_type == 'with_margins' else self.gamma[t]
+
+ Lc_t = m_step.compute_Lc_static_density(
+ self.model_type,
+ self.estimated_margins,
+ gamma_t,
+ mu_t, nu_t,
+ self.Xi_[t], self.X_j[t],
+ num_gamma_t,
+ self.min_float
+ )
+ return num_gamma_t, den_gamma_t, den_mu_t, den_nu_t, Lc_t
+
+ def _update_pi_rho(self, ZW, qzw, prior, mask):
+ if self.em_type == 'VEM':
+ pi_rho = m_step.update_pi_rho(ZW, qzw, prior, mask)
+ elif self.em_type == 'CEM':
+ pi_rho = m_step.update_pi_rho_cem(ZW, prior, mask, self.dtype)
+
+ pi_rho = m_step.correct_pi_rho(
+ pi_rho,
+ self.min_proba_mixture_proportions,
+ self.min_float
+ )
+ return np.log(pi_rho + self.min_float)
+
+ def _e_step(self, t, mode):
+ if self.em_type == 'VEM':
+ return self._e_step_vem(t, mode)
+ if self.em_type == 'CEM':
+ return self._e_step_cem(t, mode)
+ raise ValueError(self, "em_type", self.em_type)
+
+ def _e_step_cem(self, t, mode):
+ mu_t, nu_t = (self.mu[t], self.nu[t]) if self.model_type == 'with_margins' else (None, None)
+ # gamma_t = self.gamma if self.model_type == 'with_margins' else self.gamma[t]
+ if mode == 'row':
+ self.Z[t], X_red_t = e_step.e_step_t_dynamic_cem(
+ mode='row',
+ model_type=self.model_type,
+ estimated_margins=self.estimated_margins,
+ X_t=self.X[t],
+ gamma=self.gamma,
+ dtype=self.dtype,
+ min_float=self.min_float,
+ log_pi_rho=self.log_pi,
+ log_alpha_beta=self.log_alpha,
+ ind_appearing_nodes_t=self.absent_nodes.appearing_row_nodes[t],
+ ind_absent_nodes_t=self.absent_nodes.absent_row_nodes[t],
+ ZW_tm1=None if t == 0 else self.Z[t - 1],
+ ZW_tp1=None if t == (self.T - 1) else self.Z[t + 1],
+ Z=None,
+ W=self.W[t],
+ nu_t=nu_t,
+ mu_t=mu_t
+ )
+ elif mode == 'col':
+ self.W[t], X_red_t = e_step.e_step_t_dynamic_cem(
+ mode='col',
+ model_type=self.model_type,
+ estimated_margins=self.estimated_margins,
+ X_t=self.X[t],
+ gamma=self.gamma,
+ dtype=self.dtype,
+ min_float=self.min_float,
+ log_pi_rho=self.log_rho,
+ log_alpha_beta=self.log_beta,
+ ind_appearing_nodes_t=self.absent_nodes.appearing_col_nodes[t],
+ ind_absent_nodes_t=self.absent_nodes.absent_col_nodes[t],
+ ZW_tm1=None if t == 0 else self.W[t - 1],
+ ZW_tp1=None if t == (self.T - 1) else self.W[t + 1],
+ Z=self.Z[t],
+ W=None,
+ nu_t=nu_t,
+ mu_t=mu_t
+ )
+ return X_red_t
+
+ def _e_step_vem(self, t, mode):
+ """
+ Applies an E-step for a given time step t
+ on the row or column posterior distributions.
+ If t == 0:
+ updates the initial posterior proba (ZW[0])
+ else:
+ in VEM:
+ updates the transition posterior proba (qzw[t])
+ then updates posterior proba of appearing nodes (ZW_app)
+ then updates ZW[t] with qzw[t] and ZW[t-1]
+ in CEM:
+ computes an unormalized posterior proba vector zw_crit
+ then updates ZW[t] with zw_crit and ZW[t-1]
+ mode : row or col e-step
+ """
+ mu_t, nu_t = (self.mu[t], self.nu[t]) if self.model_type == 'with_margins' else (None, None)
+ # gamma_t = self.gamma if self.model_type == 'with_margins' else self.gamma[t]
+ if mode == 'row':
+ self.Z[t], X_red_t = e_step.e_step_t_dynamic_vem(
+ mode='row',
+ first_ts=(t == 0),
+ model_type=self.model_type,
+ estimated_margins=self.estimated_margins,
+ X_t=self.X[t],
+ gamma=self.gamma,
+ dtype=self.dtype,
+ min_float=self.min_float,
+ qzw_tp1=self.qz[t + 1] if (t != self.T - 1) else None,
+ log_pi_rho=self.log_pi,
+ log_alpha_beta=self.log_alpha,
+ ind_appearing_nodes_t=self.absent_nodes.appearing_row_nodes[t],
+ ind_absent_nodes_t=self.absent_nodes.absent_row_nodes[t],
+ min_proba=self.min_proba_Z,
+ ZW_tm1=None if t == 0 else self.Z[t - 1],
+ Z=None,
+ W=self.W[t],
+ nu_t=nu_t,
+ mu_t=mu_t
+ )
+ elif mode == 'col':
+ self.W[t], X_red_t = e_step.e_step_t_dynamic_vem(
+ mode='col',
+ first_ts=(t == 0),
+ model_type=self.model_type,
+ estimated_margins=self.estimated_margins,
+ X_t=self.X[t],
+ gamma=self.gamma,
+ dtype=self.dtype,
+ min_float=self.min_float,
+ qzw_tp1=self.qw[t + 1] if (t != self.T - 1) else None,
+ log_pi_rho=self.log_rho,
+ log_alpha_beta=self.log_beta,
+ ind_appearing_nodes_t=self.absent_nodes.appearing_col_nodes[t],
+ ind_absent_nodes_t=self.absent_nodes.absent_col_nodes[t],
+ min_proba=self.min_proba_W,
+ ZW_tm1=None if t == 0 else self.W[t - 1],
+ Z=self.Z[t],
+ W=None,
+ nu_t=nu_t,
+ mu_t=mu_t
+ )
+ return X_red_t
+
+ def _update_mixture_proportions(self, mode):
+ """
+ Updates the mixture proportions
+ """
+ if mode == 'init':
+ self._update_alpha()
+ self._update_beta()
+ self._update_pi()
+ self._update_rho()
+ elif mode == 'row':
+ self._update_alpha()
+ self._update_pi()
+ elif mode == 'col':
+ self._update_beta()
+ self._update_rho()
+
+ def _set_parameters(self, num_gamma, den_gamma, den_mu, den_nu):
+ """
+ Updates gamma, mu and nu with their num_... and den_...
+ The smoothing is applied here.
+ Gamma, mu, nu can be constant or not. Anyway,
+ we deal with num_... and den_... as temporal signals
+ """
+ # gamma
+ if self.model_type == 'with_margins':
+ self.gamma = m_step.get_gamma(
+ num_gamma.sum(0),
+ den_gamma.sum(0),
+ self.em_type,
+ self.min_float,
+ self.min_gamma
+ )
+ elif self.model_type == 'without_margins':
+ if self.parameter_smoothing:
+ W_tau = m_step.smoothing_matrix(self.T, self.tau, self.dtype)
+ num_gamma = num_gamma.reshape((self.T, self.Kz * self.Kw))
+ den_gamma = den_gamma.reshape((self.T, self.Kz * self.Kw))
+
+ with np.errstate(under='ignore'):
+ smoothed_num_gamma = (W_tau @ num_gamma).reshape((self.T, self.Kz, self.Kw))
+ smoothed_den_gamma = (W_tau @ den_gamma).reshape((self.T, self.Kz, self.Kw))
+ self.gamma = m_step.get_gamma(
+ smoothed_num_gamma,
+ smoothed_den_gamma,
+ self.em_type,
+ self.min_float,
+ self.min_gamma
+ )
+ else:
+ self.gamma = m_step.get_gamma(
+ num_gamma,
+ den_gamma,
+ self.em_type,
+ self.min_float,
+ self.min_gamma
+ )
+ # margins
+ if self.model_type == 'with_margins':
+ if self.parameter_smoothing:
+ W_tau = m_step.smoothing_matrix(self.T, self.tau, self.dtype)
+ if self.absent_nodes.n_absent_row_tot > 0:
+ absent_nodes.replace_vals_absent(
+ den_mu,
+ self.absent_nodes.inds_prev_rows,
+ self.absent_nodes.ts_absent_rows
+ )
+ if self.absent_nodes.n_absent_col_tot > 0:
+ absent_nodes.replace_vals_absent(
+ den_nu,
+ self.absent_nodes.inds_prev_cols,
+ self.absent_nodes.ts_absent_cols
+ )
+ with np.errstate(under='ignore'):
+ smoothed_num_mu = W_tau @ self.Xi_
+ smoothed_den_mu = W_tau @ den_mu
+ self.mu = smoothed_num_mu / (smoothed_den_mu + self.min_float)
+
+ smoothed_num_nu = W_tau @ self.X_j
+ smoothed_den_nu = W_tau @ den_nu
+ self.nu = smoothed_num_nu / (smoothed_den_nu + self.min_float)
+ else:
+ self.mu = self.Xi_ / (den_mu + self.min_float)
+ self.nu = self.X_j / (den_nu + self.min_float)
+
+ self._correct_margins()
+
+ def _update_alpha(self):
+ self.log_alpha = m_step.update_alpha_beta_dynamic(
+ self.Z,
+ self.absent_nodes.n_absent_row_tot,
+ self.absent_nodes.appearing_row_nodes,
+ self.absent_nodes.absent_row_nodes[0],
+ self.min_float,
+ self.min_proba_mixture_proportions,
+ self.dtype
+ )
+
+ def _update_beta(self):
+ self.log_beta = m_step.update_alpha_beta_dynamic(
+ self.W,
+ self.absent_nodes.n_absent_col_tot,
+ self.absent_nodes.appearing_col_nodes,
+ self.absent_nodes.absent_col_nodes[0],
+ self.min_float,
+ self.min_proba_mixture_proportions,
+ self.dtype
+ )
+
+ def _update_pi(self):
+ self.log_pi = self._update_pi_rho(
+ self.Z,
+ self.qz,
+ self.prior_pi,
+ self.pi_mask
+ )
+
+ def _update_rho(self):
+ self.log_rho = self._update_pi_rho(
+ self.W,
+ self.qw,
+ self.prior_rho,
+ self.rho_mask
+ )
+
+ def _correct_margins(self):
+ """
+ Sets margins to zero for absent nodes.
+ Useful if smoothing is applied
+
+ It should be noted that num_mu and den_mu are 0 for absent nodes.
+ When the smoothing is applied, the margin of
+ an absent node does not contribute to the
+ other margins. Moreover, for an absent node,
+ the smoothed margin can be non zero : it must then
+ be corrected.
+ """
+ np.clip(self.mu, self.min_margin, None, self.mu)
+ np.clip(self.nu, self.min_margin, None, self.nu)
+
+ # important
+ if self.model_type == 'with_margins':
+ for t in range(self.T):
+ self.mu[t][self.absent_nodes.absent_row_nodes[t]] = self.min_float
+ self.nu[t][self.absent_nodes.absent_col_nodes[t]] = self.min_float
+
+ def _init_num_den_parameters(self):
+ """
+ Initializes the time-dependant vectors
+ that will be filled in _m_step_t()
+ """
+ num_gamma = np.zeros((self.T, self.Kz, self.Kw), dtype=self.dtype)
+ den_gamma = np.zeros((self.T, self.Kz, self.Kw), dtype=self.dtype)
+
+ # we initialize den_mu, den_nu even if the model
+ # is without margins, for consistency
+ den_mu = np.zeros((self.T, self.N), dtype=self.dtype)
+ den_nu = np.zeros((self.T, self.D), dtype=self.dtype)
+
+ return num_gamma, den_gamma, den_mu, den_nu
+
+ def _init_q(self):
+ """
+ Initializes the variational probability distributions:
+ qz, Z, Z_app, qw, W, W_app,
+ where qz and qw are initialized only in VEM
+ """
+ if self.em_type == 'VEM':
+ self.Z = np.zeros((self.T, self.N, self.Kz), dtype=self.dtype)
+ self.Z[0] = init.init_ZW0(
+ self.Kz,
+ self.current_init_Z[0],
+ self.min_proba_Z,
+ self.dtype
+ )
+ self.qz = init.init_qzw(
+ self.Kz,
+ self.current_init_Z,
+ self.min_proba_Z,
+ self.dtype
+ )
+ for t in range(1, self.T):
+ self.Z[t] = e_step.update_ZW_t_vem(
+ self.Z[t - 1],
+ self.qz[t - 1],
+ None, None, None,
+ self.min_proba_Z
+ )
+ self.Z = e_step.correct_ZW(self.Z, self.min_proba_Z)
+
+ self.W = np.zeros((self.T, self.D, self.Kw), dtype=self.dtype)
+ self.W[0] = init.init_ZW0(
+ self.Kw,
+ self.current_init_W[0],
+ self.min_proba_W,
+ self.dtype
+ )
+ self.qw = init.init_qzw(
+ self.Kw,
+ self.current_init_W,
+ self.min_proba_W,
+ self.dtype
+ )
+ for t in range(1, self.T):
+ self.W[t] = e_step.update_ZW_t_vem(
+ self.W[t - 1],
+ self.qw[t - 1],
+ None, None, None,
+ self.min_proba_W
+ )
+ self.W = e_step.correct_ZW(self.W, self.min_proba_W)
+
+ elif self.em_type == 'CEM':
+ self.qz = None
+ self.Z = np.zeros((self.T, self.N, self.Kz), dtype='bool')
+ for t in range(self.T):
+ self.Z[t] = init.init_ZW0(
+ self.Kz,
+ self.current_init_Z[t],
+ None, 'bool'
+ )
+ self.W = np.zeros((self.T, self.D, self.Kw), dtype='bool')
+ for t in range(self.T):
+ self.W[t] = init.init_ZW0(
+ self.Kw,
+ self.current_init_W[t],
+ None, 'bool'
+ )
+ self.qw = None
+
+ def _set_current_init_partition(self):
+ """
+ Creates the current init partition by
+ applying noise to the global init partition
+ """
+ self.current_init_Z = init.apply_perturbation_to_init_clustering(
+ init_partition=self.global_init_Z,
+ random_state=self.random_state,
+ node_perturbation_rate=self.node_perturbation_rate,
+ cluster_perturbation_rate=self.cluster_perturbation_rate,
+ dynamic=True
+ )
+ self.current_init_W = init.apply_perturbation_to_init_clustering(
+ init_partition=self.global_init_W,
+ random_state=self.random_state,
+ node_perturbation_rate=self.node_perturbation_rate,
+ cluster_perturbation_rate=self.cluster_perturbation_rate,
+ dynamic=True
+ )
+
+ def _set_global_init_partition(self, given_Z, given_W):
+ """
+ Sets the global row and column partitions of the model.
+ At each new initialization of the model, noise will be applied to
+ this partition to create the current init partition
+ """
+ # wether we convert the obtained (N) partition to a T x N matrix
+ if given_Z is not None or given_W is not None:
+ assert (given_Z is not None) and (given_W is not None)
+ assert self.init_type == 'given'
+ assert given_Z.ndim == given_W.ndim
+
+ # X_init : data matrix used for initial clustering
+ X_init_row = init.get_X_init(
+ self.X,
+ mode='row',
+ absent_nodes=self.absent_nodes.absent_row_nodes
+ )
+ self.global_init_Z = init.get_init_partition(
+ X_init=X_init_row,
+ Kzw=self.Kz,
+ init_type=self.init_type,
+ T=self.T,
+ random_state=self.random_state,
+ n_jobs=self.n_jobs,
+ n_init=self.n_init_clustering,
+ given_partition=given_Z
+ )
+ X_init_col = init.get_X_init(
+ self.X,
+ mode='col',
+ absent_nodes=self.absent_nodes.absent_col_nodes
+ )
+ self.global_init_W = init.get_init_partition(
+ X_init=X_init_col,
+ Kzw=self.Kw,
+ init_type=self.init_type,
+ T=self.T,
+ random_state=self.random_state,
+ n_jobs=self.n_jobs,
+ n_init=self.n_init_clustering,
+ given_partition=given_W
+ )
+
+ def _mixture_part_complete_data_loglikelihood(self):
+ """
+ Computes the terms in alpha, beta, rho and pi
+ of the log likelihood of the expected complete data
+ """
+ # alpha and beta part
+ alpha_part = m_step.complete_data_loglikelihood_alpha_beta(
+ self.log_alpha,
+ self.Z,
+ self.absent_nodes.absent_row_nodes[0],
+ self.absent_nodes.appearing_row_nodes
+ )
+ beta_part = m_step.complete_data_loglikelihood_alpha_beta(
+ self.log_beta,
+ self.W,
+ self.absent_nodes.absent_col_nodes[0],
+ self.absent_nodes.appearing_col_nodes
+ )
+ pi_part = m_step.complete_data_loglikelihood_pi_rho(
+ self.em_type,
+ self.log_pi,
+ self.Z, self.qz,
+ self.pi_mask
+ )
+ rho_part = m_step.complete_data_loglikelihood_pi_rho(
+ self.em_type,
+ self.log_rho,
+ self.W, self.qw,
+ self.rho_mask
+ )
+ return alpha_part + beta_part + pi_part + rho_part
+
+ def _set_data_margins(self):
+ if isinstance(self.X, list):
+ self.Xi_ = np.array([self.X[t].sum(1).A1 for t in range(self.T)])
+ self.X_j = np.array([self.X[t].sum(0).A1 for t in range(self.T)])
+ else:
+ self.Xi_ = self.X.sum(2)
+ self.X_j = self.X.sum(1)
+
+ if self.absent_nodes.n_absent_row_tot > 0:
+ absent_nodes.replace_vals_absent(
+ self.Xi_,
+ self.absent_nodes.inds_prev_rows,
+ self.absent_nodes.ts_absent_rows
+ )
+ if self.absent_nodes.n_absent_col_tot > 0:
+ absent_nodes.replace_vals_absent(
+ self.X_j,
+ self.absent_nodes.inds_prev_cols,
+ self.absent_nodes.ts_absent_cols
+ )
+
+ def _init_margins(self, given_mu=None, given_nu=None):
+ """
+ Initializes the margins mu and nu, that
+ can not be initialized from a given partition
+ as it is the case for alpha, pi, gamma, ...
+ """
+ # time dependant margins
+ if self.model_type == 'with_margins':
+ if self.type_init_margins == 'ones':
+ self.mu = np.ones((self.T, self.N), dtype=self.dtype)
+ self.nu = np.ones((self.T, self.D), dtype=self.dtype)
+ elif self.type_init_margins == 'X.':
+ self.mu = self.Xi_.copy()
+ self.nu = self.X_j.copy()
+ elif self.type_init_margins == 'random':
+ self.mu = np.abs(self.random_state.normal(
+ loc=self.loc_random_margins,
+ scale=self.scale_random_margins,
+ size=(self.T, self.N)
+ ).astype(self.dtype))
+ self.nu = np.abs(self.random_state.normal(
+ loc=self.loc_random_margins,
+ scale=self.scale_random_margins,
+ size=(self.T, self.D)
+ ).astype(self.dtype))
+ elif self.type_init_margins == 'given':
+ assert given_mu is not None and given_nu is not None
+ self.mu = given_mu.astype(self.dtype)
+ self.nu = given_nu.astype(self.dtype)
+
+ def _set_transition_masks(self):
+ # masks used to update pi and rho efficiently
+ # by only selecting present and non-appearing nodes
+ self.pi_mask = init.pi_rho_update_mask(
+ self.T, self.N,
+ self.absent_nodes.absent_row_nodes,
+ self.absent_nodes.appearing_row_nodes
+ )
+ self.rho_mask = init.pi_rho_update_mask(
+ self.T, self.D,
+ self.absent_nodes.absent_col_nodes,
+ self.absent_nodes.appearing_col_nodes
+ )
+
+ def _set_mixture_proportions(self):
+ self.alpha = np.exp(self.log_alpha)
+ self.beta = np.exp(self.log_beta)
+ self.pi = np.exp(self.log_pi)
+ self.rho = np.exp(self.log_rho)
+
+ def set_prior_transition_matrices(self):
+ """
+ For pi and rho, we consider an informatiVe dirichlet prior
+ for each row of the transition matrices.
+ The prior is Kzw x Kzw and st prior[k, l] = 1
+ and prior[k, k] = prior_diagonal > 1.
+ Its role is to favor partitions with less class transitions.
+ During the M-step, it adds the pseudocounts, with eg
+ pi_kl propto n[k, l] + prior[k, l] - 1
+ where n[k, l] is the number of class transitions, as in
+ a classical M step.
+ """
+ assert 0. <= self.prior_diagonal_pi <= 1.
+ assert 0. <= self.prior_diagonal_rho <= 1.
+
+ prior_pi_val = self.prior_diagonal_pi * self.N + 1
+ prior_rho_val = self.prior_diagonal_rho * self.D + 1
+
+ IZ = np.eye((self.Kz), dtype=self.dtype)
+ OZ = np.ones((self.Kz, self.Kz), dtype=self.dtype)
+ self.prior_pi = (prior_pi_val - 1.) * IZ + OZ
+
+ IW = np.eye((self.Kw), dtype=self.dtype)
+ OW = np.ones((self.Kw, self.Kw), dtype=self.dtype)
+ self.prior_rho = (prior_rho_val - 1.) * IW + OW
+
+ def _full_m_step(self):
+ """
+ Applies an M-step to all the current parameters.
+ Used after the initialization of the posterior probabilities.
+ """
+ # update all mixture prop
+ self.set_prior_transition_matrices()
+ self._update_mixture_proportions(mode='init')
+
+ # update gamma, mu, nu
+ # by filling num_mu, num_nu, num_gamma, den...
+ num_gamma, den_gamma, den_mu, den_nu = self._init_num_den_parameters()
+
+ for t in range(self.T):
+ (num_gamma[t], den_gamma[t],
+ den_mu[t], den_nu[t], _) = self._m_step_t(t=t, X_red_t=None, mode='init')
+
+ # now that the num and den of the params have
+ # been set, we set the params values (with or without smoothing)
+ self._set_parameters(num_gamma, den_gamma, den_mu, den_nu)
+
+ def _init_pi_rho(self):
+ """
+ Initializes pi and rho, the transition matrices,
+ with a M step or with a matrix with diagonal
+ value diag_pi_rho_init
+ """
+ if self.diag_pi_init is not None:
+ val_off_diag = (1. - self.diag_pi_init) / (self.Kz - 1)
+ OZ = np.full((self.Kz, self.Kz), val_off_diag)
+ IZ = np.eye(self.Kz)
+ pi = OZ + (self.diag_pi_init - val_off_diag) * IZ
+ self.log_pi = np.log(pi)
+ else:
+ self._update_pi()
+
+ if self.diag_rho_init is not None:
+ val_off_diag = (1. - self.diag_rho_init) / (self.Kw - 1)
+ OW = np.full((self.Kw, self.Kw), val_off_diag)
+ IW = np.eye(self.Kw)
+ rho = OW + (self.diag_rho_init - val_off_diag) * IW
+ self.log_rho = np.log(rho)
+ else:
+ self._update_rho()
+
+ def icl(self):
+ """Compute ICL"""
+ icl_z = general.compute_mixture_exact_icl(self.Z, self.Kz, self.N)
+ icl_w = general.compute_mixture_exact_icl(self.W, self.Kw, self.D)
+ bic_penalty = (
+ -.5 * (self.Kz * self.Kw + self.T * (self.N + self.D)) *
+ np.log(self.T * self.N * self.D)
+ )
+ val = (
+ icl_z + icl_w +
+ self.density_part_Lc +
+ bic_penalty
+ )
+ return val
+
+ def best_partition(self, mode, n_first=1):
+ """
+ returns a list of tuple of partitions
+ if 1 partition: [(part_1), ..., (part_n_first)]
+ if 2 partitions: [(row_part_1, col_part_1), ...,
+ (row_part_n_first, col_part_n_first)]
+
+ mode == 'likelihood' returns the n_first best partitions
+ in terms of likelihood
+
+ if mode == 'consensus: hbgf' or mode == 'consensus: cspa'
+ returns 1 consensus partition over n_first partitions
+ """
+ assert self.fitted
+ best_partitions = metrics.sort_partitions(
+ self.all_iter_criterions,
+ [self.all_row_partitions, self.all_col_partitions],
+ n_first
+ )
+ # put absent nodes in cluster -1
+ for p1, p2 in best_partitions:
+ for t in range(self.T):
+ for i in self.absent_nodes.absent_row_nodes[t]:
+ p1[t, i] = -1
+ for j in self.absent_nodes.absent_col_nodes[t]:
+ p2[t, j] = -1
+
+ if mode == 'likelihood':
+ return best_partitions
+ if mode in ['consensus: hbgf', 'consensus: cspa']:
+ best_row_partitions = [x[0] for x in best_partitions]
+ best_col_partitions = [x[1] for x in best_partitions]
+ if mode == 'consensus: hbgf':
+ Z_consensus = consensus.hbgf(
+ best_row_partitions,
+ self.n_init_clustering_consensus
+ ).reshape(self.T, self.N)
+ W_consensus = consensus.hbgf(
+ best_col_partitions,
+ self.n_init_clustering_consensus
+ ).reshape(self.T, self.D)
+ elif mode == 'consensus: cspa':
+ Z_consensus = consensus.cspa(
+ best_row_partitions, self.n_init_clustering_consensus
+ ).reshape(self.T, self.N)
+ W_consensus = consensus.cspa(
+ best_col_partitions,
+ self.n_init_clustering_consensus
+ ).reshape(self.T, self.D)
+ return [(Z_consensus, W_consensus)]
+ raise ValueError
diff --git a/dcblockmodels/models/hlbm.py b/dcblockmodels/models/hlbm.py
new file mode 100644
index 0000000..ebcdbeb
--- /dev/null
+++ b/dcblockmodels/models/hlbm.py
@@ -0,0 +1,675 @@
+"""Pairwise semi-supervised Latent Block Model with a Hidden Markov Random Field"""
+
+
+import os
+import sys
+import warnings
+
+import numpy as np
+import scipy as sp
+from numba import NumbaPendingDeprecationWarning
+
+from .blockmodel import BlockModel
+from .. import metrics
+from .utils import (
+ e_step,
+ m_step,
+ init,
+ general,
+ similarity_matrices,
+ consensus
+)
+
+sys.stderr = open(os.devnull, "w") # pylint: disable=R1732,W1514
+
+warnings.filterwarnings("ignore", category=NumbaPendingDeprecationWarning)
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+
+
+class HLBM(BlockModel):
+ """Semi-supervised Latent Block Model with a pairwise Hidden Markov Random Field
+
+ Block Model General Parameters
+ ----------
+ Kz : int
+ Number of row clusters
+ Kw : int
+ Number of column clusters
+ init_type : {'random', 'kmeans', 'skmeans', 'given'}, optional,
+ by default 'skmeans'. The methods used for the initialization of
+ the algorithm. It is fit once for each side (rows and columns)
+ and gives two global initialization partitions. These partitions
+ are then used to initialization-specific partitions using the
+ parameters `node_perturbation_rate` and `cluster_perturbation_rate`:
+ - '`random'` randomly initialized clusters,
+ - `'kmeans'` clusters initialized using `sklearn.cluster.KMeans`,
+ - `'skmeans'` clusters initialized using `spherecluster.SphericalKMeans`,
+ - `'given'` given in the `fit()` method
+ em_type : {'VEM', 'CEM'}
+ The EM algorithm used
+ n_init : int
+ Number of initializations of the model. Each initialization
+ will start from different initial row and col partitions
+ and will converge to a partition and parameter set with
+ a corresponding complete-data log likelihood. One can then
+ select one of these partitions using the `best_partition()`
+ method.
+ n_init_clustering : int, optional
+ the number of initializations of the clustering algorithm
+ chosen in `'init_type'`, by default 100
+ node_perturbation_rate : float, optional
+ the fraction of nodes (row or cols) that are reassgined to
+ a random cluster at each new intitialization of the model, by default .1
+ model_type : {'with_margins', 'without_margins'}
+ The dynamic model used:
+ - `'with_margins'` : a model with dynamic margins
+ and static connectivity matrix gamma
+ - `'without_margins'` : a model with only a dynamic
+ connectivity matrix as presented in Matias & Miele
+ type_init_margins : {'ones', 'X.', 'random', 'given'}, optional
+ How the margins are initialized, by default 'ones':
+ - `'ones'` : mu[i] = nu[j] = 1
+ - `'X.'` : mu = X.sum((1)) nu = X.sum((0))
+ - `'random'` : mu and nu sampled from normal distribution
+ followed by an absolute value.
+ - `'given'` : mu an nu are given in the `fit()` method
+ min_float : float, optional
+ The minimum float used to avoid numerical issues, by default 1e-15
+ min_gamma : float
+ The minimum value of connectivity parameter gamma
+ especially important in CEM to avoid empty clusters
+ min_proba_Z : float, optional
+ The probability (between 0 and 1) at which the variational probabilities
+ for the row memberships are clipped, by default .05
+ min_proba_W : float, optional
+ The probability (between 0 and 1) at which the variational probabilities
+ for the column memberships are clipped, by default .05
+ min_margin : float, optional
+ The value at which the margins are clipped, by default 1e-10
+ min_proba_mixture_proportions : float, optional
+ The probability (between 0 and 1) at which the mixture proportions
+ are clipped, by default .05
+ threshold_absent_nodes : int, optional
+ Row or column nodes that have, at a given time step, a degree below
+ `threshold_absent_nodes` are considered absent and do not contribute
+ to the observed data log likelihood, by default 0
+ dtype : str, optional
+ The dtype of the floats used in the model, by default 'float32'
+ random_state : int | np.random.RandomState | None, optional
+ Creates a random state generator from a seed or uses the given
+ random state generator, by default None
+ max_iter : int, optional
+ The maximum number of EM iterations for a single
+ initialization of the model, by default 50
+ tol_iter : float, optional
+ The decrease ratio below which we consider the algorithm
+ has converged, by default 1e-5
+ n_jobs : int, optional
+ The number of jobs used in the initialization clustering algorithm
+ in sklearn or spherecluster, by default -1 i.e. all cores
+ verbose : {0, 1, 2}, optional
+ 0 is silent, 1 is normal verbose and 2 is very verbose, by default 1
+ blockmodel_params : [type], optional
+ A dictionnary of parameters can can overwrite the class parameters
+ if non empty or None, by default None. The parameters that can be changed
+ are the following:
+ - `n_iter_min`
+ - `n_init_clustering_consensus`
+ - `loc_random_margins`
+ - `scale_random_margins`
+ model_id : int | None, optional
+ The numerical id of the model, used for debugging purposes, by default None
+ debug_list : list, optional
+ a list of strings that correspond to the names of the model attributes
+ whose value we wish to keep track of during the iterations of the
+ EM algorithm, by default []. `debug_list` must be a sublist of
+ `self.model_parameters`.
+ debug_output : str, optional
+ The directory where the values of the parameters in `debug_list`
+ will be outputed in the form of .npy files, by default '.'
+
+ hLBM Specific Parameters
+ ----------
+ estimated_margins : True | False
+ Whether the margins mu and nu are estimated or are
+ set to the observed margins X.sum(1) and X.sum(0)
+ regularization_mode : {'all', 'mixture'}
+ Whether we consider the mixture proportions as
+ an external field in the HMRF (`'all'`) or we consider
+ the mixture proportions outside the HMRF (`'mixture'`)
+ compute_regularization : bool, optional
+ Whether we compute the regularization term in the criterion,
+ which is computationally costly, by default True
+ regularize_row : bool, optional
+ Whether we use the similarity matrix S_r to regularize the
+ model or not, by default False
+ regularize_col : bool, optional
+ Whether we use the similarity matrix S_c to regularize the
+ model or not, by default False
+ multiplicative_init_rows : bool, optional
+ Whether we use the Must-Link relationships in the initialization
+ of the row partition or not, by default True
+ multiplicative_init_cols : bool, optional
+ Whether we use the Must-Link relationships in the initialization
+ of the column partition or not, by default True
+ power_multiplicative_init : int if multiplicative_init_rows or
+ if multiplicative_init_cols | None, optional
+ The power to which we raise the stochastic matrix created from
+ the Must-Link relationships in the initialization,
+ by default None
+ damping_factor : float if ((regularize_row or regularize_col) and
+ em_type == 'VEM') | None, optional
+ The damping factor between 0 and 1 used in VEM, by default None
+ """
+ REGULARIZATION_MODES = ['all', 'mixture']
+ model_parameters = ['log_alpha', 'log_beta', 'gamma', 'mu', 'nu']
+
+ def __init__(
+ self,
+ # Global blockmodel arguments
+ Kz=None,
+ Kw=None,
+ init_type=None,
+ em_type=None,
+ n_init=None,
+ n_init_clustering=100,
+ node_perturbation_rate=.1,
+ model_type=None,
+ type_init_margins='ones',
+ min_float=1e-15,
+ min_gamma=None,
+ min_proba_Z=.05,
+ min_proba_W=.05,
+ min_margin=1e-10,
+ min_proba_mixture_proportions=.05,
+ threshold_absent_nodes=0,
+ dtype='float32',
+ random_state=None,
+ max_iter=50,
+ tol_iter=1e-5,
+ n_jobs=-1,
+ verbose=1,
+ blockmodel_params=None,
+ model_id=None,
+ debug_list=None,
+ debug_output='.',
+ # Specific HLBM arguments
+ estimated_margins=None,
+ regularization_mode=None,
+ compute_regularization=True,
+ regularize_row=False,
+ regularize_col=False,
+ multiplicative_init_rows=True,
+ multiplicative_init_cols=True,
+ power_multiplicative_init=None,
+ damping_factor=None,
+ ):
+ super().__init__(
+ Kz=Kz,
+ Kw=Kw,
+ init_type=init_type,
+ em_type=em_type,
+ n_init=n_init,
+ min_float=min_float,
+ n_init_clustering=n_init_clustering,
+ node_perturbation_rate=node_perturbation_rate,
+ model_type=model_type,
+ type_init_margins=type_init_margins,
+ min_gamma=min_gamma,
+ min_proba_Z=.05,
+ min_proba_W=.05,
+ min_margin=1e-10,
+ min_proba_mixture_proportions=min_proba_mixture_proportions,
+ threshold_absent_nodes=threshold_absent_nodes,
+ dtype=dtype,
+ random_state=random_state,
+ max_iter=max_iter,
+ tol_iter=tol_iter,
+ n_jobs=n_jobs,
+ verbose=verbose,
+ blockmodel_params=blockmodel_params,
+ model_id=model_id,
+ debug_list=debug_list,
+ debug_output=debug_output
+ )
+
+ for arg in ['regularization_mode']:
+ if arg is None:
+ raise (f'Argument {arg} for class {type(self).__name__} must be initialized '
+ 'explicitly. See documentation for possible values.')
+
+ assert isinstance(multiplicative_init_rows, bool)
+ assert isinstance(multiplicative_init_cols, bool)
+ assert regularization_mode in self.REGULARIZATION_MODES
+ assert 0. < min_proba_Z < 1.
+ assert 0. < min_proba_W < 1.
+ assert min_margin > 0.
+
+ if self.model_type == 'with_margins':
+ assert isinstance(estimated_margins, bool)
+ else:
+ assert not estimated_margins
+ self.estimated_margins = bool(estimated_margins)
+
+ self.regularization_mode = regularization_mode
+ self.regularize_row = regularize_row
+ self.regularize_col = regularize_col
+ self.compute_regularization = compute_regularization
+
+ if multiplicative_init_rows or multiplicative_init_cols:
+ assert power_multiplicative_init >= 1
+ self.multiplicative_init_rows = multiplicative_init_rows
+ self.multiplicative_init_cols = multiplicative_init_cols
+ self.power_multiplicative_init = power_multiplicative_init
+
+ # damping_factor used for damping in HMRF VEM
+ self.damping_factor = None
+ if self.em_type == 'VEM' and (self.regularize_row or self.regularize_col):
+ assert 0. < damping_factor < 1.
+ self.damping_factor = damping_factor
+
+ self.lambda_r, self.lambda_c = None, None
+ self.S_r, self.S_c, self.P_r, self.P_c = None, None, None, None
+
+ def fit(
+ self,
+ X,
+ lambda_r=None,
+ S_r=None,
+ lambda_c=None,
+ S_c=None,
+ given_Z=None,
+ given_W=None
+ ):
+ """Fits the model to the given data"""
+ self.X = X
+ self.N, self.D = self.X.shape
+ self._set_similarity_parameters(lambda_r, S_r, lambda_c, S_c)
+ self._set_global_init_partition(given_Z, given_W)
+ self._init_debug()
+ seeds = self.random_state.randint(np.iinfo(np.int32).max, size=self.n_init)
+
+ (all_iter_criterions,
+ all_intermediate_iter_criterions,
+ all_regularizations,
+ all_intermediate_regularizations,
+ all_row_partitions,
+ all_col_partitions) = [], [], [], [], [], []
+
+ for i, _ in enumerate(seeds):
+ self._print_verbose_msg_init(i)
+ self._set_current_init_partition()
+ self._init_q()
+ self._init_parameters()
+
+ old_iter_criterion = - np.finfo(np.float32).max
+ new_iter_criterion = - np.finfo(np.float32).max
+ iter_criterions, intermediate_iter_criterions = [], []
+ regularizations, intermediate_regularizations = [], []
+
+ for it in range(self.max_iter):
+ self._print_verbose_msg_iter(it)
+ self._debug(i)
+
+ # E + M steps
+ _old_interm_criterion, old_iter_criterion = new_iter_criterion, new_iter_criterion
+ interm_criterion, interm_reg, new_iter_criterion, new_reg = self._fit_single()
+
+ if self.regularize_row or self.regularize_col:
+ interm_criterion -= interm_reg
+ new_iter_criterion -= new_reg
+
+ self._set_best_parameters(new_iter_criterion, i, it)
+ delta_iter = general.get_delta(old_iter_criterion, new_iter_criterion)
+
+ if (it >= self.n_iter_min) and (delta_iter < self.tol_iter):
+ self._print_verbose_converged(it, None)
+ break
+
+ iter_criterions.append(new_iter_criterion)
+ intermediate_iter_criterions.append(interm_criterion)
+ regularizations.append(new_reg)
+ intermediate_regularizations.append(interm_reg)
+
+ all_row_partitions.append(general.to_dense(self.Z.copy()).argmax(1))
+ all_col_partitions.append(general.to_dense(self.W.copy()).argmax(1))
+ all_iter_criterions.append(iter_criterions)
+ all_intermediate_iter_criterions.append(intermediate_iter_criterions)
+ all_regularizations.append(regularizations)
+ all_intermediate_regularizations.append(intermediate_regularizations)
+
+ self.all_row_partitions = all_row_partitions
+ self.all_col_partitions = all_col_partitions
+ self.all_iter_criterions = all_iter_criterions
+ self.all_intermediate_iter_criterions = all_intermediate_iter_criterions
+ self.all_regularizations = all_regularizations
+ self.all_intermediate_regularizations = all_intermediate_regularizations
+ self.fitted = True
+ self._set_mixture_proportions()
+ self._write_best_parameters()
+
+ return self
+
+ def _m_step(self, X_red, mode):
+ # update mixture proportions (alpha, beta)
+ Z_ = general.get_class_counts(self.Z)
+ W_ = general.get_class_counts(self.W)
+
+ # alpha, beta
+ self.update_mixture_proportions(mode, Z_, W_)
+
+ # gamma
+ X_kl, den_gamma = m_step.update_gamma(
+ mode, self.estimated_margins,
+ self.X, self.Z, self.W, X_red, self.mu, self.nu,
+ self.dtype, self.model_type
+ )
+ self.gamma = m_step.get_gamma(
+ X_kl, den_gamma,
+ self.em_type, self.min_float, self.min_gamma
+ )
+
+ # mu, nu
+ if self.estimated_margins:
+ self.mu = m_step.update_mu(
+ self.Z, self.W, self.Xi_, self.nu, self.gamma, self.min_margin
+ )
+ self.nu = m_step.update_nu(
+ self.Z, self.W, self.X_j, self.mu, self.gamma, self.min_margin
+ )
+
+ # complete data log likelihood
+ Lc = m_step.compute_Lc_static(
+ self.model_type, self.estimated_margins,
+ self.regularization_mode, self.regularize_row, self.regularize_col,
+ self.P_r, self.P_c,
+ self.log_alpha, self.log_beta, self.gamma,
+ self.mu, self.nu, self.Xi_, self.X_j,
+ Z_, W_, X_kl, self.min_float
+ )
+
+ # regulariation term of the complete data log likelihood
+ # can be not computed for faster computations
+ if self.compute_regularization:
+ regularization = m_step.get_regularization(
+ self.regularize_row, self.regularize_col,
+ self.lambda_r, self.lambda_c,
+ self.S_r, self.S_c, self.Z, self.W
+ )
+ else:
+ regularization = 0.
+ return Lc, regularization
+
+ def _e_step(self, mode):
+ if mode == 'row':
+ self.Z, X_W = e_step.e_step_static(
+ X=self.X, gamma=self.gamma,
+ mode='row',
+ model_type=self.model_type, em_type=self.em_type,
+ estimated_margins=self.estimated_margins,
+ regularization_mode=self.regularization_mode,
+ regularize=self.regularize_row,
+ lambda_=self.lambda_r, S=self.S_r, P=self.P_r,
+ Z=self.Z, W=self.W,
+ log_alpha_beta=self.log_alpha,
+ mu=self.mu, nu=self.nu,
+ damping_factor=self.damping_factor,
+ dtype=self.dtype,
+ min_proba=self.min_proba_Z, min_float=self.min_float
+ )
+ return X_W
+ if mode == 'col':
+ self.W, X_Z = e_step.e_step_static(
+ X=self.X, gamma=self.gamma,
+ mode=mode,
+ model_type=self.model_type, em_type=self.em_type,
+ estimated_margins=self.estimated_margins,
+ regularization_mode=self.regularization_mode,
+ regularize=self.regularize_col,
+ lambda_=self.lambda_c, S=self.S_c, P=self.P_c,
+ Z=self.Z, W=self.W,
+ log_alpha_beta=self.log_beta,
+ mu=self.mu, nu=self.nu,
+ damping_factor=self.damping_factor,
+ dtype=self.dtype,
+ min_proba=self.min_proba_Z, min_float=self.min_float
+ )
+ return X_Z
+ raise ValueError(self, "mode", mode)
+
+ def _fit_single(self):
+ """
+ A single iteration of EM
+ X_Z and X_W are the reduced matrices
+ the M step returns the complete data log likelihood
+ and the regularization term
+ """
+ X_W = self._e_step(mode='row')
+ L1, reg1 = self._m_step(X_W, 'row')
+
+ X_Z = self._e_step(mode='col')
+ L2, reg2 = self._m_step(X_Z, 'col')
+ return L1, reg1, L2, reg2
+
+ def _init_parameters(self):
+ # log_alpha, beta, gamma initialized in the first M-step
+ self.gamma = np.zeros((self.Kz, self.Kw), dtype=self.dtype)
+ self.log_alpha = np.zeros((self.Kz), dtype=self.dtype)
+ self.log_beta = np.zeros((self.Kw), dtype=self.dtype)
+
+ if sp.sparse.issparse(self.X):
+ self.Xi_ = self.X.sum(1).A1
+ self.X_j = self.X.sum(0).A1
+ else:
+ self.Xi_ = self.X.sum(1)
+ self.X_j = self.X.sum(0)
+
+ if self.estimated_margins:
+ self.mu = np.ones((self.N), dtype=self.dtype)
+ self.nu = np.ones((self.D), dtype=self.dtype)
+ else:
+ self.mu, self.nu = self.Xi_, self.X_j
+ np.clip(self.mu, self.min_margin, None, self.mu)
+ np.clip(self.nu, self.min_margin, None, self.nu)
+
+ # first M step
+ self._m_step(None, mode='init')
+
+ def _set_global_init_partition(self, given_Z, given_W):
+ # X_init : data matrix used for initial clustering of the rows
+ X_init = init.get_X_init(
+ self.X,
+ mode='row',
+ absent_nodes=None
+ )
+ if self.regularize_row and self.multiplicative_init_rows:
+ X_init = similarity_matrices.init_transform(
+ X_init, self.S_r, self.power_multiplicative_init
+ )
+ self.global_init_Z = init.get_init_partition(
+ X_init=X_init,
+ Kzw=self.Kz,
+ init_type=self.init_type,
+ T=1,
+ random_state=self.random_state,
+ n_jobs=self.n_jobs,
+ n_init=self.n_init_clustering,
+ given_partition=given_Z
+ )
+ # X_init : data matrix used for initial clustering of the cols
+ X_init = init.get_X_init(
+ self.X,
+ mode='col',
+ absent_nodes=None
+ )
+ if self.regularize_col and self.multiplicative_init_cols:
+ X_init = similarity_matrices.init_transform(
+ X_init, self.S_c, self.power_multiplicative_init
+ )
+ self.global_init_W = init.get_init_partition(
+ X_init=X_init,
+ Kzw=self.Kw,
+ init_type=self.init_type,
+ T=1,
+ random_state=self.random_state,
+ n_jobs=self.n_jobs,
+ n_init=self.n_init_clustering,
+ given_partition=given_W
+ )
+
+ def _set_current_init_partition(self):
+ self.current_init_Z = init.apply_perturbation_to_init_clustering(
+ init_partition=self.global_init_Z,
+ random_state=self.random_state,
+ node_perturbation_rate=self.node_perturbation_rate,
+ dynamic=False
+ )
+ self.current_init_W = init.apply_perturbation_to_init_clustering(
+ init_partition=self.global_init_W,
+ random_state=self.random_state,
+ node_perturbation_rate=self.node_perturbation_rate,
+ dynamic=False
+ )
+
+ def _init_q(self):
+ if self.em_type == 'VEM':
+ self.Z = init.init_ZW0(
+ Kzw=self.Kz,
+ partition=self.current_init_Z,
+ min_proba=self.min_proba_Z,
+ dtype=self.dtype
+ )
+ self.W = init.init_ZW0(
+ Kzw=self.Kw,
+ partition=self.current_init_W,
+ min_proba=self.min_proba_W,
+ dtype=self.dtype
+ )
+ elif self.em_type == 'CEM':
+ self.Z = init.init_ZW0(
+ Kzw=self.Kz,
+ partition=self.current_init_Z,
+ min_proba=0,
+ dtype='bool'
+ )
+ self.Z = general.to_sparse(self.Z)
+
+ self.W = init.init_ZW0(
+ Kzw=self.Kw,
+ partition=self.current_init_W,
+ min_proba=0,
+ dtype='bool'
+ )
+ self.W = general.to_sparse(self.W)
+
+ def update_mixture_proportions(self, mode, Z_, W_):
+ """Update mixture proportions (alpha, beta) for rows and columns"""
+ if mode in ['row', 'all']:
+ self.log_alpha = Z_ / self.N
+ np.clip(self.log_alpha, self.min_proba_mixture_proportions, None, self.log_alpha)
+ np.divide(self.log_alpha, self.log_alpha.sum(), self.log_alpha)
+ self.log_alpha = np.log(self.log_alpha + self.min_float)
+ if mode in ['col', 'all']:
+ self.log_beta = W_ / self.D
+ np.clip(self.log_beta, self.min_proba_mixture_proportions, None, self.log_beta)
+ np.divide(self.log_beta, self.log_beta.sum(), self.log_beta)
+ self.log_beta = np.log(self.log_beta + self.min_float)
+
+ def icl(self):
+ """
+ Returns the ICL values for each init of the model,
+ sorted from best to worse complete data log likelihood
+ """
+ assert hasattr(self, 'best_parameters')
+ penality = (
+ 0.5 * (self.Kz - 1) * np.log(self.N) +
+ 0.5 * (self.Kw - 1) * np.log(self.D) +
+ 0.5 * self.Kz * self.Kw * np.log(self.N * self.D)
+ )
+ icls = [params[0] - penality for params in self.best_parameters]
+ return icls
+
+ def _set_similarity_parameters(self, lambda_r, S_r, lambda_c, S_c):
+ """
+ P_r and P_c : ndarray of length N and D such that
+ P_r[i] = True if we have prior knowledge about node i
+ that is there exists i' such that S_r[i, i'] > 0
+ """
+ if self.regularize_row:
+ assert lambda_r > 0
+ similarity_matrices.check_similarity_matrix(S_r)
+
+ self.lambda_r = lambda_r
+ self.S_r = general.to_sparse(S_r)
+
+ # since S is symmetric
+ self.P_r = np.asarray((self.S_r != 0.).sum(0))[0] > 0
+ else:
+ self.lambda_r = None
+ self.S_r = None
+ self.P_r = np.zeros(self.N, dtype='bool')
+
+ if self.regularize_col:
+ assert lambda_c > 0
+ similarity_matrices.check_similarity_matrix(S_c)
+
+ self.lambda_c = lambda_c
+ self.S_c = general.to_sparse(S_c)
+ self.P_c = np.asarray((self.S_c != 0.).sum(0))[0] > 0
+ else:
+ self.lambda_c = None
+ self.S_c = None
+ self.P_c = np.zeros(self.D, dtype='bool')
+
+ def _set_mixture_proportions(self):
+ self.alpha = np.exp(self.log_alpha)
+ self.beta = np.exp(self.log_beta)
+
+ def best_partition(self, mode, n_first=1):
+ """
+ returns a list of tuple of partitions
+ if 1 partition: [(part_1), ..., (part_n_first)]
+ if 2 partitions: [(row_part_1, col_part_1), ...,
+ (row_part_n_first, col_part_n_first)]
+
+ mode == 'likeelihood' returns the n_first best partitions
+ in terms of likelihood
+
+ if mode == 'consensus: hbgf' or mode == 'consensus: cspa'
+ returns 1 consensus partition
+ """
+ assert self.fitted
+
+ best_partitions = metrics.sort_partitions(
+ self.all_iter_criterions,
+ [self.all_row_partitions, self.all_col_partitions],
+ n_first
+ )
+ if mode == 'likelihood':
+ return best_partitions
+
+ if mode in ['consensus: hbgf', 'consensus: cspa']:
+ best_row_partitions = [x[0] for x in best_partitions]
+ best_col_partitions = [x[1] for x in best_partitions]
+
+ if mode == 'consensus: hbgf':
+ Z_consensus = consensus.hbgf(
+ best_row_partitions,
+ self.n_init_clustering_consensus
+ )
+ W_consensus = consensus.hbgf(
+ best_col_partitions,
+ self.n_init_clustering_consensus
+ )
+ elif mode == 'consensus: cspa':
+ Z_consensus = consensus.cspa(
+ best_row_partitions,
+ self.n_init_clustering_consensus
+ )
+ W_consensus = consensus.cspa(
+ best_col_partitions,
+ self.n_init_clustering_consensus
+ )
+ return [(Z_consensus, W_consensus)]
+
+ raise ValueError
diff --git a/dcblockmodels/models/utils/__init__.py b/dcblockmodels/models/utils/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/dcblockmodels/models/utils/absent_nodes.py b/dcblockmodels/models/utils/absent_nodes.py
new file mode 100644
index 0000000..eb6c2d9
--- /dev/null
+++ b/dcblockmodels/models/utils/absent_nodes.py
@@ -0,0 +1,202 @@
+"""class to efficiently deal with the indexes of absent and appearing nodes"""
+
+import functools
+
+import numpy as np
+from numba import njit
+
+
+class AbsentNodes:
+ """
+ class of nodes that dont interract with others at a given
+ time step, i.e. their in and out degrees are below/above threshold
+
+ both : True for dSBM, False for dLBM
+ """
+
+ def __init__(self, X, threshold, mode):
+
+ self.T = len(X)
+
+ if mode == 'SBM':
+ (n_absent_tot, n_appearing_tot,
+ absent, appearing, n_abs, n_app) = present_and_absent_nodes(X, threshold, 'both')
+
+ (self.n_absent_row_tot,
+ self.n_appearing_row_tot,
+ self.absent_row_nodes,
+ self.appearing_row_nodes,
+ self.n_absent_row_nodes,
+ self.n_appearing_row_nodes) = (n_absent_tot, n_appearing_tot,
+ absent, appearing, n_abs, n_app)
+
+ (self.n_absent_col_tot,
+ self.n_appearing_col_tot,
+ self.absent_col_nodes,
+ self.appearing_col_nodes,
+ self.n_absent_col_nodes,
+ self.n_appearing_col_nodes) = (n_absent_tot, n_appearing_tot,
+ absent, appearing, n_abs, n_app)
+
+ elif mode == 'LBM':
+ (self.n_absent_row_tot,
+ self.n_appearing_row_tot,
+ self.absent_row_nodes,
+ self.appearing_row_nodes,
+ self.n_absent_row_nodes,
+ self.n_appearing_row_nodes) = present_and_absent_nodes(X, threshold, 'row')
+
+ (self.n_absent_col_tot,
+ self.n_appearing_col_tot,
+ self.absent_col_nodes,
+ self.appearing_col_nodes,
+ self.n_absent_col_nodes,
+ self.n_appearing_col_nodes) = present_and_absent_nodes(X, threshold, 'col')
+
+ self.set_replace_vals_absent()
+
+ def __str__(self):
+ return ('Row nodes : \n'
+ f'n_absent_row_nodes = {self.n_absent_row_nodes}\n'
+ f'absent_row_nodes = {self.absent_row_nodes}\n'
+ f'n_appearing_row_nodes = {self.n_appearing_row_nodes}\n'
+ f'appearing_row_nodes = {self.appearing_row_nodes}\n\n'
+ 'Column nodes : \n'
+ f'n_absent_col_nodes = {self.n_absent_col_nodes}\n'
+ f'absent_col_nodes = {self.absent_col_nodes}\n'
+ f'n_appearing_col_nodes = {self.n_appearing_col_nodes}\n'
+ f'appearing_col_nodes = {self.appearing_col_nodes}\n')
+
+ def __repr__(self):
+ return self.__str__()
+
+ def set_replace_vals_absent(self):
+ """
+ previous timestep at which
+ node i was present
+ """
+ self.inds_prev_rows = {}
+ self.inds_prev_cols = {}
+ self.ts_absent_rows = {}
+ self.ts_absent_cols = {}
+
+ for absent_dic, inds_prev_dic, timesteps_absent in zip(
+ [self.absent_row_nodes, self.absent_col_nodes],
+ [self.inds_prev_rows, self.inds_prev_cols],
+ [self.ts_absent_rows, self.ts_absent_cols]
+ ):
+ inds_absent = functools.reduce(
+ np.union1d, (absent_dic[t] for t in range(self.T))
+ ).astype('int64')
+
+ for i in inds_absent:
+ ts_absent = [t for t in range(self.T) if i in absent_dic[t]]
+ inds_prev_dic[i] = inds_prev_absent(self.T, ts_absent)
+ timesteps_absent[i] = np.array(ts_absent)
+
+
+def present_and_absent_nodes(X, threshold, axis):
+ """
+ Assess nodes that are absent at each time, deduce present & appearing nodes at each time
+
+ Absent nodes are nodes that dont interact with others at a given time step, i.e. their in and
+ out degrees are below/above threshold.
+ """
+
+ if isinstance(X, np.ndarray):
+ T = X.shape[0]
+ if axis == 'row':
+ nodes_degrees = X.sum(2)
+ elif axis == 'col':
+ nodes_degrees = X.sum(1)
+ elif axis == 'both':
+ nodes_out_degrees = X.sum(1)
+ nodes_in_degrees = X.sum(2)
+ nodes_degrees = nodes_in_degrees + nodes_out_degrees
+ elif isinstance(X, list):
+ T = len(X)
+ if axis == 'row':
+ nodes_degrees = np.array([X[t].sum(1).A1 for t in range(T)])
+ elif axis == 'col':
+ nodes_degrees = np.array([X[t].sum(0).A1 for t in range(T)])
+ elif axis == 'both':
+ nodes_out_degrees = np.array([X[t].sum(0).A1 for t in range(T)])
+ nodes_in_degrees = np.array([X[t].sum(1).A1 for t in range(T)])
+ nodes_degrees = nodes_in_degrees + nodes_out_degrees
+
+ absent = np.where(nodes_degrees <= threshold)
+
+ absent_dic = {}
+ appearing_dic = {0: np.array([], dtype='int')}
+
+ n_absent_tot, n_appearing_tot = 0, 0
+ n_absent_dic, n_appearing_dic = {}, {}
+ for t in range(T):
+ ind0_t = np.where(absent[0] == t)
+ absent_dic[t] = absent[1][ind0_t]
+ n_absent_tot += len(absent_dic[t])
+ if t >= 1:
+ appearing_t = []
+ for i in absent_dic[t - 1]:
+ if i not in absent_dic[t]:
+ appearing_t.append(i)
+ n_appearing_tot += 1
+ appearing_dic[t] = np.array(appearing_t, dtype='int')
+
+ n_absent_dic[t] = len(absent_dic[t])
+ n_appearing_dic[t] = len(appearing_dic[t])
+
+ return (n_absent_tot, n_appearing_tot, absent_dic,
+ appearing_dic, n_absent_dic, n_appearing_dic)
+
+
+def replace_vals_absent(arr, inds_prev, ts_absent):
+ """
+ Replace values for an absent node by previous values at which it was present
+ """
+ for i, ts_absent_i, inds_prev_i in zip(
+ ts_absent.keys(),
+ ts_absent.values(),
+ inds_prev.values()
+ ):
+ arr[ts_absent_i, i] = arr[inds_prev_i, i]
+
+
+@njit
+def inds_prev_absent(T, inds):
+ """Computes the timesteps used to replace the values
+ of the smoothed arrays for absent nodes
+
+ Parameters
+ ----------
+ T : int
+ number timesteps
+ inds : list
+ indexes of the timesteps at which a a given node is absent
+
+ Returns
+ -------
+ np.ndarray
+ array of same size as inds that maps each timestep
+ at which a a given node is absent to the previous timestep
+ at which the node was present
+ """
+ inds_prev = []
+ i_prev = -1
+ len_inds = 0
+ for i in range(T):
+ if i not in inds:
+ i_next = i
+ break
+ for i in range(T):
+ if i in inds:
+ len_inds += 1
+ if i_prev != -1:
+ inds_prev.append(i_prev)
+ else:
+ inds_prev.append(i_next)
+ else:
+ i_prev = i
+ if len_inds == T:
+ break
+ return np.array(inds_prev, dtype=np.int64)
diff --git a/dcblockmodels/models/utils/consensus.py b/dcblockmodels/models/utils/consensus.py
new file mode 100644
index 0000000..b154e34
--- /dev/null
+++ b/dcblockmodels/models/utils/consensus.py
@@ -0,0 +1,63 @@
+"""Different kinds of consensus clustering based on a series of partitions of the data"""
+
+import numpy as np
+from sklearn.cluster import SpectralCoclustering, SpectralClustering
+
+
+def build_partition_matrix(partition_list):
+ """
+ partition_list : list of hard partitions Z np.array of shape (T, N)
+ returns : (N*T, sum_r K_r) np.array
+ K_r : nb of clusters in partition r
+ """
+ partitions = [(p.flatten() == k)[:, None]
+ for p in partition_list
+ for k in np.unique(p)]
+ return np.concatenate(partitions, axis=1).astype(int)
+
+
+def hbgf(partition_list, n_init_clustering_consensus):
+ """
+ consensus with bipartite graph W_{ij} = 1 if
+ item i is in cluster j. Then applies co-clustering
+ """
+ H = build_partition_matrix(partition_list)
+ K = np.array([np.unique(Z).shape[0] for Z in partition_list]).max()
+
+ spcl = SpectralCoclustering(
+ n_clusters=K,
+ svd_method='randomized',
+ n_svd_vecs=None,
+ mini_batch=False,
+ init='k-means++',
+ n_init=n_init_clustering_consensus,
+ n_jobs=-1,
+ random_state=None
+ )
+ spcl.fit(H)
+ return spcl.row_labels_
+
+
+def cspa(partition_list, n_init_clustering_consensus):
+ """
+ item based consensus : builds matrix C,
+ where c_{ij} is the number of times i and j
+ have been put in the same cluster in the partitions
+ in partition_list. Then clusters the items
+ using spectral clustering on C seen as an affinity matrix
+ """
+ H = build_partition_matrix(partition_list)
+ K = np.array([np.unique(Z).shape[0] for Z in partition_list]).max()
+ R = len(partition_list)
+ C = (H.dot(H.T)) / R
+ spcl = SpectralClustering(
+ n_clusters=K,
+ eigen_solver=None,
+ random_state=None,
+ n_init=n_init_clustering_consensus,
+ affinity='precomputed',
+ assign_labels='kmeans',
+ n_jobs=-1
+ )
+ spcl.fit(C)
+ return spcl.labels_
diff --git a/dcblockmodels/models/utils/e_step.py b/dcblockmodels/models/utils/e_step.py
new file mode 100644
index 0000000..cb9b50b
--- /dev/null
+++ b/dcblockmodels/models/utils/e_step.py
@@ -0,0 +1,437 @@
+"""E-step-related general functions"""
+
+import numpy as np
+
+from . import general
+
+nax = np.newaxis
+
+
+def log_density_t(
+ X_t, gamma, mode,
+ model_type, estimated_margins, dtype, min_float,
+ Z=None, W=None,
+ mu_t=None, nu_t=None
+):
+ """
+ deals with static and dynamic cases
+ """
+ log_gamma = np.log(gamma + min_float)
+ if mode == 'row':
+ X_W = (X_t @ W).astype(dtype)
+ dens = X_W @ log_gamma.T
+ if model_type in ['without_margins', 'without_margins+free_gamma']:
+ W_ = general.get_class_counts(W).astype(dtype)
+ np.add(dens, - (gamma @ W_.T)[nax, :], out=dens)
+ elif ((model_type == 'with_margins' and estimated_margins)
+ or model_type == 'free_margins+constant_gamma'):
+ nu_t_W = (nu_t @ W).astype(dtype)
+ np.add(dens, - mu_t[:, nax] * (gamma @ nu_t_W.T)[nax, :], out=dens)
+ X_red = X_W
+
+ elif mode == 'col':
+ X_Z = (X_t.T @ Z).astype(dtype)
+ dens = X_Z @ log_gamma
+ if model_type in ['without_margins', 'without_margins+free_gamma']:
+ Z_ = general.get_class_counts(Z).astype(dtype)
+ np.add(dens, - (Z_ @ gamma)[nax, :], out=dens)
+ elif ((model_type == 'with_margins' and estimated_margins)
+ or model_type == 'free_margins+constant_gamma'):
+ mu_t_Z = (mu_t @ Z).astype(dtype)
+ np.add(dens, - nu_t[:, nax] * (mu_t_Z @ gamma)[nax, :], out=dens)
+ X_red = X_Z
+ return dens, X_red
+
+
+def _get_ZW_t_app(
+ dens,
+ em_type,
+ ind_appearing_nodes_t,
+ qzw_tp1,
+ ZW_tm1,
+ ZW_tp1,
+ log_alpha_beta,
+ log_pi_rho,
+ min_float,
+ min_proba
+):
+ appearing = len(ind_appearing_nodes_t) > 0
+ if not appearing:
+ return None
+
+ ND = dens.shape[0]
+ dens_app = dens[ind_appearing_nodes_t, :]
+ if em_type == 'VEM':
+ log_prop_app = _get_log_prop_dynamic_vem(
+ 'appearing',
+ qzw_tp1,
+ ND,
+ log_pi_rho,
+ log_alpha_beta,
+ min_float
+ )
+ ZW_app = dens_app + log_prop_app[ind_appearing_nodes_t, :]
+ ZW_app = _normalize_posterior(
+ ZW_app,
+ min_proba,
+ ind_absent_nodes_t=None,
+ dynamic=False
+ )
+ elif em_type == 'CEM':
+ log_prop_app = _get_log_prop_dynamic_cem(
+ ZW_tm1, ZW_tp1,
+ log_pi_rho, log_alpha_beta
+ )
+ ZW_app = dens_app + log_prop_app[ind_appearing_nodes_t, :]
+ ZW_app = _ce_step_static(ZW_app)
+ return ZW_app
+
+
+def _get_log_prop_dynamic_vem(
+ mode,
+ qzw_tp1,
+ ND,
+ log_pi_rho,
+ log_alpha_beta,
+ min_float
+):
+ """
+ """
+ if qzw_tp1 is not None:
+ # dkl_tp1[i, k]
+ dkl_tp1 = (qzw_tp1 * (np.log(qzw_tp1 + min_float) - log_pi_rho[nax, :, :])).sum(2)
+
+ if mode in ['appearing', 't0']:
+ log_prop = log_alpha_beta[nax, :] - dkl_tp1
+ else:
+ log_prop = log_pi_rho[nax, :, :] - dkl_tp1[:, :, nax]
+ else:
+ # case t = T
+ if mode in ['appearing', 't0']:
+ log_prop = np.tile(log_alpha_beta[nax, :], (ND, 1))
+ else:
+ log_prop = log_pi_rho[nax, :, :]
+ return log_prop
+
+
+def e_step_t_dynamic_vem(
+ mode, first_ts, model_type, estimated_margins,
+ X_t, gamma,
+ dtype, min_float,
+ qzw_tp1, log_pi_rho, log_alpha_beta,
+ ind_appearing_nodes_t, ind_absent_nodes_t,
+ min_proba, ZW_tm1,
+ Z=None, W=None, nu_t=None, mu_t=None
+):
+ """
+ E step at time t on either column or row for VEM
+ """
+ dens, X_red = log_density_t(
+ X_t, gamma, mode,
+ model_type, estimated_margins,
+ dtype, min_float,
+ Z=Z, W=W, nu_t=nu_t, mu_t=mu_t
+ )
+ ZW_t_app = _get_ZW_t_app(
+ dens,
+ 'VEM',
+ ind_appearing_nodes_t,
+ qzw_tp1,
+ None,
+ None,
+ log_alpha_beta,
+ log_pi_rho,
+ min_float,
+ min_proba
+ )
+ log_prop_mode = 't0' if first_ts else 'regular'
+ ND = X_red.shape[0]
+
+ log_prop = _get_log_prop_dynamic_vem(
+ log_prop_mode,
+ qzw_tp1,
+ ND,
+ log_pi_rho, log_alpha_beta,
+ min_float
+ )
+ if first_ts:
+ ZW = log_prop + dens
+ ZW = _normalize_posterior(
+ ZW, min_proba,
+ ind_absent_nodes_t=ind_absent_nodes_t,
+ dynamic=False
+ )
+ else:
+ qzw_t = log_prop + dens[:, nax, :]
+ qzw_t = _normalize_posterior(
+ qzw_t, min_proba,
+ ind_absent_nodes_t=ind_absent_nodes_t,
+ dynamic=True
+ )
+ ZW = update_ZW_t_vem(
+ ZW_tm1,
+ qzw_t,
+ ind_appearing_nodes_t,
+ ZW_t_app,
+ ind_absent_nodes_t,
+ min_proba
+ )
+ return ZW, X_red
+
+
+def e_step_t_dynamic_cem(
+ mode, model_type, estimated_margins,
+ X_t, gamma,
+ ZW_tm1, ZW_tp1,
+ dtype, min_float,
+ log_pi_rho, log_alpha_beta,
+ ind_appearing_nodes_t, ind_absent_nodes_t,
+ Z=None, W=None, nu_t=None, mu_t=None
+):
+ """
+ E step at time t on either column or row for CEM
+ """
+ dens, X_red = log_density_t(
+ X_t, gamma, mode,
+ model_type, estimated_margins,
+ dtype, min_float,
+ Z=Z, W=W, nu_t=nu_t, mu_t=mu_t
+ )
+ ZW_t_app = _get_ZW_t_app(
+ dens,
+ 'CEM',
+ ind_appearing_nodes_t,
+ None,
+ ZW_tm1,
+ ZW_tp1,
+ log_alpha_beta,
+ log_pi_rho,
+ min_float,
+ None
+ )
+ log_prop = _get_log_prop_dynamic_cem(
+ ZW_tm1, ZW_tp1,
+ log_pi_rho, log_alpha_beta
+ )
+ ZW_t = log_prop + dens
+ ZW_t = _ce_step_dynamic_t(
+ ZW_t,
+ ind_appearing_nodes_t,
+ ZW_t_app,
+ ind_absent_nodes_t
+ )
+ return ZW_t, X_red
+
+
+def update_ZW_t_vem(
+ ZW_tm1,
+ qzw_t,
+ ind_appearing_t,
+ ZW_t_app,
+ ind_absent_nodes_t,
+ min_proba
+):
+ """
+ marginal posterior probas
+ ZW, qzw and ZW_t_app must be proba not log proba
+ """
+ # here ZW_t are the posterior probabilities
+ # at the previous iteration of EM
+ with np.errstate(under='ignore'):
+ ZW_t = (ZW_tm1[:, :, nax] * qzw_t).sum(axis=1)
+
+ # we replace the previous calculations for
+ # the appearing nodes with ZW_t_app
+ if ZW_t_app is not None:
+ ZW_t[ind_appearing_t] = ZW_t_app
+
+ # to avoid numerical issues
+ ZW_t[ind_absent_nodes_t] = 1.
+
+ # normalize
+ np.divide(ZW_t, ZW_t.sum(1, keepdims=True), ZW_t)
+ np.clip(ZW_t, min_proba, None, ZW_t)
+ np.divide(ZW_t, ZW_t.sum(1, keepdims=True), ZW_t)
+
+ # set zero proba for absent nodes
+ # so that when we compute Z.T @ X @ W
+ # the absent nodes are not counted
+ # which is necessary to correctly update
+ # gamma (in case thr_absent_nodes != 0)
+ # and to compute the complete data log
+ # likelihood
+ ZW_t[ind_absent_nodes_t] = 0.
+ return ZW_t
+
+
+def e_step_static(
+ log_alpha_beta,
+ regularize,
+ regularization_mode,
+ lambda_, S, P,
+ damping_factor,
+ em_type,
+ X, gamma, mode,
+ model_type, estimated_margins,
+ dtype, min_float, min_proba,
+ Z, W, nu, mu
+):
+ """
+ E step on either column or row in static mode
+ """
+ ZW = Z if mode == 'row' else W
+ log_prop = _get_log_prop_static(
+ log_alpha_beta,
+ regularize,
+ regularization_mode,
+ lambda_, S, P, ZW
+ )
+ dens, X_red = log_density_t(
+ X, gamma, mode,
+ model_type, estimated_margins,
+ dtype, min_float,
+ Z=Z, W=W, nu_t=nu, mu_t=mu
+ )
+ ZW_new = log_prop + dens
+
+ if em_type == 'VEM':
+ ZW_new = _normalize_posterior(
+ ZW_new,
+ min_proba,
+ ind_absent_nodes_t=None,
+ dynamic=False
+ )
+ ZW_new = _apply_damping(ZW, ZW_new, damping_factor)
+ elif em_type == 'CEM':
+ ZW_new = _ce_step_static(ZW_new)
+ ZW_new = general.to_sparse(ZW_new)
+
+ return ZW_new, X_red
+
+
+def _normalize_posterior(
+ qzw_ZW, min_proba,
+ ind_absent_nodes_t=None, dynamic=False
+):
+ """
+ qzw_ZW : could be qzw or ZW
+ """
+ axis = 2 if dynamic else 1
+ max_log = np.max(qzw_ZW, axis=axis, keepdims=True)
+ with np.errstate(divide='ignore', under='ignore'):
+ np.exp(qzw_ZW - max_log, qzw_ZW)
+ np.divide(qzw_ZW, qzw_ZW.sum(axis=axis, keepdims=True), qzw_ZW)
+
+ # only fill un informative values
+ # for absent nodes to avoid numerical isssues
+ if ind_absent_nodes_t is not None:
+ qzw_ZW[ind_absent_nodes_t] = 1.
+
+ np.clip(qzw_ZW, min_proba, None, qzw_ZW)
+ with np.errstate(divide='ignore', under='ignore'):
+ np.divide(qzw_ZW, qzw_ZW.sum(axis=axis, keepdims=True), qzw_ZW)
+
+ return qzw_ZW
+
+
+def _apply_damping(ZW_old, ZW_new, damping_factor):
+ if damping_factor is None:
+ return ZW_new
+ return damping_factor * ZW_new + (1. - damping_factor) * ZW_old
+
+
+def _get_log_prop_static(
+ log_alpha_beta,
+ regularize,
+ regularization_mode,
+ lambda_, S, P, ZW
+):
+ """
+ returns a ND x Kzw array containing the log proportions
+ that is, for instance for the rows, log_alpha[i, k] if there
+ is no regularization, or S_r[i].dot(Z[:, k]) + log_alpha[i, k]
+ if there is regularization
+ """
+ if regularize:
+ reg = general.to_dense(lambda_ * (S @ ZW))
+ if regularization_mode == 'mixture':
+ not_P = ~ P
+ log_prop = not_P[:, nax] * log_alpha_beta[nax, :] + P[:, nax] * reg
+ elif regularization_mode == 'all':
+ log_prop = log_alpha_beta[nax, :] + reg
+ else:
+ log_prop = log_alpha_beta[nax, :]
+
+ return log_prop
+
+
+def _ce_step_dynamic_t(crit, ind_appearing_t, ZW_t_app, ind_absent_nodes_t):
+ """
+ Computes the Classification E step in dynamic
+ where z_i = argmax_k crit
+ """
+ indexes = np.asarray(crit.argmax(1)).T
+ ZW_t_new = np.zeros_like(crit, dtype='bool')
+ ND = crit.shape[0]
+ ZW_t_new[np.arange(ND), indexes] = True
+
+ # we replace the previous calculations for
+ # the appearing nodes with ZW_t_app
+ if ZW_t_app is not None:
+ ZW_t_new[ind_appearing_t] = ZW_t_app
+
+ # put constant proba for absent nodes
+ # for we do not care
+ ZW_t_new[ind_absent_nodes_t] = False
+
+ return ZW_t_new
+
+
+def _ce_step_static(crit):
+ """
+ Computes the Classification E step in static
+ where z_i = argmax_k crit
+ """
+ indexes = np.asarray(crit.argmax(1)).T
+ ZW_t_new = np.zeros_like(crit, dtype='bool')
+ ZW_t_new[np.arange(crit.shape[0]), indexes] = True
+ return ZW_t_new
+
+
+def _get_log_prop_dynamic_cem(
+ ZW_tm1, ZW_tp1,
+ log_pi_rho, log_alpha_beta
+):
+ """
+ """
+ ND = ZW_tm1.shape[0] if ZW_tm1 is not None else ZW_tp1.shape[0]
+ log_pi_rho = np.tile(log_pi_rho[nax], (ND, 1, 1))
+ arange = np.arange(ND)
+
+ if ZW_tm1 is not None:
+ ind_tm1 = ZW_tm1.argmax(1)
+ log_pi_prev = log_pi_rho[arange, ind_tm1, :]
+ log_prop = log_pi_prev
+ else:
+ # case t == 1
+ log_prop = log_alpha_beta[nax]
+
+ if ZW_tp1 is not None:
+ ind_tp1 = ZW_tp1.argmax(1)
+ log_pi_next = log_pi_rho[arange, :, ind_tp1]
+ log_prop = log_prop + log_pi_next
+ else:
+ # case t=T
+ pass
+
+ return log_prop
+
+
+def correct_ZW(ZW, min_proba):
+ """
+ ZW must be a proba (not a log proba)
+ """
+ np.clip(ZW, min_proba, None, ZW)
+ with np.errstate(divide='ignore', under='ignore'):
+ np.divide(ZW, ZW.sum(axis=2, keepdims=True), ZW)
+ return ZW
diff --git a/dcblockmodels/models/utils/general.py b/dcblockmodels/models/utils/general.py
new file mode 100644
index 0000000..7a0d42a
--- /dev/null
+++ b/dcblockmodels/models/utils/general.py
@@ -0,0 +1,107 @@
+"""General functions that can be used in all modules"""
+
+import pickle
+
+import numpy as np
+import scipy as sp
+
+
+def get_class_counts(ZW):
+ """Get number of nodes in each cluster"""
+ res = ZW.sum(0)
+ if sp.sparse.issparse(ZW):
+ res = res.A1
+ return res
+
+
+def encode(ZW, Kzw):
+ """Encode cluster membership in a boolean matrix"""
+ ND = ZW.shape[0]
+ ZW_ = np.zeros((ND, Kzw), dtype='bool')
+ ZW_[np.arange(ND), ZW] = True
+ return ZW_
+
+
+def load_model(path):
+ """Load a given model from its directory"""
+ with open(path + '/model.pickle', 'rb') as f:
+ model = pickle.load(f)
+ return model
+
+
+# TODO Only usable on dynamic X, check if keep specific in dlbm.py or generalize and keep here.
+def check_X(X, is_graph, self_loops, directed=True):
+ """
+ is_graph: square adjacency matrix?
+ directed: whether the graph is directed or not
+ this requires a symetric adjacency matrix.
+ Only useful for (d)SBM
+ """
+ T = len(X)
+
+ for t in range(T):
+ X_dens_t = to_dense(X[t])
+ assert (X_dens_t >= 0).all()
+ assert (X_dens_t > 0).any()
+ # assert X_dens_t.dtype in [np.int32, np.int64]
+ assert X_dens_t.ndim == 2
+
+ if is_graph:
+ assert X_dens_t.shape[1] == X_dens_t.shape[0]
+
+ if not self_loops:
+ assert (np.diag(X_dens_t) == 0).all()
+
+ if not directed:
+ assert np.array_equal(X_dens_t, symmetrize(X_dens_t))
+
+
+def symmetrize(a):
+ """Symmetrize a matrix by keeping only triangular part and copying it on lower triangular"""
+ a = np.tril(a)
+ return a + a.T - np.diag(a.diagonal())
+
+
+def get_delta(old, new):
+ """
+ returns the criterion used to determine
+ if the model has converged from the
+ old and new likelihoods
+ """
+ return abs((new - old) / new)
+
+
+def to_dense(mat):
+ """From sp.sparse sparse matrix format to dense Numpy arrays"""
+ if sp.sparse.issparse(mat):
+ return mat.toarray()
+
+ return mat
+
+
+def to_sparse(mat):
+ """From dense Numpy arrays to sp.sparse sparse matrix format"""
+ if not sp.sparse.issparse(mat):
+ return sp.sparse.csr_matrix(mat)
+
+ return mat
+
+
+def compute_mixture_exact_icl(ZW, Kzw, ND):
+ """Compute ICL for either lines or columns"""
+ from scipy.special import loggamma
+ nax = np.newaxis
+
+ cst = (
+ loggamma(.5 * Kzw) * (1 - Kzw) -
+ .5 * Kzw * (Kzw + 1) * np.log(np.pi) -
+ loggamma(.5 * Kzw + ND)
+ )
+
+ icl_zw_t1 = loggamma(ZW[0].sum(0) + .5).sum()
+
+ trans = (ZW[:-1, :, :, nax] * ZW[1:, :, nax, :]).sum((0, 1))
+ trans_ = trans.sum(1)
+ icl_zw_t2 = - loggamma(trans_ + .5 * Kzw).sum() + loggamma(trans + .5).sum()
+
+ return cst + icl_zw_t1 + icl_zw_t2
diff --git a/dcblockmodels/models/utils/init.py b/dcblockmodels/models/utils/init.py
new file mode 100644
index 0000000..62b2893
--- /dev/null
+++ b/dcblockmodels/models/utils/init.py
@@ -0,0 +1,239 @@
+"""Functions linked to initialization of the model"""
+
+import numpy as np
+from sklearn.cluster import KMeans
+try:
+ from spherecluster import SphericalKMeans
+except ImportError:
+ pass
+
+from . import general
+
+nax = np.newaxis
+
+
+def init_ZW0(Kzw, partition, min_proba, dtype):
+ """
+ Initializes Z or W with a static partition that is
+ not encoded (i.e. partition[i] = k).
+ Can be used in the static case or in dynamic CEM,
+ where we do not deal with transitions probabilites qzw
+ """
+ ND = partition.shape[0]
+ ZW0 = np.full((ND, Kzw), min_proba, dtype=dtype)
+ if min_proba is not None:
+ val = 1. - min_proba
+ else:
+ val = True
+ ZW0[np.arange(ND), partition] = val
+ return ZW0
+
+
+def init_qzw(Kzw, partition, min_proba, dtype):
+ """
+ Initializes qz or qw with a dynamic partition that is
+ not encoded (i.e. partition[t, i] = k).
+ Can be used in the dynamic case.
+ """
+ T, ND = partition.shape
+ qzw = np.full((T, ND, Kzw, Kzw), min_proba, dtype=dtype)
+ arr_ND = np.arange(ND)
+ for t in range(1, T):
+ qzw[t, arr_ND, :, partition[t]] = 1. - min_proba
+ return qzw
+
+
+def apply_perturbation_to_init_clustering(
+ init_partition,
+ random_state,
+ node_perturbation_rate,
+ cluster_perturbation_rate=0.,
+ dynamic=False
+):
+ """Apply some noise to the initial partitions"""
+ res_init_partition = init_partition.copy()
+
+ if node_perturbation_rate > 0.:
+ if dynamic:
+ T, ND = init_partition.shape
+ Kzw = np.unique(init_partition.flatten()).shape[0]
+ n_nodes_shuffled = int(node_perturbation_rate * ND)
+ for t in range(T):
+ nodes_replaced = random_state.choice(
+ ND, size=n_nodes_shuffled, replace=False
+ )
+ res_init_partition[t, nodes_replaced] = random_state.choice(
+ Kzw, size=n_nodes_shuffled
+ )
+ if cluster_perturbation_rate > 0.:
+ res_init_partition[t] = _apply_cluster_perturbation(
+ res_init_partition[t],
+ Kzw,
+ random_state,
+ cluster_perturbation_rate
+ )
+ else:
+ ND = init_partition.shape[0]
+ Kzw = np.unique(init_partition.flatten()).shape[0]
+ res_init_partition = init_partition.copy()
+ if node_perturbation_rate > 0.:
+ n_nodes_shuffled = int(node_perturbation_rate * ND)
+ nodes_replaced = random_state.choice(
+ ND, size=n_nodes_shuffled, replace=False
+ )
+ res_init_partition[nodes_replaced] = random_state.choice(
+ Kzw, size=n_nodes_shuffled
+ )
+ return res_init_partition
+
+
+def _apply_cluster_perturbation(
+ init_partition,
+ Kzw,
+ random_state,
+ cluster_perturbation_rate
+):
+ def permute_clusters(Z, k, l):
+ Z[Z == k] = -1
+ Z[Z == l] = k
+ Z[Z == -1] = l
+ return Z
+
+ partition = init_partition.copy()
+
+ seen = []
+ u = random_state.rand()
+ while u <= cluster_perturbation_rate:
+ not_permutated_clusters = np.setdiff1d(np.arange(Kzw), np.array(seen))
+ if not_permutated_clusters.shape[0] >= 2:
+ k, k_ = random_state.choice(not_permutated_clusters, size=2, replace=False)
+ partition = permute_clusters(partition, k, k_)
+ seen += [k, k_]
+ u = random_state.rand()
+ else:
+ break
+ return partition
+
+
+def get_X_init(X, mode, absent_nodes=None):
+ """
+ returns the data matrix used for the initialization
+ of the clusters. Could be dynamic or not and provides
+ row or col concatenation for init of Z or W. The profiles
+ X[t, i, :] (or X[t, :, j]) of absent nodes at time t are
+ replaced with their mean profile over time to avoid
+ initialization issues due to absent nodes.
+ """
+ dynamic = True if isinstance(X, list) else (X.ndim == 3)
+
+ if not dynamic:
+ if mode == 'row':
+ res = X
+ elif mode == 'col':
+ res = X.T
+ else:
+ T = len(X)
+ if isinstance(X, list):
+ # when X is a list of sparse matrices
+ X_ = np.concatenate([
+ general.to_dense(X[t])[nax] for t in range(T)],
+ axis=0
+ )
+ else:
+ X_ = X.copy()
+
+ # replaces row os cols of absent nodes by
+ # their mean values
+ # then stacks the columns/rows
+ if mode == 'row':
+ if absent_nodes is not None:
+ X_mean = X_.mean(0)
+ for t in range(T):
+ X_[t, absent_nodes[t], :] = X_mean[absent_nodes[t], :]
+ res = np.hstack([X_[t] for t in range(T)])
+ elif mode == 'col':
+ if absent_nodes is not None:
+ X_mean = X_.mean(0)
+ for t in range(T):
+ X_[t, :, absent_nodes[t]] = X_mean[:, absent_nodes[t]].T
+ res = np.vstack([X_[t] for t in range(T)]).T
+ return res
+
+
+def _random_init(Kzw, ND, random_state):
+ return random_state.randint(0, Kzw, size=(ND))
+
+
+def _given_partition_init(Kzw, given_partition):
+ clusters = np.unique(given_partition.flatten())
+ assert clusters.shape[0] <= Kzw
+ assert np.in1d(clusters, np.arange(Kzw)).all()
+ # absent nodes can not be directly put in cluster K
+ # : put it instead in a random cluster for given_partition
+ # it will then be managed with threshold_absent_nodes
+ return given_partition
+
+
+def _kmeans_init(X, K, n_init, random_state, n_jobs):
+ clustering = KMeans(
+ n_clusters=K,
+ random_state=random_state,
+ n_init=n_init)
+ return clustering.fit(X).labels_
+
+
+def _skmeans_init(X, K, n_init, random_state, n_jobs):
+ clustering = SphericalKMeans(
+ n_clusters=K,
+ init="k-means++",
+ n_init=n_init,
+ max_iter=150,
+ tol=1e-6,
+ n_jobs=n_jobs,
+ verbose=0,
+ random_state=random_state,
+ copy_x=True,
+ normalize=True)
+ return clustering.fit(X).labels_
+
+
+def get_init_partition(
+ X_init, Kzw, init_type, T, random_state, n_jobs,
+ n_init, given_partition):
+ """
+ Returns the row or col partition used
+ for the first initialization of the algo.
+ init_partition is not one-hot-encoded: ie
+ init_partition[t, i] = k
+ Noise will be apllied later
+ """
+ ND = X_init.shape[0]
+ if init_type == 'random':
+ part = _random_init(Kzw, ND, random_state)
+ elif init_type == 'kmeans':
+ part = _kmeans_init(X_init, Kzw, n_init, random_state, n_jobs)
+ elif init_type == 'skmeans':
+ part = _skmeans_init(X_init, Kzw, n_init, random_state, n_jobs)
+ elif init_type == 'given':
+ part = _given_partition_init(Kzw, given_partition)
+ else:
+ raise ValueError
+ if T > 1:
+ part = np.concatenate(
+ [part[nax] for _ in range(T)],
+ axis=0
+ )
+ return part
+
+
+def pi_rho_update_mask(T, ND, absent_nodes, appearing_nodes):
+ """
+ returns a (T-1) x ND boolean mask, set to False
+ for absent nodes and appearing nodes.
+ """
+ mask = np.ones((T, ND), dtype='bool')
+ for t in range(T):
+ mask[t][absent_nodes[t]] = False
+ mask[t][appearing_nodes[t]] = False
+ mask = mask[1:]
+ return mask
diff --git a/dcblockmodels/models/utils/m_step.py b/dcblockmodels/models/utils/m_step.py
new file mode 100644
index 0000000..1fe5cff
--- /dev/null
+++ b/dcblockmodels/models/utils/m_step.py
@@ -0,0 +1,364 @@
+"""M-step-related general functions"""
+
+import numpy as np
+
+from . import general
+
+nax = np.newaxis
+
+
+def update_gamma(
+ mode, estimated_margins,
+ X, Z, W, X_red, mu, nu,
+ dtype, model_type
+):
+ """
+ In the static case or for parameters at a given time step
+
+ Note that here, mu and nu can be the estimated margins or
+ the non estimated margins xi_, x_j
+
+ X_red is the reduced matrix (X_W or X_Z), that has been computed
+ in the e step (row or col), where row or col is
+ determined by the 'mode' keyword.
+
+ For the first m step, at initialization,
+ there has not been any e step
+ previously, so the computation is different, which
+ is indicated by mode == 'init'
+ """
+ # computes the numerator and denominator of gamma
+ if mode == 'init':
+ X_kl = general.to_dense(Z.T @ X @ W)
+ if model_type == 'with_margins':
+ mu_Z = Z.T @ mu
+ nu_W = W.T @ nu
+ margins_kl = np.outer(mu_Z, nu_W).astype(dtype)
+
+ elif mode == 'row':
+ X_W = X_red
+ X_kl = general.to_dense(Z.T @ X_W)
+ if model_type == 'with_margins':
+ mu_Z = Z.T @ mu
+ if estimated_margins:
+ nu_W = W.T @ nu
+ else:
+ nu_W = X_W.sum(0)
+ nu_W = general.to_dense(nu_W)
+ margins_kl = np.outer(mu_Z, nu_W)
+ elif mode == 'col':
+ X_Z = X_red
+ X_kl = general.to_dense(X_Z.T @ W)
+ if model_type == 'with_margins':
+ nu_W = W.T @ nu
+ if estimated_margins:
+ mu_Z = Z.T @ mu
+ else:
+ mu_Z = X_Z.sum(0)
+ mu_Z = general.to_dense(mu_Z)
+ margins_kl = np.outer(mu_Z, nu_W)
+
+ if model_type == 'without_margins':
+ Z_ = general.get_class_counts(Z)
+ W_ = general.get_class_counts(W)
+ ZW_kl = np.outer(Z_, W_)
+ den_gamma = ZW_kl
+ elif model_type == 'with_margins':
+ den_gamma = margins_kl
+
+ return X_kl, den_gamma
+
+
+def get_gamma(num_gamma, den_gamma, em_type, min_float, min_gamma):
+ """Get usable gamma depending on EM type, with minimum values for denominator & gamma itself"""
+ gamma = num_gamma / (den_gamma + min_float)
+ if em_type == 'CEM':
+ # very important in CEM to avoid empty clusters
+ np.clip(gamma, min_gamma, None, gamma)
+ return gamma
+
+
+def get_denominator_mu(Z, W, nu, gamma):
+ """Compute denominator of mu"""
+ nu_W = W.T @ nu
+ den_mu = Z @ gamma @ nu_W
+ return den_mu
+
+
+def get_denominator_nu(Z, W, mu, gamma):
+ """Compute denominator of nu"""
+ mu_Z = Z.T @ mu
+ den_nu = W @ gamma.T @ mu_Z
+ return den_nu
+
+
+def update_pi_rho(ZW, qzw, prior, mask):
+ """
+ Update pi/rho in VEM mode
+ """
+ res = prior - 1. + (
+ ZW[:-1, :, :, nax] *
+ qzw[1:, :, :, :] *
+ mask[:, :, nax, nax]
+ ).sum((0, 1))
+ return res
+
+
+def correct_pi_rho(mat, min_proba, min_float):
+ """
+ for pi/rho
+ """
+ np.divide(mat, mat.sum(axis=1, keepdims=True) + min_float, mat)
+ np.clip(mat, min_proba, None, mat)
+ np.divide(mat, mat.sum(axis=1, keepdims=True) + min_float, mat)
+ return mat
+
+
+def update_alpha_beta_dynamic(
+ ZW, n_absent_nodes, appearing_nodes,
+ absent_nodes_t0, min_float, min_proba, dtype
+):
+ """
+ Update alpha/beta in dynamic mode
+ """
+ if n_absent_nodes > 0:
+ present_t0 = np.setdiff1d(
+ np.arange(ZW.shape[1]),
+ absent_nodes_t0
+ )
+ res = ZW[0][present_t0].sum(0).astype(dtype)
+
+ T = len(appearing_nodes)
+ for t in range(1, T):
+ np.add(res, ZW[t][appearing_nodes[t]].sum(0).astype(dtype), res)
+ else:
+ res = ZW[0].sum(0).astype(dtype)
+
+ np.divide(res, res.sum(), res)
+ np.clip(res, min_proba, None, res)
+ np.divide(res, res.sum(), res)
+ return np.log(res + min_float)
+
+
+def update_pi_rho_cem(ZW, prior, mask, dtype):
+ """
+ Update pi/rho in CEM mode
+ """
+ pi_rho = (prior - 1.) + (
+ ZW[:-1, :, :, nax] *
+ ZW[1:, :, nax, :] *
+ mask[:, :, nax, nax]
+ ).sum((0, 1)).astype(dtype)
+ return pi_rho
+
+
+def smoothing_matrix(T, tau, dtype):
+ """
+ Return the weights used for temporal smoothing
+ """
+ if tau == 0:
+ return np.ones((T, T), dtype=dtype) / T
+ if tau == 1:
+ return np.eye((T), dtype=dtype)
+
+ arr = np.arange(T)
+ t = arr[:, nax]
+ t_ = arr[nax, :]
+ delta = (t_ - t).astype(dtype)
+
+ with np.errstate(under='ignore'):
+ # W = np.exp(- (tau / (1 - tau)) * delta ** 2)
+ W = np.exp(- (1. / ((1. / tau) - 1.)) * delta ** 2)
+ np.divide(W, W.sum(axis=1, keepdims=True), W)
+ return W
+
+
+def compute_Lc_static_density(
+ model_type, estimated_margins,
+ gamma, mu, nu, Xi_, X_j,
+ X_kl, min_float
+):
+ """
+ Computes the terms in gamma, mu,nu in complete data log likelihood
+ """
+ Lc = 0.
+ log_gamma = np.log(gamma + min_float)
+ if model_type == 'without_margins':
+ Lc += (X_kl * log_gamma).sum()
+ elif model_type == 'with_margins':
+ if estimated_margins:
+ log_mu = np.log(mu + min_float)
+ log_nu = np.log(nu + min_float)
+ Lc += (
+ (X_kl * log_gamma).sum() +
+ Xi_.dot(log_mu) + X_j.dot(log_nu)
+ )
+ else:
+ Lc += (X_kl * log_gamma).sum()
+ return Lc
+
+
+def complete_data_loglikelihood_alpha_beta(
+ log_alpha_beta, ZW,
+ absent_nodes_t0,
+ appearing_nodes
+):
+ """
+ Computes the terms in alpha or beta of the log likelihood of the expected complete data
+ """
+ T = len(ZW)
+ ND = ZW[0].shape[0]
+
+ present_t0 = np.setdiff1d(
+ np.arange(ND),
+ absent_nodes_t0
+ )
+ nzw = ZW[0][present_t0].sum(0)
+
+ for t in range(T):
+ np.add(nzw, ZW[t][appearing_nodes[t], :].sum(), out=nzw)
+
+ return nzw.dot(log_alpha_beta)
+
+
+def complete_data_loglikelihood_pi_rho(
+ em_type,
+ log_pi_rho,
+ ZW, qzw,
+ mask
+):
+ """
+ Computes the terms in pi or rho of the log likelihood of the expected complete data
+ """
+ if em_type == 'VEM':
+ res = (
+ ZW[:-1, :, :, nax] *
+ qzw[1:, :, :, :] *
+ log_pi_rho[nax, nax, :, :]
+ ).sum((2, 3))
+
+ elif em_type == 'CEM':
+ res = (
+ ZW[:-1, :, :, nax] *
+ ZW[1:, :, nax, :] *
+ log_pi_rho[nax, nax, :, :]
+ ).sum((2, 3))
+
+ return res.sum(where=mask)
+
+
+def get_regularization(
+ regularize_row, regularize_col, lambda_r, lambda_c, S_r, S_c, Z, W):
+ """
+ Returns the regularization term : lambda_c R_c + lambda_r R_r
+ """
+ regularization = 0.
+ if regularize_row:
+ regularization += (.5 * lambda_r * (S_r.sum() - S_r.multiply(Z @ Z.T).sum()))
+ if regularize_col:
+ regularization += (.5 * lambda_c * (S_c.sum() - S_c.multiply(W @ W.T).sum()))
+ return regularization
+
+
+def update_mu(Z, W, Xi_, nu, gamma, min_margin):
+ """
+ Update mu by combining numerator (X_i.) and denominator
+ """
+ den_mu = get_denominator_mu(Z, W, nu, gamma)
+ mu = Xi_ / den_mu
+ np.clip(mu, min_margin, None, mu)
+ return mu
+
+
+def update_nu(Z, W, X_j, mu, gamma, min_margin):
+ """
+ Update nu by combining numerator (X_.j) and denominator
+ """
+ den_nu = get_denominator_nu(Z, W, mu, gamma)
+ nu = X_j / den_nu
+ np.clip(nu, min_margin, None, nu)
+ return nu
+
+
+def _compute_static_mixture_part_Lc(
+ regularization_mode, regularize_row, regularize_col, P_r, P_c,
+ Z_, W_, log_alpha, log_beta
+):
+ """
+ Computes the terms in alpha and beta in complete data log likelihood
+ """
+ # the terms in alpha and beta in Lc
+ if (not (regularize_row or regularize_col) or regularization_mode == 'all'):
+ Lc_part = Z_.dot(log_alpha) + W_.dot(log_beta)
+ elif regularization_mode == 'mixture':
+ not_P_r = ~ P_r
+ not_P_c = ~ P_c
+ Lc_part = ((not_P_r[:, nax] * log_alpha[nax, :]).sum() +
+ (not_P_c[:, nax] * log_beta[nax, :]).sum())
+ return Lc_part
+
+
+def compute_Lc_static(
+ model_type, estimated_margins,
+ regularization_mode, regularize_row, regularize_col,
+ P_r, P_c, log_alpha, log_beta, gamma, mu, nu, Xi_, X_j,
+ Z_, W_, X_kl, min_float
+):
+ """
+ computes the complete data log likelihood for a static HLBM (T = 1)
+ """
+ # the terms in alpha and beta in Lc
+ Lc_mixture = _compute_static_mixture_part_Lc(
+ regularization_mode, regularize_row, regularize_col,
+ P_r, P_c, Z_, W_, log_alpha, log_beta
+ )
+ # the terms in gamma (and mu, nu) in Lc
+ Lc_density = compute_Lc_static_density(
+ model_type, estimated_margins,
+ gamma, mu, nu, Xi_, X_j,
+ X_kl, min_float
+ )
+ return Lc_mixture + Lc_density
+
+
+def entropy_static(ZW, min_float):
+ """
+ Entropy, static part
+ """
+ # ZW[absent_nodes] = 0, so no pb
+ return - (ZW * np.log(ZW + min_float)).sum()
+
+
+def entropy_appearing(ZW, appearing_nodes, min_float):
+ """
+ Entropy, appearing nodes part (if any)
+ """
+ entr = 0.
+ for t, _ind_app_t in appearing_nodes.items():
+ entr -= (ZW[t] * np.log(ZW[t] + min_float)).sum(0, where=appearing_nodes).sum()
+ return entr
+
+
+def entropy_present(ZW, qzw, min_float):
+ """
+ Entropy, present nodes part
+ """
+ # ZW[absent_nodes] = 0, so no pb
+ return (ZW[:-1, :, :, nax] * qzw[:1] * np.log(qzw[:1] + min_float)).sum()
+
+
+def entropy_dynamic(
+ ZW,
+ qzw,
+ appearing_nodes,
+ min_float
+):
+ """
+ Total entropy
+ """
+ H = 0.
+ H += entropy_static(ZW, min_float)
+ if appearing_nodes is not None:
+ H += entropy_appearing(ZW, appearing_nodes, min_float)
+ H += entropy_present(ZW, qzw, min_float)
+ return H
diff --git a/dcblockmodels/models/utils/similarity_matrices.py b/dcblockmodels/models/utils/similarity_matrices.py
new file mode 100644
index 0000000..4ac42fd
--- /dev/null
+++ b/dcblockmodels/models/utils/similarity_matrices.py
@@ -0,0 +1,425 @@
+"""
+Methods that deal with the pairwise similarity matrices (i.e. the semi-supervision) used for HLBM
+"""
+
+from itertools import combinations, product
+
+import numpy as np
+import scipy as sp
+from numba import jit, prange
+
+
+def check_similarity_matrix(S):
+ """Checks on similarity matrix correctness"""
+ def is_sparse_mat_symetric(S):
+ S_res = S - S.T
+ return np.all(np.abs(S_res.data) < 1e-3)
+
+ assert is_sparse_mat_symetric(S)
+ assert (S[np.diag_indices_from(S)] == 0.).all()
+
+
+def init_transform(X, S, p):
+ """
+ Returns the np.array X transformed according
+ to a multiplicative initialization technique
+ that uses the similarity matrix S. The order p
+ can be used to propagate the similarity relationship
+ to p-neighborhoods.
+ Note that X_transformed is a dense array, such that
+ running kmeans or skmeans on it can be inefficient
+ in a high-dimensional setting
+ """
+ # transform similarity matrix S into a stochastic matrix W
+ W = np.clip(S.copy().toarray(), 0., None)
+ np.fill_diagonal(W, 1.)
+
+ with np.errstate(divide='ignore'):
+ W_ = W.sum(1)
+ np.divide(W, W_, where=(W_ > 0.), out=W)
+ np.nan_to_num(W, copy=False, nan=0., posinf=0., neginf=None)
+
+ if p > 1:
+ with np.errstate(under='ignore'):
+ W = np.linalg.matrix_power(W, p)
+
+ X_transformed = W @ X
+
+ np.clip(X_transformed, 1e-10, None, out=X_transformed)
+ return X_transformed
+
+
+def build_S_mask(N, frac):
+ """
+ Returns a random N x N symmetric boolean matrix
+ without self loops such that int(frac * N * (N - 1))
+ of its elements are equal to True
+ Used for sampling a given fraction of all the
+ possible constraints
+ """
+ def i_max(k):
+ return (k + 1) * N - int((k + 1) * (k + 2) / 2) - 1
+ i_max_ = [i_max(k) for k in range(N)]
+
+ S = np.zeros((N, N), dtype='bool')
+ n_pairs = int(N * (N - 1) / 2)
+ n_pairs_sampled = int(frac * n_pairs)
+ indexes = np.random.choice(n_pairs, size=n_pairs_sampled, replace=False)
+ indexes = sorted(indexes)
+
+ # k, l the row and columns indexes corresponding
+ # to the edge number 'ind' in S
+ # k_0 the current row index, to speed up
+ # computations, since 'indexes' is sorted
+ k_0 = 0
+ for ind in indexes:
+ for k in range(k_0, N):
+ i_min_k = i_max_[k - 1] + 1 if k != 0 else 0
+ i_max_k = i_max_[k]
+ if i_min_k <= ind <= i_max_k:
+ l = ind - i_min_k + (k + 1)
+ k_0 = k
+ # print(ind, k, l, i_max_k, i_min_k)
+ S[k, l], S[l, k] = True, True
+ break
+ return S
+
+
+def build_S_strat(ZW, frac, frac_noise=0., path_only=False):
+ """
+ returns a similarity matrix build from the
+ true classes ZW, by sampling a fraction frac
+ of the nodes of each class and adding noise
+ to a fraction frac_noise of the nodes
+ """
+ ND = ZW.shape[0]
+ Kzw = np.unique(ZW).shape[0]
+ S = np.zeros((ND, ND))
+ for kzw in range(Kzw):
+ ind_kzw = np.where(ZW == kzw)[0]
+ nb_nodes = int(frac * ind_kzw.shape[0])
+ ind_sampled = np.random.choice(ind_kzw, nb_nodes, replace=False)
+ if not path_only:
+ S[np.ix_(ind_sampled, ind_sampled)] = 1
+ else:
+ for i_1, i_2 in zip(ind_sampled[:-1], ind_sampled[1:]):
+ S[i_1, i_2] = 1
+ S[i_2, i_1] = 1
+ if frac_noise > 0.:
+ nb_nodes_random = int(frac_noise * ND)
+ ind_rnd = np.random.choice(ND, nb_nodes_random, replace=False)
+ S[np.ix_(ind_rnd, ind_rnd)] = 1 - S[np.ix_(ind_rnd, ind_rnd)]
+
+ S[np.diag_indices(ND)] = 0.
+ assert ((S - S.T) == 0).all()
+ return S
+
+
+def build_S(ZW, frac, frac_noise=None):
+ """
+ returns a similarity matrix build from the
+ true classes ZW, by sampling a fraction frac
+ of the nodes of each class
+
+ consider using build_S_sparse
+ """
+ ND = ZW.shape[0]
+ clusters = np.unique(ZW)
+
+ # S_ZW[i, j] = 1 if i and j are in the same
+ # cluster in the partition ZW otherwise S_ZW[i, j] = -1
+ S_ZW = np.zeros((ND, ND), dtype='int')
+ for k in clusters:
+ ind_k = np.where(ZW == k)[0]
+ z_k = np.zeros((ND, 1), dtype='int')
+ z_k[ind_k, :] = True
+ S_ZW += z_k.dot(z_k.T)
+
+ S_ZW[S_ZW == 0] = -1
+ S_mask = build_S_mask(ND, frac)
+ S = S_ZW * S_mask
+
+ if frac_noise is not None:
+ S_mask_noise = build_S_mask(ND, frac_noise)
+ S = (S * (~S_mask_noise) - S_mask_noise * S)
+
+ S[np.diag_indices(ND)] = 0.
+ assert ((S - S.T) == 0).all()
+ return S
+
+
+def build_S_ssl(y):
+ """
+ Given an array y of size N with integer values, where
+ y[i] = k : means that observation i is in cluster k
+ and y[i] = -1 : means that we have no information about
+ the cluster of observation i;
+ returns a N x N array S where:
+ S[i, j] = 1 if i and j are in the same cluster,
+ S[i, j] = -1 if they are in different clusters
+ S[i, j] = 0 if we have no information
+ Allows to compare classical semi-supervised
+ approaches to pairwise semi-supervised clustering.
+ """
+ N = y.shape[0]
+ clusters = np.unique(y)
+
+ S = np.zeros((N, N), dtype='int')
+ for k in clusters:
+ if k >= 0:
+ ind_k = np.where(y == k)[0]
+ y_k = np.zeros((N, 1), dtype='int')
+ y_k[ind_k, :] = True
+ S += y_k.dot(y_k.T)
+
+ S[S == 0] = -1
+ ind_masked = np.where(y == -1)[0]
+ S[ind_masked] = 0
+ S[:, ind_masked] = 0
+ S[np.diag_indices(N)] = 0.
+ assert ((S - S.T) == 0).all()
+ return S
+
+
+@jit(nopython=True)
+def i_max_sparse(N, k):
+ """Temporary value computation for i_max_ (for numba parallel computation)"""
+ return (k + 1) * N - int((k + 1) * (k + 2) / 2) - 1
+
+
+@jit(nopython=True)
+def indexes_S_mask_sparse(N, frac):
+ """
+ returns a random N x N symmetric boolean matrix
+ without self loops such that int(frac * N * (N - 1))
+ of its elements are equal to True
+ """
+ i_max_ = [i_max_sparse(N, k) for k in prange(N)] # pylint: disable=not-an-iterable
+
+ n_pairs = int(N * (N - 1) / 2)
+ n_pairs_sampled = int(frac * n_pairs)
+ indexes = np.random.choice(n_pairs, size=n_pairs_sampled, replace=False)
+ indexes = sorted(indexes)
+
+ row_indexes, col_indexes = [], []
+
+ # k, l the row and columns indexes corresponding
+ # to the edge number 'ind' in S
+ # k_0 the current row index, to speed up
+ # computations, since 'indexes' is sorted
+ k_0 = 0
+ for ind in indexes:
+ for k in range(k_0, N):
+ i_min_k = i_max_[k - 1] + 1 if k != 0 else 0
+ i_max_k = i_max_[k]
+ if i_min_k <= ind <= i_max_k:
+ l = ind - i_min_k + (k + 1)
+ k_0 = k
+ # print(ind, k, l, i_max_k, i_min_k)
+ # S[k, l], S[l, k] = True, True
+ row_indexes.append(k)
+ col_indexes.append(l)
+
+ row_indexes.append(l)
+ col_indexes.append(k)
+ break
+
+ return row_indexes, col_indexes
+
+
+def build_S_mask_sparse(n, row_indexes, col_indexes):
+ """Build a mask on a sparse matrix"""
+ S = sp.sparse.dok_matrix((n, n), dtype='bool')
+ dic_indexes = zip(row_indexes, col_indexes)
+ dic_values = (1 for _ in range(len(row_indexes)))
+ update_dic = dict(zip(dic_indexes, dic_values))
+ S._update(update_dic) # pylint: disable=W0212
+ return S.tocsr()
+
+
+def build_S_sparse(ZW, frac, stratified=False):
+ """
+ returns a similarity matrix build from the
+ true classes ZW, by sampling a fraction frac
+ of all the possible ML and CL relationships (stratified = True)
+ or by sampling a fraction frac of the nodes of each class
+
+ Caution : the scales for frac are very different depending
+ on the value of wether stratified is true or not
+ """
+ ND = ZW.shape[0]
+ clusters = np.unique(ZW)
+
+ # S_ZW[i, j] = 1 if i and j are in the same
+ # cluster in the partition ZW otherwise S_ZW[i, j] = -1
+ S_ML = sp.sparse.csr_matrix((ND, ND), dtype='bool')
+ S_CL = sp.sparse.csr_matrix((ND, ND), dtype='bool')
+
+ # build S_ML and S_CL that conatain all relationships
+ if stratified:
+ # stores sampled indices for S
+ inds = {}
+ for k in clusters:
+ ind_k = np.where(ZW == k)[0]
+ n_nodes_k = len(ind_k)
+ n_nodes_sampled = int(frac * n_nodes_k)
+ ind_k = np.random.choice(ind_k, size=n_nodes_sampled, replace=False)
+ inds[k] = ind_k
+
+ # build S_ML and S_CL
+ for k, ind_k in inds.items():
+ z_k = np.zeros((ND, 1), dtype='bool')
+ z_k[ind_k, :] = True
+
+ # z_k_bar[i] = 1 if i is known not to be in cluster k, 0 otherwise
+ z_k_bar = np.zeros((ND, 1), dtype='bool')
+ for k_, ind_k_ in inds.items():
+ if k_ != k:
+ z_k_bar[ind_k_, :] = True
+
+ z_k = sp.sparse.csr_matrix(z_k)
+ z_k_bar = sp.sparse.csr_matrix(z_k_bar)
+ S_ML += z_k * z_k.T
+ S_CL += z_k * z_k_bar.T
+
+ S = (S_ML.astype('int') - S_CL.astype('int'))
+ S[np.diag_indices_from(S)] = 0
+ else:
+ for k in clusters:
+ ind_k = np.where(ZW == k)[0]
+ z_k_ = np.zeros((ND, 1), dtype='bool')
+ z_k_[ind_k, :] = True
+
+ # z_k[i] = 1 if i is known to be in cluster k, 0 otherwise
+ # z_k_bar[i] = 1 if i is known not to be in cluster k, 0 otherwise
+ z_k = sp.sparse.csr_matrix(z_k_)
+ z_k_bar = sp.sparse.csr_matrix((1 - z_k_).astype('bool'))
+ S_ML += z_k * z_k.T
+ S_CL += z_k * z_k_bar.T
+
+ # sample a fraction frac of the relatioships
+ # and builds the integer valued matrix S
+ # S_mask determines which relatiships are sampled
+ row_indexes, col_indexes = indexes_S_mask_sparse(ND, frac)
+ S_mask = build_S_mask_sparse(ND, row_indexes, col_indexes)
+ S = sp.sparse.csr_matrix.multiply(
+ S_ML.astype('int') - S_CL.astype('int'),
+ S_mask
+ )
+ return S
+
+
+def must_link_and_cannot_link_closure(S):
+ """
+ Given a must link and cannot link matrix S
+ such that S[i, j] = 1 for ML, S[i, j] = -1 for CL
+ and S[i, j] = 0 otherwise, returns a matrix with
+ the transitive and reflexive closure of the ML
+ relationship and the closure of the CL relationship.
+ """
+ def transitive_closure(list_tuples):
+ closure = set(list_tuples)
+ while True:
+ new_relations = set((x, w) for x, y in closure for q, w in closure if q == y)
+ closure_until_now = closure | new_relations
+ if closure_until_now == closure:
+ break
+ closure = closure_until_now
+ return closure
+
+ if sp.sparse.issparse(S):
+ S = S.toarray()
+
+ assert S.shape[0] == S.shape[1]
+ assert (S.T == S).all()
+ assert np.isin(np.unique(S), np.array([-1, 0, 1])).all()
+
+ # builds S_res that contains the transitive closure
+ # of the ML relationship
+ S_tu = np.triu(S)
+ t_clos_ml = transitive_closure(list(zip(*np.where(S_tu > 0.))))
+ S_res = np.zeros_like(S)
+ for i, j in t_clos_ml:
+ S_res[i, j], S_res[j, i] = 1., 1.
+
+ # builds the neighborhoods given by the transitive
+ # closure of the ML relationship
+ N = S.shape[0]
+ S_ = S_res + np.eye(N)
+ neigh_ml = []
+ for i in range(N):
+ neigh = np.where(S_[i] > 0.)[0]
+ new_neigh = np.array([len(np.intersect1d(n, neigh)) == 0 for n in neigh_ml]).all()
+ if new_neigh:
+ neigh_ml.append(neigh)
+
+ # add the CL relatioships between all pairs
+ # of nodes of two neighborhoods if there is
+ # a CL relationship between two nodes of
+ # these neighborhoods
+ for n1, n2 in combinations(neigh_ml, 2):
+ if (S[np.ix_(n1, n2)] < 0.).any():
+ for i, j in product(n1, n2):
+ S_res[i, j], S_res[j, i] = -1, -1
+
+ return S_res
+
+
+def normalize_S(S):
+ """
+ Normalizes the similarity matrix S
+ as presented in the paper
+ """
+ S_ = S.sum(0)
+ S_inv = np.zeros((S.shape[0]))
+ np.divide(1, S_, where=(S_ > 0.), out=S_inv)
+ root_D_inv = np.diag(np.sqrt(S_inv))
+ return root_D_inv @ S @ root_D_inv
+
+
+def similarity_discordance(ZW, S, weighted):
+ """
+ Counts the proportion of ML or CL constraints
+ that are not satisfied in the partition ZW.
+ If ZW is the true partition, the returned value
+ represents the discordance between the true classes
+ and the given similarity information.
+ If ZW is the partition returned by the algorithm
+ without similarity matrix, the returned value
+ represents the information brought by the given
+ similarity matrix.
+ If ZW is the partition returned by the algorithm
+ with similarity matrix, the returned value
+ represents the proportion of constraints from the
+ similarity matrix that are not respected after
+ regularisation.
+ """
+ clusters = np.unique(ZW)
+ ND = ZW.shape[0]
+ not_S_ZW = sp.sparse.csr.csr_matrix((ND, ND), dtype='bool')
+ # not_S_ZW[i, j] = True if ZW[i] != ZW[j] else False
+ for k in clusters:
+ ind_k = np.where(ZW == k)[0]
+ z_k0 = np.zeros((ND, 1), dtype='bool')
+ z_k0[ind_k, :] = True
+ z_k = sp.sparse.csr.csr_matrix(z_k0)
+ not_z_k = sp.sparse.csr.csr_matrix(~z_k0)
+ not_S_ZW += z_k.dot(not_z_k.T)
+
+ S_ZW = sp.sparse.csr.csr_matrix(~not_S_ZW.toarray()) # pylint: disable=E1130
+
+ S_ml = S.copy()
+ S_ml[S_ml < 0.] = 0.
+ S_cl = S.copy()
+ S_cl[S_cl > 0.] = 0.
+ S_cl.data = np.abs(S_cl.data)
+
+ if weighted:
+ total_edges = np.abs(S.data).sum()
+ S_contraints = sp.sparse.csr_matrix.multiply(S_ml, not_S_ZW.astype('float'))
+ S_contraints += sp.sparse.csr_matrix.multiply(S_cl, S_ZW.astype('float'))
+ else:
+ total_edges = S.data.shape[0]
+ S_contraints = sp.sparse.csr_matrix.multiply(S_ml.astype('bool'), not_S_ZW)
+ S_contraints += sp.sparse.csr_matrix.multiply(S_cl.astype('bool'), S_ZW)
+ return S_contraints.data.sum() / total_edges
diff --git a/dcblockmodels/models/utils/smoothing_schedule.py b/dcblockmodels/models/utils/smoothing_schedule.py
new file mode 100644
index 0000000..8dfaed2
--- /dev/null
+++ b/dcblockmodels/models/utils/smoothing_schedule.py
@@ -0,0 +1,54 @@
+"""Smoothing schedule for dLBM"""
+
+import numpy as np
+import matplotlib.pyplot as plt
+
+
+class SmoothingSchedule:
+ """Smoothing schedule for dLBM"""
+
+ def __init__(self, schedule_type, length, tau0=1e-3, x0=-6., x1=6.):
+ self.schedule_type = schedule_type
+ self.length = length
+
+ self.x0 = x0
+ self.x1 = x1
+ self.tau0 = tau0
+
+ if self.schedule_type == 'sigmoid':
+ schedule = np.linspace(x0, x1, self.length)
+ # TODO check
+ # func = lambda x: 1. / (1. + np.exp(- x))
+ # schedule = func(schedule)
+ schedule = 1. / (1. + np.exp(- schedule))
+
+ elif self.schedule_type == 'linear':
+ schedule = np.linspace(tau0, 1., self.length)
+
+ # TODO check whether this is desirable (keeping a last schedule value != 1 amounts to a sort
+ # of regularization)
+ schedule[-1] = 1.
+
+ self.schedule = schedule
+
+ def plot(self):
+ """Plot the schedule in Matplotlib"""
+ f, ax = plt.subplots()
+ ax.plot(self.schedule)
+ f.suptitle('Smoothing Schedule')
+ return ax
+
+ def __str__(self):
+ s = (
+ 'Smoothing schedule:\n'
+ f'schedule type : {self.schedule_type}\n'
+ f'{self.length} steps\n'
+ )
+ if self.schedule_type == 'linear':
+ s += f'from {self.tau0} to 1.'
+ else:
+ s += f'sigmoid on [{self.x0}, {self.x1}]'
+ return s
+
+ def __repr__(self):
+ return self.__str__()
diff --git a/dcblockmodels/plot.py b/dcblockmodels/plot.py
new file mode 100644
index 0000000..f656e55
--- /dev/null
+++ b/dcblockmodels/plot.py
@@ -0,0 +1,542 @@
+"""Plotting functions for model results"""
+
+import time
+import warnings
+
+import numpy as np
+import pandas as pd
+import networkx as nx
+
+import prince
+
+import matplotlib.pyplot as plt
+import seaborn as sns
+import plotly.graph_objects as go
+
+
+def plot_criterions(model, thr_decrease, i_start=0, i_end=-1, legend=True):
+ """
+ Plots the criterions as a function of the iteration
+ number for each initialization of a model.
+
+ In the case of (d)LBM models, two criteria are
+ computed for a given E step. We thus plot the
+ 2 * n_iter_tot values of this criterion.
+
+ The iterations steps where there is a "decrease" in
+ the criterion are highlighed with a 'o' marker.
+ thr_decrease determines by how much an iteration
+ must increase the likelihood to be considered "decreasing"
+ """
+ assert thr_decrease > 0
+ _f, ax = plt.subplots(figsize=(16, 5))
+ pal = sns.color_palette('colorblind', n_colors=model.n_init)
+
+ for init in range(model.n_init):
+ crits = model.all_iter_criterions[init]
+ if hasattr(model, 'all_intermediate_iter_criterions'):
+ interm_crits = model.all_intermediate_iter_criterions[init]
+ # all_crits = [interm[0], crit[0], interm[1], crit[1],...]
+ all_crits = np.array([j
+ for i in zip(interm_crits, crits)
+ for j in i])
+ else:
+ all_crits = np.array(crits)
+ all_crits = all_crits[i_start: i_end]
+
+ diff = all_crits[1:] - all_crits[:-1]
+ increase = (diff > -thr_decrease)
+ if not increase.all():
+ iters_pb = np.where(~increase)[0] + 1
+ ax.plot(iters_pb, all_crits[iters_pb], marker='o', color=pal[init], lw=0)
+
+ ax.plot(all_crits, label=f'{init}', alpha=.9, color=pal[init], lw=2.)
+ if legend:
+ ax.legend(loc='best', title='initializations', fancybox=True)
+ return ax
+
+
+def CA_plot(X, Z, W, absent_row_nodes=None, absent_col_nodes=None, ax=None):
+ """
+ plots the projction of the rows and the columns of
+ the matrix X onto the factorial plane found by
+ correspondance analysis.
+ """
+ warnings.filterwarnings("ignore", category=FutureWarning)
+
+ assert X.ndim == 2
+
+ N, D = X.shape
+ present_row = np.setdiff1d(np.arange(N), absent_row_nodes)
+ present_col = np.setdiff1d(np.arange(D), absent_col_nodes)
+ X_ = X[np.ix_(present_row, present_col)]
+ X_ = X_ + 1e-10
+ N_, D_ = X_.shape
+
+ if N == D and not np.array_equal(present_row, present_col):
+ print('Special case: SBM with different absent row and col nodes.')
+
+ # for directed SBM with different absent
+ # row and col nodes, X_ is not square
+ # and W_ is taken from Z with absent col nodes
+ Z_ = Z[present_row]
+ if W is not None:
+ W_ = W[present_col]
+ else:
+ W_ = Z[present_col]
+
+ if ax is None:
+ _f, ax = plt.subplots(1, 2, figsize=(10, 5))
+ row_clusters = np.unique(Z_)
+ col_clusters = np.unique(W_)
+ n_clusters_tot = row_clusters.shape[0] + col_clusters.shape[0]
+
+ ca = prince.CA(
+ n_components=10,
+ n_iter=30,
+ copy=True,
+ check_input=True,
+ engine='auto',
+ random_state=42
+ )
+
+ df = pd.DataFrame(X_)
+ ca = ca.fit(df)
+ row_factor_score = ca.row_coordinates(df).values
+ col_factor_score = ca.column_coordinates(df).values
+ pal = sns.color_palette('colorblind', n_colors=n_clusters_tot)
+ pal_rows = pal[:row_clusters.shape[0]]
+ pal_cols = pal[row_clusters.shape[0]:]
+
+ for k in row_clusters:
+ ix = np.where(Z_ == k)[0]
+ prop = 100 * ix.shape[0] / N_
+ ax[0].scatter(row_factor_score[ix, 0],
+ row_factor_score[ix, 1],
+ c=[pal_rows[k]],
+ label=f'{k}: {prop:.0f}%',
+ edgecolors='black',
+ alpha=.8, s=200)
+ for l in col_clusters:
+ ix = np.where(W_ == l)[0]
+ prop = 100 * ix.shape[0] / D_
+ ax[1].scatter(col_factor_score[ix, 0],
+ col_factor_score[ix, 1],
+ c=[pal_cols[l]],
+ label=f'{l}: {prop:.0f}%',
+ edgecolors='black',
+ alpha=.8, s=200)
+
+ exp_var = 100 * np.array(ca.explained_inertia_)
+ s = 'explained inertia {:.2f}%'
+ for i in range(2):
+ ax[i].set_xlabel(s.format(exp_var[0]))
+ ax[i].set_ylabel(s.format(exp_var[1]))
+
+ for x, y, name in zip(row_factor_score[:, 0],
+ row_factor_score[:, 1],
+ df.index.values):
+ ax[0].text(x, y, name)
+ for x, y, name in zip(col_factor_score[:, 0],
+ col_factor_score[:, 1],
+ df.columns.values):
+ ax[1].text(x, y, name)
+
+ ax[0].legend(title='clusters')
+ ax[1].legend(title='clusters')
+ warnings.filterwarnings('default', category=FutureWarning)
+ return ax
+
+
+def plot_reorganised_matrix(X, Z, logscale=False, light=1.3, snapshot_titles=None):
+ """
+ Reorganized matrix with line/column permutations
+ """
+ T = X.shape[0]
+ _f, ax = plt.subplots(T, 1, figsize=(6, 6 * T))
+ cmap = sns.cubehelix_palette(light=light, as_cmap=True)
+ for t in range(T):
+ if snapshot_titles is not None:
+ ax[t].title.set_text(snapshot_titles[t])
+ indices = np.argsort(Z[t].astype(int))
+ X_reorg = X[t, indices, :]
+ X_reorg = X_reorg[:, indices]
+ df_plot = pd.DataFrame(X_reorg)
+ if logscale:
+ df_plot = np.log10(df_plot + 1) # pour logscale
+ sns.heatmap(df_plot, ax=ax[t], cmap=cmap, linewidths=.005)
+ # plots the lines that separates the blocks
+ _, unique_indices = np.unique(Z[t], return_counts=True)
+ x_indices = np.cumsum(unique_indices)
+ for x in x_indices:
+ ax[t].axvline(x, linewidth=2.5)
+ ax[t].axhline(x, linewidth=2.5)
+
+
+def plot_connectivity_matrix(gamma, subclusters=None):
+ """
+ Row and column normalized connectivity matrix
+ at a given time step (or constant connectivity)
+ """
+ if subclusters is not None:
+ gamma = gamma[np.ix_(subclusters, subclusters)]
+
+ Kz, Kw = gamma.shape
+ _f, ax = plt.subplots(1, 2, figsize=(2 * 6, 5))
+ cmap = sns.cubehelix_palette(light=1, as_cmap=True)
+
+ g1 = gamma / gamma.sum(axis=1, keepdims=True)
+ g2 = gamma / gamma.sum(axis=0, keepdims=True)
+
+ sns.heatmap(g1, cmap=cmap, ax=ax[0], annot=True, fmt='.2f')
+ sns.heatmap(g2, cmap=cmap, ax=ax[1], annot=True, fmt='.2f')
+
+ for y in range(Kz + 2):
+ ax[0].axhline(y, linewidth=2.5)
+ for x in range(Kw + 2):
+ ax[1].axvline(x, linewidth=2.5)
+
+ if subclusters is not None:
+ for i in range(2):
+ ax[i].set_xticklabels(subclusters)
+ ax[i].set_yticklabels(subclusters)
+
+ for i in range(2):
+ ax[i].set_xlabel('destination cluster')
+ ax[i].set_ylabel('origin cluster')
+ ax[i].set_title('destination connection profiles in %')
+
+
+def to_gephi(X, Z=None, filename=None, model=None):
+ """
+ Plot the matrix in the Gephi (Open Graph Viz Platform) format
+ """
+ assert X.ndim == 2
+
+ if not filename:
+ filename = f'graph_{time.time()}'
+
+ G = nx.from_numpy_matrix(np.matrix(X, dtype=[('weight', int)]))
+ deg = X.sum(axis=0)
+ for i in G.nodes:
+ G.node[i]['node_degree'] = deg[i]
+
+ if Z is not None:
+ assert Z.ndim == 1
+ for i in G.nodes:
+ G.node[i]['true_cluster'] = str(int(Z[i]))
+ if model:
+ for i in G.nodes:
+ G.node[i]['found_cluster'] = str(int(model.best_Z[0, i]))
+
+ nx.write_gexf(G, filename + '.gexf')
+
+
+def plot_alluvial(Z, df_stations=None, dates=None, dataset='mtr'):
+ """
+ Plot the clusters in an alluvial graph (only lines or columns at a time)
+ """
+
+ def parse_stations_bart(stat_numbers):
+ stat_numbers = np.array(list(stat_numbers)).astype('int')
+ quadrigrams = df_stations.iloc[stat_numbers]['quadrigram'].values
+ res = 'Stations :
+' + "
".join(quadrigrams) + "
" + return res + + def parse_stations_mtr(stat_numbers): + """ + from a set of stations to an htlm + formated string that describes the nodes + transitionning between two clusters + """ + stat_numbers = np.array(list(stat_numbers)).astype('int') + d = pd.DataFrame({'station_number': stat_numbers}) + + d = df_stations.merge( + d, + on='station_number', + how='inner' + ).groupby('line_').agg({'station_code': list}).reset_index().values + + res = 'Stations :
' + for l in range(d.shape[0]): + res += '' + d[l, 0] + '' + '
' + trigrams = np.unique(d[l, 1]) + for i, x in enumerate(trigrams): + if (i > 0) and (i % 5 == 0): + res += '
' + res += ' ' + x + res += '
' + return res + + if dates is not None: + assert len(dates) == Z.shape[0] + + T = Z.shape[0] + K_values = np.sort(np.unique(Z.ravel()).astype(int)) + K = K_values.shape[0] + + trans = [set(np.where(np.logical_and(Z[t] == s, Z[t + 1] == d))[0]) + for t in range(T - 1) + for s in K_values + for d in K_values] + + # colors_plotly = ['red', 'green', 'blue', 'yellow', 'orange'] + colors_plotly = np.array(['aliceblue', 'antiquewhite', 'aqua', 'aquamarine', 'azure', + 'beige', 'bisque', 'black', 'blanchedalmond', 'blue', + 'blueviolet', 'brown', 'burlywood', 'cadetblue', + 'chartreuse', 'chocolate', 'coral', 'cornflowerblue', + 'cornsilk', 'crimson', 'cyan', 'darkcyan', + 'darkmagenta', 'darkolivegreen', 'darkorange', + 'darkorchid', 'darkred', 'darksalmon', 'darkseagreen']) + + # Nodes + if dates is not None: + if dataset == 'mtr': + label = [f"K{k}, {dates[t]}" + for t in range(T) + for k in K_values] + elif dataset == 'bart': + label = [f"K{k} {dates[t][1]}" + for t in range(T) + for k in K_values] + else: + label = [f"K{k} t{t}" + for t in range(T) + for k in K_values] + node_color = np.random.choice(colors_plotly, K, replace=False).tolist() * T + + # Links + source = [s + t * K + for t in range(T - 1) + for s in range(K) + for d in range(K)] + target = [d + (t + 1) * K + for t in range(T - 1) + for s in range(K) + for d in range(K)] + + value = np.zeros((K * K * (T - 1))) + for t in range(T - 1): + for s in range(K): + for d in range(K): + flow_tsd = len(trans[t * K**2 + s * K + d]) + value[t * K**2 + s * K + d] = flow_tsd if flow_tsd > 0 else -1 + value = value.astype(int) + + link_colors = ['grey' if s == d else 'black' + for t in range(T - 1) + for s in range(K) + for d in range(K)] + + if df_stations is not None: + if dataset == 'mtr': + link_label = [parse_stations_mtr(x) + if len(x) > 0 else 'None' + for x in trans] + elif dataset == 'bart': + link_label = [parse_stations_bart(x) + if len(x) > 0 else 'None' + for x in trans] + else: + link_label = [str(x) if len(x) > 0 else 'None' for x in trans] + + if dates is not None: + if dataset == 'mtr': + title = ('MTR Network sum-up
Station groups evolution ' + f'from {dates[0]} to {dates[-1]}' + ) + elif dataset == 'bart': + title = ('BART Network sum-up
Station' + ' groups evolution ' + f'from {dates[0][0]} {dates[0][1]}h ' + f'to {dates[-1][0]} {dates[-1][1]}h' + ) + else: + title = f'Clusters evolution over {T} snapshots' + + # Layout + layout = dict(title=title, font=dict(size=12), height=700, width=1200) + + # Figure + f = go.Figure(data=[go.Sankey( + + valueformat=".f", + valuesuffix=" stations", + + node=dict( + pad=30, + thickness=10, + line=dict(color="black", width=0.5), + label=label, + color=node_color, + hoverlabel=dict( + bgcolor=node_color, + font=dict( + size=15 + ) + ) + ), + link=dict( + source=source, + target=target, + value=value, + label=link_label, + color=link_colors, + hoverlabel=dict( + bgcolor='white', + font=dict( + size=12 + ) + ) + ), + textfont=dict( + size=7, + color='black' + ) + + )], + layout=layout + ) + + return f + + +def plot_gamma(gamma, dates=None, step=None, show_cluster_ids=True): + """ + Plot the gamma matrix in Matplotlib + """ + assert gamma.ndim == 3 + if dates is not None: + assert gamma.shape[0] == len(dates) + if step is None: + step = 1 + + K = gamma.shape[1] + T = gamma.shape[0] + time_range = np.arange(0, T, step) + dates = [d for i, d in enumerate(dates) if i % step == 0] + _f, ax = plt.subplots(K, K, figsize=(5 * K, 5 * K), sharex=True, sharey=True) + + for k in range(K): + if show_cluster_ids: + ax[k, 0].set_ylabel(f'from cluster {k}') + ax[K - 1, k].set_xlabel(f'to cluster {k}') + ax[0, k].set_xlabel(f'to cluster {k}') + ax[0, k].xaxis.set_label_position('top') + for l in range(K): + ax[k, l].plot(gamma[:, k, l]) + if dates is not None: + ax[k, l].set_xticks(time_range) + ax[k, l].set_xticklabels(dates) + + +def plot_margins(model, always_present, dates=None, subset=None): + """ + always_present = utils.get_always_present_nodes() + subset : subset of nodes to be considered + dates : ndarray of strings of dates + """ + _f, ax = plt.subplots(2, 1, figsize=(16, 12)) + + if subset is not None: + always_present = np.intersect1d(subset, always_present) + + ax[0].plot(model.best_mu[:, always_present]) + ax[1].plot(model.best_nu[:, always_present]) + + if dates is not None: + ax[0].set_xticklabels(dates) + ax[1].set_xticklabels(dates) + + +def plot_mu_nu_during_optim(debug_values_mu_nu, indexes=None): + """ + Plots mu_{i}^t^(c) as a function of (c), + for each timestep t, each i in indexes and each run + """ + n_init = len(debug_values_mu_nu) + T = debug_values_mu_nu[0][0].shape[0] + _f, ax = plt.subplots(n_init, 1, figsize=(8, 4 * n_init)) + + if indexes is None: + indexes = range(debug_values_mu_nu[0][0].shape[1]) + for init in range(n_init): + mu_nu = np.array(debug_values_mu_nu[init]) + mu_nu = mu_nu[:, :, ] if indexes is not None else mu_nu + for t in range(T): + for i in indexes: + ax[init].plot( + mu_nu[:, t, i], + label=f'mu/nu_t={t},i={i}' + ) + ax[init].legend() + ax[init].title.set_text(f'Run {init + 1}') + plt.tight_layout() + + +def plot_alphas_during_optim(debug_values_alpha_beta): + """ + Plots \alpha_{k}^(c) as a function of (c), + for each cluster and run + """ + n_init = len(debug_values_alpha_beta) + Kzw = debug_values_alpha_beta[0][0].shape[0] + _f, ax = plt.subplots(n_init, 1, figsize=(8, 4 * n_init)) + + for init in range(n_init): + alpha_beta = np.exp(debug_values_alpha_beta[init]) + for k in range(Kzw): + ax[init].plot( + alpha_beta[:, k], + label=f'alpha/beta_{k}' + ) + ax[init].legend() + ax[init].title.set_text(f'Run {init + 1}') + plt.tight_layout() + + +def plot_pi_rho_during_optim(debug_values_pi_rho): + """ + Plots \alpha_{k}^(c) as a function of (c), + for each cluster and run + """ + n_init = len(debug_values_pi_rho) + Kzw = debug_values_pi_rho[0][0].shape[0] + _f, ax = plt.subplots(n_init, 1, figsize=(8, 4 * n_init)) + + for init in range(n_init): + pi_rho = np.exp(debug_values_pi_rho[init]) + for k in range(Kzw): + for l in range(Kzw): + ax[init].plot( + pi_rho[:, k, l], + label=f'pi_rho_{(k, l)}' + ) + ax[init].legend() + ax[init].title.set_text(f'Run {init + 1}') + plt.tight_layout() + + +def plot_gamma_during_optim(debug_values_gamma): + """ + Plots gamma_{kl}^(c) as a function of (c), + for each pair of cluster k and l and run + """ + assert debug_values_gamma[0][0].ndim == 2 + + n_init = len(debug_values_gamma) + Kzw = debug_values_gamma[0][0].shape[0] + _f, ax = plt.subplots(n_init, 1, figsize=(8, 4 * n_init)) + + for init in range(n_init): + gamma = np.exp(debug_values_gamma[init]) + for k in range(Kzw): + for l in range(Kzw): + ax[init].plot( + gamma[:, k, l], + label=f'gamma_{(k, l)}' + ) + ax[init].legend() + ax[init].title.set_text(f'Run {init + 1}') + plt.tight_layout() diff --git a/dcblockmodels/tests/__init__.py b/dcblockmodels/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dcblockmodels/tests/test_dlbm.py b/dcblockmodels/tests/test_dlbm.py new file mode 100644 index 0000000..86b2dd2 --- /dev/null +++ b/dcblockmodels/tests/test_dlbm.py @@ -0,0 +1,303 @@ +import os +import sys +import warnings + +import pathlib +import pytest + +import numpy as np +from sklearn.metrics import adjusted_rand_score + +from ..models.dlbm import dLBM +from .. import data +from ..models.utils import general +from ..models.utils.smoothing_schedule import SmoothingSchedule + + +sys.stderr = open(os.devnull, "w") + +warnings.filterwarnings("ignore", category=DeprecationWarning) + + +# Model test thresholds +ARI_ROW_F = { + 'easy': .4, + 'medium': .15, + 'hard': .1 +} +ARI_COL_F = ARI_ROW_F.copy() +ARI_ROW_T = ARI_ROW_F.copy() +ARI_COL_T = ARI_ROW_T.copy() + +# Model fixed params +smoothing_schedule = SmoothingSchedule('sigmoid', 20) + +debug_output = pathlib.Path(r'./dcblockmodels/model_debug_output') +random_state = None +n_jobs = -1 +verbose = 0 +debug_list = [] +n_iter_supp_smoothing = 5 + +diag_pi_init = 0.7 +diag_rho_init = 0.7 +prior_diagonal_pi = 0. +prior_diagonal_rho = 0. + +max_iter = 100 +tol_iter = 1e-6 +min_float = 1e-15 +min_proba_Z, min_proba_W = .05, .05 +min_proba_mixture_proportions = 1e-2 # to avoid empty clusters +min_margin = 1e-10 +min_gamma = 1e-8 + +init_type = 'skmeans' # 'skmeans', 'kmeans' +given_mu, given_nu = None, None +given_Z, given_W = None, None +n_init_clustering = 10 +node_perturbation_rate = .15 +cluster_perturbation_rate = 0. +threshold_absent_nodes = 0 +blockmodel_params = { + "n_iter_min": 5, + "loc_random_margins": 1e-8, # mean and std used for the initialization + "scale_random_margins": 1e-3, # of the margins if self.type_init_margins == 'random' + "n_init_clustering_consensus": 100 # for best_partition() method of model +} +parameter_smoothing = True +type_init_margins = 'ones' + + +class TestDLBM: + + # Data variable params + # T, N, D, Kz, Kw, level, gamma_0, with_margins, with_absent_nodes + test_data_setups = [ + (10, 100, 120, 3, 4, 'easy', True, False), # incresing difficulty + (10, 100, 120, 3, 4, 'medium', True, False), # incresing difficulty + (10, 100, 120, 3, 4, 'hard', True, False), # incresing difficulty + (10, 100, 100, 3, 3, 'medium', True, False), # N = D, Kz = Kw + (10, 100, 120, 3, 4, 'medium', True, True), # Absent nodes + (100, 100, 120, 3, 4, 'medium', True, False), # T big + (10, 500, 800, 3, 4, 'medium', True, False) # N and D big + ] + + n_init = 10 + n_first = 2 # n_first partitions that will be checked in the test, must be <= n_int + + def fit_model( + self, + test_data, + sparse_X, + em_type, + dtype='float64'): + + (X, Z, W, T, N, D, Kz, Kw, + level, with_margins, with_absent_nodes) = test_data + + print('Test data = \n') + print('level', level) + print('with_margins', with_margins) + print('with_absent_nodes', with_absent_nodes) + + model_type = 'with_margins' if with_margins else 'without_margins' + + if sparse_X: + X_ = [general.to_sparse(X[t]) for t in range(T)] + else: + X_ = X.copy() + + model = dLBM( + model_type=model_type, + em_type=em_type, + parameter_smoothing=parameter_smoothing, + Kz=Kz, Kw=Kw, + n_init=TestDLBM.n_init, + model_id=1, + max_iter=max_iter, + type_init_margins=type_init_margins, + smoothing_schedule=smoothing_schedule.schedule, + n_iter_supp_smoothing=n_iter_supp_smoothing, + prior_diagonal_pi=prior_diagonal_pi, + prior_diagonal_rho=prior_diagonal_rho, + diag_pi_init=diag_pi_init, + diag_rho_init=diag_rho_init, + init_type=init_type, + n_init_clustering=n_init_clustering, + node_perturbation_rate=node_perturbation_rate, + cluster_perturbation_rate=cluster_perturbation_rate, + threshold_absent_nodes=threshold_absent_nodes, + min_proba_mixture_proportions=min_proba_mixture_proportions, + min_gamma=min_gamma, + min_margin=min_margin, + min_proba_Z=min_proba_Z, + min_proba_W=min_proba_W, + dtype=dtype, + blockmodel_params=blockmodel_params, + random_state=random_state, + tol_iter=tol_iter, + n_jobs=n_jobs, verbose=verbose, + debug_list=debug_list, + debug_output=debug_output + ) + model.fit( + X_, + given_Z=given_Z, given_W=given_W, + given_mu=given_mu, given_nu=given_nu + ) + return model, Z, W, level + + def assert_metrics(self, model, Z, W, level, n_first): + for init, (Z_model, W_model) in enumerate(model.best_partition(mode='likelihood', + n_first=n_first)): + ari_row_f = adjusted_rand_score(Z.flatten(), Z_model.flatten()) + ari_col_f = adjusted_rand_score(W.flatten(), W_model.flatten()) + + print(f'level = {level}, ari global = {(ari_row_f, ari_col_f)}') + + assert ari_row_f > ARI_ROW_F[level] + assert ari_col_f > ARI_COL_F[level] + + for t in range(model.T): + ari_row_t = adjusted_rand_score(Z[t], Z_model[t]) + ari_col_t = adjusted_rand_score(W[t], W_model[t]) + assert ari_row_t > ARI_ROW_T[level] + assert ari_col_t > ARI_COL_T[level] + + @pytest.fixture + def test_data(self, request): + (T, N, D, Kz, Kw, + level, with_margins, with_absent_nodes) = request.param + + model_type = 'LBM' + dimensions = {'N': N, 'D': D} + n_clusters = {'Kz': Kz, 'Kw': Kw} + + alphas_dirichlet = { + 'very_easy': 10, + 'easy': 8, + 'medium': 6, + 'hard': 4 + } + diag_vals = { + 'diag': 0, + 'easy': .9, + 'medium': .75, + 'hard': .6 + } + + gamma0_level_dic = { + 'easy': 0.02, + 'medium': 0.01, + 'hard': 0.005 + } + + alpha = data.generate_initial_proportions(Kz, alphas_dirichlet[level]) + beta = data.generate_initial_proportions(Kw, alphas_dirichlet[level]) + prior_init = {'alpha': alpha, 'beta': beta} + + pi = data.generate_diag_transition_matrix(Kz, diag_vals[level]) + rho = data.generate_diag_transition_matrix(Kw, diag_vals[level]) + prior_trans = {'pi': pi, 'rho': rho} + + gamma_0 = gamma0_level_dic[level] + + # Data fixed params + + block_sparsity_matrix = None # \beta_{kl}^t of Matias + # block_sparsity_matrix = 0.1 * np.ones((Kz, Kw), dtype='float') + + constant_margins = True # True, False + start, stop, step = 1, 50, .1 + order_power_law = -1.5 # margins ~ Unif(start, stop)^order_power_law + # mu ~ AR1 : mu_{t+1} = N(a mu_t + c, sigma2) + # (c s.t. mu increasing if sigma2 = 0) + ar_margins, a_ar, sigma2_ar = False, 1.1, .1 + + # absent nodes + min_proba_t = .0 + max_proba_t = .2 + proba_absent = None + + directed = True + self_loops = True + dtype = 'int32' + + if with_margins: + mu, nu = data.generate_margins( + T, N, D, constant_margins, start, stop, step, + directed, order_power_law, + ar_margins, a_ar, sigma2_ar + ) + margins = {'mu': mu, 'nu': nu} + else: + margins = None + + noise_level_ = 0. + + if Kz == 3 and Kw == 4: + gamma = gamma_0 * np.array([ + [1, 2, 4, 1], + [3, 1, 2, 3], + [2, 3, 1, 3] + ]) + elif Kz == 3 and Kw == 3: + gamma = gamma_0 * np.array([ + [1, 2, 3], + [3, 1, 2], + [2, 3, 1] + ]) + else: + raise ValueError + + if T > 1: + gamma = np.stack([gamma for _ in range(T)], axis=0) + if block_sparsity_matrix is not None: + block_sparsity_matrix = np.stack([block_sparsity_matrix for _ in range(T)], axis=0) + + if with_absent_nodes: + absent_row_nodes = data.sample_absent_nodes( + T, N, + min_proba_t=min_proba_t, + max_proba_t=max_proba_t, + proba_absent=proba_absent + ) + if not directed: + absent_col_nodes = absent_row_nodes.copy() + else: + absent_col_nodes = data.sample_absent_nodes( + T, D, + min_proba_t=min_proba_t, + max_proba_t=max_proba_t, + proba_absent=proba_absent + ) + else: + absent_row_nodes, absent_col_nodes = [], [] + + absent_nodes = { + 'absent_row_nodes': absent_row_nodes, + 'absent_col_nodes': absent_col_nodes + } + + X, Z, W = data.generate_data( + T, + model_type, + dimensions, + n_clusters, + prior_init, + prior_trans, + gamma, + with_margins, + margins, + self_loops, + directed, + noise_level_, + with_absent_nodes, + absent_nodes, + dtype, + block_sparsity_matrix + ) + + return (X, Z, W, T, N, D, Kz, Kw, + level, with_margins, with_absent_nodes) diff --git a/dcblockmodels/tests/test_dlbm_long.py b/dcblockmodels/tests/test_dlbm_long.py new file mode 100644 index 0000000..32eca5e --- /dev/null +++ b/dcblockmodels/tests/test_dlbm_long.py @@ -0,0 +1,21 @@ +import pytest +from .test_dlbm import TestDLBM + + +class TestLong(TestDLBM): + + @pytest.mark.parametrize('test_data', TestDLBM.test_data_setups, indirect=True) + @pytest.mark.parametrize('sparse_X', [False, True]) + @pytest.mark.parametrize('em_type', ['VEM', 'CEM']) + def test_fitted_model( + self, + test_data, + sparse_X, + em_type): + + model, Z, W, level = self.fit_model( + test_data, + sparse_X, + em_type, + ) + self.assert_metrics(model, Z, W, level, TestDLBM.n_first) diff --git a/dcblockmodels/tests/test_dlbm_short.py b/dcblockmodels/tests/test_dlbm_short.py new file mode 100644 index 0000000..53bf451 --- /dev/null +++ b/dcblockmodels/tests/test_dlbm_short.py @@ -0,0 +1,45 @@ +import pytest +from .test_dlbm import TestDLBM + + +class TestShort(TestDLBM): + + @pytest.mark.parametrize('test_data', [TestDLBM.test_data_setups[1]], indirect=True) + @pytest.mark.parametrize('sparse_X', [False, True]) + def test_sparse(self, test_data, sparse_X): + model, Z, W, level = self.fit_model( + test_data=test_data, + sparse_X=sparse_X, + em_type='VEM' + ) + self.assert_metrics(model, Z, W, level, TestDLBM.n_first) + + @pytest.mark.parametrize('test_data', [TestDLBM.test_data_setups[1]], indirect=True) + @pytest.mark.parametrize('em_type', ['VEM', 'CEM']) + def test_em_type(self, test_data, em_type): + model, Z, W, level = self.fit_model( + test_data=test_data, + sparse_X=False, + em_type=em_type + ) + self.assert_metrics(model, Z, W, level, TestDLBM.n_first) + + @pytest.mark.parametrize('test_data', [TestDLBM.test_data_setups[4]], indirect=True) + def test_absent(self, test_data): + model, Z, W, level = self.fit_model( + test_data=test_data, + sparse_X=False, + em_type='VEM' + ) + self.assert_metrics(model, Z, W, level, TestDLBM.n_first) + + @pytest.mark.parametrize('test_data', [TestDLBM.test_data_setups[1]], indirect=True) + @pytest.mark.parametrize('dtype', ['float32', 'float64']) + def test_dtype(self, test_data, dtype): + model, Z, W, level = self.fit_model( + test_data=test_data, + sparse_X=False, + em_type='VEM', + dtype=dtype + ) + self.assert_metrics(model, Z, W, level, TestDLBM.n_first) diff --git a/dcblockmodels/tests/test_hlbm.py b/dcblockmodels/tests/test_hlbm.py new file mode 100644 index 0000000..270686e --- /dev/null +++ b/dcblockmodels/tests/test_hlbm.py @@ -0,0 +1,266 @@ +import os +import sys +import warnings + +import pathlib +import pytest + +import numpy as np +from sklearn.metrics import adjusted_rand_score + +from ..models.hlbm import HLBM +from .. import data +from ..models.utils import general, similarity_matrices + + +sys.stderr = open(os.devnull, "w") + +warnings.filterwarnings("ignore", category=DeprecationWarning) + +# Model test thresholds +ARI_ROW = { + 'easy': .4, + 'medium': .3, + 'hard': .1 +} +ARI_COL = ARI_ROW.copy() + + +# Model fixed params +debug_output = pathlib.Path(r'./dcblockmodels/model_debug_output') +random_state = None +n_jobs = -1 +verbose = 0 +debug_list = [] +power_multiplicative_init = 1 # True, False +n_iter_supp_smoothing = 5 + +frac_r, frac_c = .01, .01 +frac_noise = 0. +regularization_mode = 'all' +lambda_0 = 2. +lambda_r, lambda_c = lambda_0, lambda_0 +S_r, S_c = None, None +damping_factor = .7 + +max_iter = 100 +tol_iter = 1e-6 +min_float = 1e-15 +min_proba_Z, min_proba_W = .05, .05 +min_proba_mixture_proportions = 1e-2 # to avoid empty clusters +min_margin = 1e-10 +min_gamma = 1e-8 + +init_type = 'skmeans' # 'skmeans', 'kmeans' +given_mu, given_nu = None, None +given_Z, given_W = None, None +n_init_clustering = 20 +node_perturbation_rate = .15 +cluster_perturbation_rate = 0. +threshold_absent_nodes = 0 +blockmodel_params = { + "n_iter_min": 5, + "loc_random_margins": 1e-8, # mean and std used for the initialization + "scale_random_margins": 1e-3, # of the margins if self.type_init_margins == 'random' + "n_init_clustering_consensus": 100 # for best_partition() method of model +} +parameter_smoothing = True +type_init_margins = 'ones' +dtype = 'float64' +self_loops = True + + +class TestHLBM: + # Data variable params + # T, N, D, Kz, Kw, level, gamma_0, with_margins + test_data_setups = [ + (100, 120, 3, 4, 'easy', True), # incresing difficulty + (100, 120, 3, 4, 'medium', True), # incresing difficulty + (100, 120, 3, 4, 'hard', True), # incresing difficulty + (100, 100, 3, 3, 'medium', True), # N = D + (500, 520, 3, 4, 'medium', True) # N and D big + ] + + n_init = 20 + n_first = 10 # n_first partitions that will be checked in the test, must be <= n_int + + def fit_model( + self, + test_data, + sparse_X, + estimated_margins, + regularize, + em_type, + multiplicative_init + ): + (X, Z, W, N, D, Kz, Kw, + level, with_margins) = test_data + + print('Test data = \n') + print('level', level) + print('with_margins', with_margins) + + model_type = 'with_margins' if with_margins else 'without_margins' + regularize_row, regularize_col = regularize, regularize + multiplicative_init_rows, multiplicative_init_cols = ( + multiplicative_init, + multiplicative_init + ) + S_r = general.to_sparse( + similarity_matrices.build_S(Z, frac_r, frac_noise).astype(dtype) + ) + S_c = general.to_sparse( + similarity_matrices.build_S(W, frac_c, frac_noise).astype(dtype) + ) + if sparse_X: + X_ = general.to_sparse(X) + else: + X_ = X.copy() + + model = HLBM( + Kz=Kz, Kw=Kw, + model_type=model_type, + estimated_margins=estimated_margins, + regularization_mode=regularization_mode, + regularize_row=regularize_row, regularize_col=regularize_col, + n_init=TestHLBM.n_init, + max_iter=max_iter, + em_type=em_type, + damping_factor=damping_factor, + multiplicative_init_rows=multiplicative_init_rows, + multiplicative_init_cols=multiplicative_init_cols, + power_multiplicative_init=power_multiplicative_init, + min_float=min_float, + min_proba_Z=min_proba_Z, + min_proba_W=min_proba_W, + min_proba_mixture_proportions=min_proba_mixture_proportions, + min_margin=min_margin, + min_gamma=min_gamma, + init_type=init_type, + n_init_clustering=n_init_clustering, + node_perturbation_rate=node_perturbation_rate, + model_id=1, + dtype='float64', + threshold_absent_nodes=threshold_absent_nodes, + blockmodel_params=blockmodel_params, + random_state=None, # np.random.RandomState(42) + tol_iter=tol_iter, + n_jobs=-1, + verbose=0, debug_list=[], + debug_output=debug_output + ) + model.fit( + X_, + given_Z=given_Z, given_W=given_W, + S_r=S_r, lambda_r=lambda_r, S_c=S_c, lambda_c=lambda_c + ) + return model, Z, W, level + + def assert_metrics(self, model, Z, W, level, n_first): + for init, (Z_model, W_model) in enumerate(model.best_partition(mode='likelihood', + n_first=n_first)): + ari_row_f = adjusted_rand_score(Z.flatten(), Z_model.flatten()) + ari_col_f = adjusted_rand_score(W.flatten(), W_model.flatten()) + + print(f'level = {level}, ari global = {(ari_row_f, ari_col_f)}') + + assert ari_row_f > ARI_ROW[level] + assert ari_col_f > ARI_COL[level] + + @pytest.fixture + def test_data(self, request): + (N, D, Kz, Kw, + level, with_margins) = request.param + + model_type = 'LBM' + dimensions = {'N': N, 'D': D} + n_clusters = {'Kz': Kz, 'Kw': Kw} + T = 1 + directed = True + dtype = 'int32' + + alphas_dirichlet = { + 'very_easy': 10, + 'easy': 8, + 'medium': 6, + 'hard': 4 + } + gamma0_level_dic = { + 'easy': 0.02, + 'medium': 0.01, + 'hard': 0.005 + } + alpha = data.generate_initial_proportions(Kz, alphas_dirichlet[level]) + beta = data.generate_initial_proportions(Kw, alphas_dirichlet[level]) + prior_init = {'alpha': alpha, 'beta': beta} + + prior_trans = {'pi': None, 'rho': None} + + gamma_0 = gamma0_level_dic[level] + + # Data fixed params + + block_sparsity_matrix = None # \beta_{kl}^t of Matias + # block_sparsity_matrix = 0.1 * np.ones((Kz, Kw), dtype='float') + + constant_margins = True # True, False + start, stop, step = 1, 50, .1 + order_power_law = -1.5 # margins ~ Unif(start, stop)^order_power_law + + if with_margins: + mu, nu = data.generate_margins( + T, N, D, constant_margins, start, stop, step, + directed, order_power_law, + ar_margins=None, a_ar=None, sigma2_ar=None + ) + margins = {'mu': mu, 'nu': nu} + else: + margins = None + + noise_level_ = 0. + + if Kz == 3 and Kw == 4: + gamma = gamma_0 * np.array([ + [1, 2, 4, 1], + [3, 1, 2, 3], + [2, 3, 1, 3] + ]) + elif Kz == 3 and Kw == 3: + gamma = gamma_0 * np.array([ + [1, 2, 3], + [3, 1, 2], + [2, 3, 1] + ]) + else: + raise ValueError + + if T > 1: + gamma = np.stack([gamma for _ in range(T)], axis=0) + if block_sparsity_matrix is not None: + block_sparsity_matrix = np.stack([block_sparsity_matrix for _ in range(T)], axis=0) + + with_absent_nodes = False + absent_nodes = { + 'absent_row_nodes': [], + 'absent_col_nodes': [] + } + X, Z, W = data.generate_data( + T, + model_type, + dimensions, + n_clusters, + prior_init, + prior_trans, + gamma, + with_margins, + margins, + self_loops, + directed, + noise_level_, + with_absent_nodes, + absent_nodes, + dtype, + block_sparsity_matrix + ) + return (X, Z, W, N, D, Kz, Kw, + level, with_margins) diff --git a/dcblockmodels/tests/test_hlbm_long.py b/dcblockmodels/tests/test_hlbm_long.py new file mode 100644 index 0000000..0ba4583 --- /dev/null +++ b/dcblockmodels/tests/test_hlbm_long.py @@ -0,0 +1,31 @@ +import pytest + +from .test_hlbm import TestHLBM + + +class TestLong(TestHLBM): + + @pytest.mark.parametrize('test_data', TestHLBM.test_data_setups, indirect=True) + @pytest.mark.parametrize('sparse_X', [False, True]) + @pytest.mark.parametrize('estimated_margins', [False, True]) + @pytest.mark.parametrize('regularize', [False, True]) + @pytest.mark.parametrize('em_type', ['VEM', 'CEM']) + @pytest.mark.parametrize('multiplicative_init', [False, True]) + def test_fitted_model( + self, + test_data, + sparse_X, + estimated_margins, + regularize, + em_type, + multiplicative_init): + + model, Z, W, level = self.fit_model( + test_data, + sparse_X, + estimated_margins, + regularize, + em_type, + multiplicative_init + ) + self.assert_metrics(model, Z, W, level, TestHLBM.n_first) diff --git a/dcblockmodels/tests/test_hlbm_short.py b/dcblockmodels/tests/test_hlbm_short.py new file mode 100644 index 0000000..d087622 --- /dev/null +++ b/dcblockmodels/tests/test_hlbm_short.py @@ -0,0 +1,61 @@ +import warnings +import pytest + +from .test_hlbm import TestHLBM + +warnings.filterwarnings("ignore", category=DeprecationWarning) + + +class TestShort(TestHLBM): + + @pytest.mark.parametrize('test_data', [TestHLBM.test_data_setups[1]], indirect=True) + @pytest.mark.parametrize('sparse_X', [False, True]) + def test_sparse(self, test_data, sparse_X): + model, Z, W, level = self.fit_model( + test_data=test_data, + sparse_X=sparse_X, + estimated_margins=True, + regularize=False, + em_type='VEM', + multiplicative_init=False + ) + self.assert_metrics(model, Z, W, level, TestHLBM.n_first) + + @pytest.mark.parametrize('test_data', [TestHLBM.test_data_setups[1]], indirect=True) + @pytest.mark.parametrize('estimated_margins', [False, True]) + def test_estimated_margins(self, test_data, estimated_margins): + model, Z, W, level = self.fit_model( + test_data=test_data, + sparse_X=False, + estimated_margins=estimated_margins, + regularize=False, + em_type='VEM', + multiplicative_init=False + ) + self.assert_metrics(model, Z, W, level, TestHLBM.n_first) + + @pytest.mark.parametrize('test_data', [TestHLBM.test_data_setups[1]], indirect=True) + @pytest.mark.parametrize('regularize', [False, True]) + def test_regularize(self, test_data, regularize): + model, Z, W, level = self.fit_model( + test_data=test_data, + sparse_X=False, + estimated_margins=True, + regularize=regularize, + em_type='VEM', + multiplicative_init=False + ) + self.assert_metrics(model, Z, W, level, TestHLBM.n_first) + + @pytest.mark.parametrize('test_data', [TestHLBM.test_data_setups[1]], indirect=True) + @pytest.mark.parametrize('em_type', ['VEM', 'CEM']) + def test_em_type(self, test_data, em_type): + model, Z, W, level = self.fit_model( + test_data=test_data, + sparse_X=False, + estimated_margins=True, + regularize=False, + em_type=em_type, + multiplicative_init=False + ) + self.assert_metrics(model, Z, W, level, TestHLBM.n_first) diff --git a/docs/codestyle.md b/docs/codestyle.md new file mode 100644 index 0000000..36dac15 --- /dev/null +++ b/docs/codestyle.md @@ -0,0 +1,19 @@ +# Code style specifics + +This codebase strives to follow PEP8 standards, but some specifics have been needed for code readability and easier links to the articles describing the algorithm. + +## Variable names + +Since a lot of long equations are implemented, shortened variable names need to be accepted. Also, the maths involve a lot of upper case variables that describe matrices, functions, which cannot be lower case'd without loss of understandability in the code. Therefore, there is some tolerance to the following variables: + +`ND`, `Kz`, `Kw`, `nax`, `ax`, `N`, `D`, `N_`, `D_`, `T`, `G`, `H`, `a`, `c`, `d`, `f`, `df`, `ix`, `ca`, `g1`, `g2`, `i`, `j`, `k`, `k_`, `l`, `n`, `n1`, `n2`, `p`, `s`, `t`, `t_`, `u`, `y`, `it`, `mu`, `nu`, `pi`, `p1`, `p2`, `L1`, `L2`, `dLBM`, `L`, `N_`, `V`, `B`, `D_`, `Hz`, `Hw`, `qw`, `qz`, `root_D_inv`, `x`, `x0`, `x1`, `R`, `C`, `CA_plot` + +and any variables that respect the following regular expressions: + +`Z.*`, `W.*`, `.*Z`, `.*W`, `.*X.*`, `K.*`, `.*S.*`, `.*P.*`, `.*Lc.*`, `.*ND`, `AFD.*`, `AR.*` + +## Code structure + +Some leeway has been left to line length for modules, functions, classes, and for number of arguments/methods. + +Line length is set to 100 for equation readability. diff --git a/notebooks/dlbm_example.ipynb b/notebooks/dlbm_example.ipynb new file mode 100644 index 0000000..0aedae0 --- /dev/null +++ b/notebooks/dlbm_example.ipynb @@ -0,0 +1,1416 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "ac617c7b", + "metadata": {}, + "source": [ + "**This notebook allows to simulate data, classify it using a DLBM model and evaluate the model in a controlled environnement. The model is a dynamic LBM `dLBM` for data represented as a series of adjacency matrices.**" + ] + }, + { + "cell_type": "markdown", + "id": "3bf00e47", + "metadata": {}, + "source": [ + "# Imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3666b39", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T14:12:31.205763Z", + "start_time": "2022-02-03T14:12:31.190085Z" + } + }, + "outputs": [], + "source": [ + "import sys, os\n", + "os.path.dirname(sys.executable), sys.version, sys.path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "da11811b", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T14:12:32.532417Z", + "start_time": "2022-02-03T14:12:31.799891Z" + }, + "scrolled": true + }, + "outputs": [], + "source": [ + "%config Completer.use_jedi = False\n", + "#!jt -t onedork -fs 100 -altp -tfs 11 -nfs 100 -cellw 60% -T -N\n", + "%pip list" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bec266b8", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-10T09:23:31.751414Z", + "start_time": "2022-02-10T09:23:28.903594Z" + } + }, + "outputs": [], + "source": [ + "import sys\n", + "import pathlib\n", + "import numpy as np\n", + "import scipy as sp\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "%matplotlib inline\n", + "\n", + "from sklearn.metrics import confusion_matrix\n", + "\n", + "import warnings\n", + "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n", + "warnings.filterwarnings(\"ignore\", message='Deprecation')\n", + "\n", + "from dcblockmodels.models.dlbm import dLBM\n", + "\n", + "from dcblockmodels import metrics, plot, data\n", + "from dcblockmodels.models.utils import similarity_matrices, general, init\n", + "from dcblockmodels.models.utils.smoothing_schedule import SmoothingSchedule" + ] + }, + { + "cell_type": "markdown", + "id": "8413e799", + "metadata": {}, + "source": [ + "# Data" + ] + }, + { + "cell_type": "markdown", + "id": "7ac65838", + "metadata": {}, + "source": [ + "## Sampling the data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "76075b11", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-10T09:23:31.757910Z", + "start_time": "2022-02-10T09:23:31.753531Z" + } + }, + "outputs": [], + "source": [ + "# whether we sample from a SBM or LBM\n", + "model_type = 'LBM'\n", + "# in case of SBM, whether the graph is directed\n", + "directed = True\n", + "# number of time steps\n", + "T = 10\n", + "# nb of row nodes, nb of column nodes\n", + "N, D = 100, 200\n", + "# nb row clusters, nb of column clusters \n", + "Kz, Kw = 3, 4" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ddd59632", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-10T09:23:31.773339Z", + "start_time": "2022-02-10T09:23:31.759331Z" + } + }, + "outputs": [], + "source": [ + "level_alpha = 'medium'\n", + "level_beta = 'medium'\n", + "level_pi = 'medium'\n", + "level_rho = 'medium'\n", + "\n", + "alphas_dirichlet = {\n", + " 'very_easy': 10,\n", + " 'easy': 8,\n", + " 'medium': 6,\n", + " 'hard': 4\n", + "}\n", + "diag_vals = {\n", + " 'diag': 0,\n", + " 'easy': .9,\n", + " 'medium': .75,\n", + " 'hard': .6\n", + "}\n", + "\n", + "alpha = data.generate_initial_proportions(Kz, alphas_dirichlet[level_alpha])\n", + "beta = data.generate_initial_proportions(Kw, alphas_dirichlet[level_beta])\n", + "prior_init = {'alpha': alpha, 'beta': beta}\n", + "\n", + "pi = data.generate_diag_transition_matrix(Kz, diag_vals[level_pi]) \n", + "rho = data.generate_diag_transition_matrix(Kw, diag_vals[level_rho])\n", + "prior_trans = {'pi': pi, 'rho': rho}\n", + "\n", + "alpha, pi, beta, rho" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "68dbbadf", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-10T09:23:32.496038Z", + "start_time": "2022-02-10T09:23:32.332025Z" + } + }, + "outputs": [], + "source": [ + "with_margins = True # True, False\n", + "constant_margins = False # True, False\n", + "start, stop, step = 1, 50, .1\n", + "order_power_law = -1.5 # margins ~ Unif(start, stop)^order_power_law\n", + "ar_margins, a_ar, sigma2_ar = True, 1.1, .05 # mu ~ AR1 : mu_{t+1} = N(a mu_t + c, sigma2) (c s.t. mu increasing if sigma2 = 0)\n", + "\n", + "if with_margins:\n", + " mu, nu = data.generate_margins(\n", + " T, N, D, constant_margins, start, stop, step,\n", + " directed, order_power_law,\n", + " ar_margins, a_ar, sigma2_ar\n", + " )\n", + " margins = {'mu': mu, 'nu': nu}\n", + "else:\n", + " margins = None\n", + " \n", + "margins" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8db1a89", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-10T09:23:49.335157Z", + "start_time": "2022-02-10T09:23:49.310317Z" + } + }, + "outputs": [], + "source": [ + "with_absent_nodes = True # True, False\n", + "min_proba_t = .0\n", + "max_proba_t = .2\n", + "proba_absent = None\n", + "\n", + "if with_absent_nodes:\n", + " absent_row_nodes = data.sample_absent_nodes(\n", + " T, N,\n", + " min_proba_t=min_proba_t,\n", + " max_proba_t=max_proba_t,\n", + " proba_absent=proba_absent\n", + " )\n", + " if not directed:\n", + " absent_col_nodes = absent_row_nodes.copy()\n", + " else:\n", + " absent_col_nodes = data.sample_absent_nodes(\n", + " T, D,\n", + " min_proba_t=min_proba_t,\n", + " max_proba_t=max_proba_t,\n", + " proba_absent=proba_absent\n", + " )\n", + "else:\n", + " absent_row_nodes, absent_col_nodes = [], []\n", + "\n", + "absent_nodes = {\n", + " 'absent_row_nodes': absent_row_nodes,\n", + " 'absent_col_nodes': absent_col_nodes\n", + "}\n", + "\n", + "absent_nodes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dab2198a", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-10T09:23:56.082978Z", + "start_time": "2022-02-10T09:23:56.062983Z" + } + }, + "outputs": [], + "source": [ + "# scaling factor for the matrix gamma : determines the separability level\n", + "# lower is harder and more sparse\n", + "gamma_0 = .01\n", + "\n", + "# defines the sparsity in a block\n", + "# block_sparsity_matrix[t, k, l] is the proba of a zero\n", + "# in block (k, l) at time t\n", + "# corresponds to the \\beta_{kl}^t of Matias\n", + "block_sparsity_matrix = None\n", + "# block_sparsity_matrix = 0.1 * np.ones((Kz, Kw), dtype='float')\n", + "\n", + "# if we add gaussian noise to the sampled graph\n", + "# not advised since it can make models with lower\n", + "# complete data log likelihood give better classification results\n", + "# than model with higher complete data log likelihood\n", + "noise_level_ = 0.\n", + "\n", + "\n", + "if Kz == 3 and Kw == 4:\n", + " gamma = gamma_0 * np.array([\n", + " [1, 2, 3, 1 ],\n", + " [3, 1, 2, 3 ],\n", + " [2, 3, 1, 4 ]\n", + " ])\n", + "elif Kz == 3 and Kw == 3:\n", + " gamma = gamma_0 * np.array([\n", + " [1, 2, 3],\n", + " [3, 1, 2],\n", + " [2, 3, 1]\n", + " ])\n", + "else:\n", + " raise ValueError\n", + "\n", + "if T > 1:\n", + " gamma = np.stack([gamma for _ in range(T)], axis=0)\n", + " if block_sparsity_matrix is not None:\n", + " block_sparsity_matrix = np.stack([block_sparsity_matrix for _ in range(T)], axis=0)\n", + " \n", + "gamma" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "05ea268c", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-10T09:24:00.099758Z", + "start_time": "2022-02-10T09:23:58.117497Z" + } + }, + "outputs": [], + "source": [ + "dimensions = {'N': N, 'D': D}\n", + "n_clusters = {'Kz': Kz, 'Kw': Kw}\n", + "\n", + "self_loops = True\n", + "dtype = 'int32'\n", + "\n", + "X, Z, W = data.generate_data(\n", + " T,\n", + " model_type,\n", + " dimensions,\n", + " n_clusters,\n", + " prior_init,\n", + " prior_trans,\n", + " gamma,\n", + " with_margins,\n", + " margins,\n", + " self_loops,\n", + " directed,\n", + " noise_level_,\n", + " with_absent_nodes,\n", + " absent_nodes,\n", + " dtype,\n", + " block_sparsity_matrix\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "90c020d5", + "metadata": {}, + "source": [ + "## Plot" + ] + }, + { + "cell_type": "markdown", + "id": "c74df45b", + "metadata": {}, + "source": [ + "### Block view & link with matrix factorization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "46f3dc63", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T11:40:22.239985Z", + "start_time": "2022-02-03T11:40:21.551891Z" + } + }, + "outputs": [], + "source": [ + "t_plot = 0\n", + "X_ = X[t_plot] if X.ndim == 3 else X\n", + "Z_ = Z[t_plot] if X.ndim == 3 else Z\n", + "W_ = W[t_plot] if X.ndim == 3 else W\n", + "gamma_ = gamma[t_plot] if X.ndim == 3 else gamma\n", + "\n", + "row_indices = np.argsort(Z_.astype(int))\n", + "col_indices = np.argsort(W_.astype(int))\n", + "\n", + "cmap = sns.cubehelix_palette(light=1., as_cmap=True)\n", + "f, ax = plt.subplots(1, 4, figsize=(4 * 5, 5))\n", + "\n", + "sns.heatmap(X_, ax=ax[0], cbar=False, square=False, xticklabels=False, yticklabels=False, cmap=cmap)\n", + "ax[0].set_title('Raw data')\n", + "\n", + "sns.heatmap(X_[row_indices, :], ax=ax[1], cbar=False, square=False, xticklabels=False, yticklabels=False, cmap=cmap)\n", + "ax[1].set_title('Row-reorganized data')\n", + "\n", + "sns.heatmap(X_[np.ix_(row_indices, col_indices)], ax=ax[2], cbar=False, square=False, xticklabels=False, yticklabels=False, cmap=cmap)\n", + "ax[2].set_title('Row and column-reorganized data')\n", + "\n", + "Z_encoded = general.encode(Z_, Kz)\n", + "W_encoded = general.encode(W_, Kw)\n", + "X_approx = Z_encoded.dot(gamma_).dot(W_encoded.T)\n", + "sns.heatmap(X_approx[np.ix_(row_indices, col_indices)], ax=ax[3], cbar=False, square=False, xticklabels=False, yticklabels=False, cmap=cmap)\n", + "ax[3].set_title('Connectivity-approximized data')\n", + "\n", + "plt.tight_layout()" + ] + }, + { + "cell_type": "markdown", + "id": "38da6976", + "metadata": {}, + "source": [ + "### Dimensionality reduction with Correspondence Analysis" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4f7c9930", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T11:40:39.150051Z", + "start_time": "2022-02-03T11:40:34.318621Z" + } + }, + "outputs": [], + "source": [ + "t_plot = 0\n", + "t_plot = [t for t in range(T)]\n", + "t_plot = [0, T//2, T - 1]\n", + "\n", + "if X.ndim == 2:\n", + " plot.CA_plot(X, Z, W)\n", + "else:\n", + " if type(t_plot) == int:\n", + " t_plot = [t_plot]\n", + "\n", + " n_plots = len(t_plot)\n", + " f, ax = plt.subplots(n_plots, 2, figsize=(10, 5 * n_plots))\n", + " for i, t in enumerate(t_plot):\n", + " W_plot = W[t] if W is not None else None\n", + "\n", + " absent_row = [tup[1] for tup in absent_row_nodes if tup[0] == t]\n", + " absent_col = [tup[1] for tup in absent_col_nodes if tup[0] == t]\n", + "\n", + " plot.CA_plot(\n", + " X[t],\n", + " Z[t], W_plot,\n", + " absent_row, absent_col,\n", + " ax=ax[i]\n", + " )\n", + " ax[i, 0].set_title(f't = {t}')\n", + " ax[i, 1].set_title(f't = {t}')\n" + ] + }, + { + "cell_type": "markdown", + "id": "137afcd1", + "metadata": {}, + "source": [ + "### True margins" + ] + }, + { + "cell_type": "markdown", + "id": "2de3e036", + "metadata": {}, + "source": [ + "Plot margins over time, in the dynamic case" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23024ee1", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T11:40:44.260515Z", + "start_time": "2022-02-03T11:40:44.011289Z" + } + }, + "outputs": [], + "source": [ + "n_nodes = 20\n", + "f, ax = plt.subplots(1, 2, figsize=(2 * 6, 4))\n", + "\n", + "ax[0].plot(margins['mu'][:, np.random.choice(N, size=n_nodes)]);\n", + "ax[0].set_title('True row margins mu');\n", + "\n", + "ax[1].plot(margins['nu'][:, np.random.choice(D, size=n_nodes)]);\n", + "ax[1].set_title('True col margins nu');" + ] + }, + { + "cell_type": "markdown", + "id": "224c9caf", + "metadata": {}, + "source": [ + "### Factorial Discriminant Analysis" + ] + }, + { + "cell_type": "markdown", + "id": "5bfc8b85", + "metadata": {}, + "source": [ + "Measures the level of linear separability of the classes after projection onto R^N using correspondence analysis\n", + "\n", + "See Discriminative Factorial Analysis : http://www.math.u-bordeaux.fr/~mchave100p/wordpress/wp-content/uploads/2013/10/AFD.pdf" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b8dc5130", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T11:40:49.139055Z", + "start_time": "2022-02-03T11:40:47.753899Z" + } + }, + "outputs": [], + "source": [ + "t_plot = 0\n", + "t_plot = [t for t in range(T)]\n", + "t_plot = [0, T//2, -1]\n", + "\n", + "n_components = 3\n", + "\n", + "f, ax = plt.subplots(len(t_plot), 2, squeeze=False, sharex=True, sharey=True, figsize=(5 * len(t_plot), 8))\n", + "xs = np.arange(n_components, dtype='int')\n", + "\n", + "if X.ndim == 3:\n", + " for i, t in enumerate(t_plot):\n", + " res = metrics.AFD_CA_linear_separation(\n", + " X[t], Z[t], W[t],\n", + " n_components=n_components,\n", + " absent_row_nodes=absent_row_nodes,\n", + " absent_col_nodes=absent_col_nodes\n", + " )\n", + "\n", + " ax[i, 0].bar(xs, res[0])\n", + " ax[i, 1].bar(xs, res[1])\n", + " ax[i, 0].set_xlabel('factorial axis')\n", + " ax[i, 1].set_xlabel('factorial axis')\n", + " ax[i, 0].set_title(f'Rows, T = {t}')\n", + " ax[i, 1].set_title(f'Cols, T = {t}')\n", + "else:\n", + " res = metrics.AFD_CA_linear_separation(\n", + " X, Z, W,\n", + " n_components=n_components,\n", + " absent_row_nodes=absent_row_nodes,\n", + " absent_col_nodes=absent_col_nodes\n", + " )\n", + " ax[0, 0].bar(xs, res[0])\n", + " ax[0, 1].bar(xs, res[1])\n", + " ax[0, 0].set_xlabel('factorial axis')\n", + " ax[0, 1].set_xlabel('factorial axis')\n", + " ax[0, 0].set_title(f'Rows')\n", + " ax[0, 1].set_title(f'Cols')\n", + " \n", + "plt.suptitle('CA AFD linear separability', y=1);\n", + "plt.tight_layout()" + ] + }, + { + "cell_type": "markdown", + "id": "fdf4c251", + "metadata": {}, + "source": [ + "### Distribution of the values of the cells of the data matrix" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2371b3cf", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T11:41:00.800991Z", + "start_time": "2022-02-03T11:41:00.136830Z" + } + }, + "outputs": [], + "source": [ + "t_plot = 0\n", + "t_plot = [t for t in range(T)]\n", + "t_plot = [0, T//2, -1]\n", + "\n", + "bins = 50\n", + "val_min = 1\n", + "val_max = 100 # int or None\n", + "\n", + "f, ax = plt.subplots(len(t_plot), 1, sharex=True, sharey=True, figsize=(10, 1.5 * len(t_plot)))\n", + "\n", + "for i, t in enumerate(t_plot):\n", + " values = X[t].flatten()\n", + " values = values[values >= val_min]\n", + " if val_max is not None:\n", + " values = values[values < val_max]\n", + " ax[i].hist(values, bins=bins)\n", + " ax[i].set_title(f'time t = {t}')\n", + " \n", + "f.suptitle('Histogram of the values of the cells of the data matrix over time');\n", + "plt.tight_layout();" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8fe333fa", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T11:41:02.120634Z", + "start_time": "2022-02-03T11:41:01.567850Z" + } + }, + "outputs": [], + "source": [ + "t_plot = 0\n", + "t_plot = [t for t in range(T)]\n", + "t_plot = [0, T//2, -1]\n", + "\n", + "bins = 50\n", + "val_min = 0\n", + "val_max = None #int or None\n", + "\n", + "f, ax = plt.subplots(len(t_plot), 1, sharex=True, sharey=True, figsize=(10, 1.5 * len(t_plot)))\n", + "\n", + "for i, t in enumerate(t_plot):\n", + " values = X[t].sum(0).flatten()\n", + " values = values[values >= val_min]\n", + " if val_max is not None:\n", + " values = values[values < val_max]\n", + " ax[i].hist(values, bins=bins)\n", + " ax[i].set_title(f'time t = {t}')\n", + " \n", + "f.suptitle('Histogram of the degrees of the nodes over time');\n", + "plt.tight_layout();" + ] + }, + { + "cell_type": "markdown", + "id": "eb52f014", + "metadata": {}, + "source": [ + "# Models" + ] + }, + { + "cell_type": "markdown", + "id": "37944c14", + "metadata": {}, + "source": [ + "## DLBM" + ] + }, + { + "cell_type": "markdown", + "id": "3c24ab54", + "metadata": {}, + "source": [ + "### Algo params" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2031578", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-10T09:24:05.598531Z", + "start_time": "2022-02-10T09:24:05.416239Z" + } + }, + "outputs": [], + "source": [ + "smoothing_schedule = SmoothingSchedule('sigmoid', 50, tau0=1e-3, x0=-6., x1=5.)\n", + "smoothing_schedule.plot()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "89c67479", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-10T09:24:06.496074Z", + "start_time": "2022-02-10T09:24:06.485304Z" + } + }, + "outputs": [], + "source": [ + "model_type = 'with_margins' # # 'with_margins', # 'without_margins'\n", + "parameter_smoothing = True # True, False\n", + "n_iter_supp_smoothing = 10\n", + "sparse_X = True # True, False\n", + "\n", + "n_init = 5\n", + "em_type = 'VEM' # 'VEM', 'CEM'\n", + "max_iter = 500\n", + "tol_iter = 1e-6\n", + "min_float = 1e-15\n", + "min_proba_Z, min_proba_W = .1, .1\n", + "min_proba_mixture_proportions = 1e-1 # to avoid empty clusters\n", + "min_margin = 1e-10\n", + "min_gamma = 1e-10\n", + "prior_diagonal_pi, prior_diagonal_rho = 0., 0. #.2, .2\n", + "diag_pi_init, diag_rho_init = .7, .7\n", + "\n", + "init_type = 'kmeans' #'given' # 'skmeans', 'kmeans', 'given'\n", + "type_init_margins = 'ones' # ones, X.\n", + "given_mu, given_nu = None, None\n", + "n_init_clustering = 20\n", + "node_perturbation_rate = .15\n", + "cluster_perturbation_rate = 0.\n", + "threshold_absent_nodes = -1\n", + "\n", + "debug_output = pathlib.Path(r'../dcblockmodels/model_debug_output')\n", + "dtype = 'float64'\n", + "random_state = None\n", + "n_jobs = -1\n", + "verbose = 1\n", + "model_id = 12\n", + "\n", + "# debug_list contains the names of the parameters fo the models\n", + "# or of the variational distribution that we wish to monitor\n", + "# during the fitting of the model\n", + "# This is done by writing the values of the model to disk\n", + "# so it takes time and space. Providing an empty list\n", + "# is the normal behavior\n", + "debug_list = ['log_alpha', 'gamma', 'log_pi', 'Z', 'mu'] # []" + ] + }, + { + "cell_type": "markdown", + "id": "d30d6d4d", + "metadata": {}, + "source": [ + "### Initialization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5fc56de5", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-10T09:24:17.200760Z", + "start_time": "2022-02-10T09:24:13.514583Z" + } + }, + "outputs": [], + "source": [ + "if init_type == 'given':\n", + " # one could get initial partitions\n", + " # using any clustering algo\n", + " given_Z = init._skmeans_init(\n", + " np.concatenate([X[t] for t in range(T)], axis=1),\n", + " Kz, n_init_clustering, random_state=None, n_jobs=-1\n", + " )\n", + " given_W = init._skmeans_init(\n", + " np.concatenate([X[t] for t in range(T)], axis=0).T,\n", + " Kw, n_init_clustering, random_state=None, n_jobs=-1\n", + " )\n", + "else:\n", + " given_Z, given_W = None, None\n", + "\n", + "if sparse_X:\n", + " X_ = [general.to_sparse(X[t]) for t in range(T)]\n", + "else:\n", + " X_ = X.copy()" + ] + }, + { + "cell_type": "markdown", + "id": "3540682e", + "metadata": {}, + "source": [ + "### Fitting the model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5a1df072", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-10T09:24:49.158307Z", + "start_time": "2022-02-10T09:24:18.418387Z" + }, + "scrolled": true + }, + "outputs": [], + "source": [ + "model = dLBM(\n", + " model_type=model_type,\n", + " em_type=em_type,\n", + " parameter_smoothing=parameter_smoothing,\n", + " Kz=Kz, Kw=Kw,\n", + " n_init=n_init,\n", + " model_id=model_id,\n", + " max_iter=max_iter,\n", + " type_init_margins=type_init_margins,\n", + " smoothing_schedule=smoothing_schedule.schedule,\n", + " n_iter_supp_smoothing=n_iter_supp_smoothing,\n", + " prior_diagonal_pi=prior_diagonal_pi,\n", + " prior_diagonal_rho=prior_diagonal_rho,\n", + " diag_pi_init=diag_pi_init,\n", + " diag_rho_init=diag_rho_init,\n", + " init_type=init_type,\n", + " n_init_clustering=n_init_clustering,\n", + " node_perturbation_rate=node_perturbation_rate,\n", + " cluster_perturbation_rate=cluster_perturbation_rate,\n", + " threshold_absent_nodes=threshold_absent_nodes,\n", + " min_proba_mixture_proportions=min_proba_mixture_proportions,\n", + " min_gamma=min_gamma,\n", + " min_margin=min_margin,\n", + " min_proba_Z=min_proba_Z,\n", + " min_proba_W=min_proba_W,\n", + " dtype=dtype,\n", + " blockmodel_params=None,\n", + " random_state=random_state,\n", + " tol_iter=tol_iter,\n", + " n_jobs=n_jobs, verbose=verbose,\n", + " debug_list=debug_list,\n", + " debug_output=debug_output\n", + ")\n", + "model.fit(\n", + " X_, \n", + " given_Z=given_Z, given_W=given_W,\n", + " given_mu=given_mu, given_nu=given_nu\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "23bc1bb2", + "metadata": {}, + "source": [ + "## Load/save model" + ] + }, + { + "cell_type": "markdown", + "id": "4e7b20a3", + "metadata": {}, + "source": [ + "### Save" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "05b2141c", + "metadata": { + "ExecuteTime": { + "end_time": "2022-01-24T16:04:42.325906Z", + "start_time": "2022-01-24T16:04:42.312022Z" + } + }, + "outputs": [], + "source": [ + "model.save(path='../saved_models', modelname='my_model')" + ] + }, + { + "cell_type": "markdown", + "id": "3399143e", + "metadata": {}, + "source": [ + "### Load" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c6118fd1", + "metadata": { + "ExecuteTime": { + "end_time": "2022-01-24T16:04:56.102664Z", + "start_time": "2022-01-24T16:04:56.094125Z" + } + }, + "outputs": [], + "source": [ + "model = general.load_model('../saved_models/my_model')\n", + "model" + ] + }, + { + "cell_type": "markdown", + "id": "7c5a2cbe", + "metadata": {}, + "source": [ + "# Metrics & visualizations" + ] + }, + { + "cell_type": "markdown", + "id": "b53b5c62", + "metadata": {}, + "source": [ + "## Partitions and criterion" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d7e162d", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T14:15:04.857333Z", + "start_time": "2022-02-03T14:15:04.628455Z" + } + }, + "outputs": [], + "source": [ + "Z_model, W_model = model.best_partition(mode='likelihood', n_first=1)[0]\n", + "#Z_model, W_model = model.best_partition(mode='consensus: hbgf', n_first=(model.n_init) // 2)[0]\n", + "\n", + "plot.plot_criterions(\n", + " model,\n", + " thr_decrease=1000,\n", + " i_start=0, i_end=-1,\n", + " legend=True\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "dea43b42", + "metadata": {}, + "source": [ + "## DLBM" + ] + }, + { + "cell_type": "markdown", + "id": "208429c4", + "metadata": {}, + "source": [ + "### Metrics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27a56550", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T14:16:03.021136Z", + "start_time": "2022-02-03T14:16:02.939999Z" + }, + "scrolled": true + }, + "outputs": [], + "source": [ + "metrics.print_metrics(\n", + " Z_model, W_model, Z, W,\n", + " absent_nodes=model.absent_nodes,\n", + " print_each_timestep=True\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "fbc3f88a", + "metadata": {}, + "source": [ + "### Distribution of the metrics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6436214f", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T11:45:46.181865Z", + "start_time": "2022-02-03T11:45:45.707107Z" + }, + "scrolled": false + }, + "outputs": [], + "source": [ + "f, ax = plt.subplots(figsize=(8, 4.5))\n", + "\n", + "caris = np.array([\n", + " metrics.get_metrics(\n", + " Z_model, W_model,\n", + " Z, W,\n", + " absent_nodes=model.absent_nodes\n", + " )['cari_f_without_absent']\n", + " for Z_model, W_model in model.best_partition(mode='likelihood', n_first=model.n_init)\n", + "])\n", + "sns.kdeplot(data=caris, ax=ax, bw=.2, clip=(caris.min() - .1, caris.max() + .1));\n", + "ax.set_title(f'{model.__class__.__name__}: max CARI = {100 * caris.max():.2f}');\n", + "ax.set_xlabel('global CARI values');" + ] + }, + { + "cell_type": "markdown", + "id": "6a66f1c9", + "metadata": {}, + "source": [ + "### Confusion matrices" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b945db71", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T11:56:44.940497Z", + "start_time": "2022-02-03T11:56:44.912597Z" + }, + "cell_style": "split" + }, + "outputs": [], + "source": [ + "for t in range(T):\n", + " print('t = {}'.format(t), end='\\n')\n", + " print(metrics.cmat_clustering(confusion_matrix(\n", + " Z_model[t], Z[t])), end='\\n\\n')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e0026a9e", + "metadata": { + "ExecuteTime": { + "end_time": "2022-01-24T16:17:38.322717Z", + "start_time": "2022-01-24T16:17:38.295008Z" + }, + "cell_style": "split" + }, + "outputs": [], + "source": [ + "for t in range(T):\n", + " print('t = {}'.format(t), end='\\n')\n", + " print(metrics.cmat_clustering(confusion_matrix(\n", + " W_model[t], W[t])), end='\\n\\n')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9cbf20d0", + "metadata": { + "ExecuteTime": { + "end_time": "2022-01-24T16:17:41.190033Z", + "start_time": "2022-01-24T16:17:40.984682Z" + } + }, + "outputs": [], + "source": [ + "plot.plot_alluvial(Z)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "00373f9f", + "metadata": { + "ExecuteTime": { + "end_time": "2022-01-24T16:17:43.163869Z", + "start_time": "2022-01-24T16:17:43.132552Z" + } + }, + "outputs": [], + "source": [ + "plot.plot_alluvial(Z_model)" + ] + }, + { + "cell_type": "markdown", + "id": "e5d9696b", + "metadata": {}, + "source": [ + "## Model parameters" + ] + }, + { + "cell_type": "markdown", + "id": "918e2bd5", + "metadata": {}, + "source": [ + "`model.best_parameters` : a list of length the number of initializations of the model. Each element of the list is a tuple in the form `(crit, param_dic)`, where `crit` is the best value of the objective criterion of the model of the given init and `param_dic` contains the parameters of the model that gave this `crit` " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4795bdd6", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T12:01:59.387254Z", + "start_time": "2022-02-03T12:01:59.379744Z" + } + }, + "outputs": [], + "source": [ + "model.best_parameters[0]" + ] + }, + { + "cell_type": "markdown", + "id": "a09928b3", + "metadata": {}, + "source": [ + "Mapping the indexes of the found clusters to the indexes of the true clusters using the Kuhn Munkres/Hungarian algorithm on the confusion matrix" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1d6e8bb4", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T12:09:17.156827Z", + "start_time": "2022-02-03T12:09:17.148111Z" + } + }, + "outputs": [], + "source": [ + "from scipy.optimize import linear_sum_assignment\n", + "from sklearn.metrics import confusion_matrix\n", + "\n", + "Z_model, W_model = model.best_partition(mode='likelihood', n_first=1)[0]\n", + "\n", + "cmat_Z = confusion_matrix(Z_model.flatten(), Z.flatten())\n", + "cmat_W = confusion_matrix(W_model.flatten(), W.flatten())\n", + " \n", + "indexes_Z = linear_sum_assignment(-cmat_Z)[1]\n", + "indexes_W = linear_sum_assignment(-cmat_W)[1]" + ] + }, + { + "cell_type": "markdown", + "id": "c6d18e68", + "metadata": {}, + "source": [ + "### Fixme: Problem with confusion matrix computation & absent nodes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eea23259", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T12:09:18.353259Z", + "start_time": "2022-02-03T12:09:17.993155Z" + } + }, + "outputs": [], + "source": [ + "\"\"\"f, ax = plt.subplots(1, 2, figsize=(12, 6))\n", + "\n", + "gamma_model = model.best_parameters[0][1]['gamma']\n", + "reordered_gamma_model = gamma_model[np.ix_(indexes_Z, indexes_W)]\n", + "\n", + "sns.heatmap(reordered_gamma_model, ax=ax[0], square=True)\n", + "sns.heatmap(gamma[0], ax=ax[1], square=True)\n", + "\n", + "ax[0].set_title('Estimated gamma');\n", + "ax[1].set_title('True gamma');\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d0befdee", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T12:10:29.511205Z", + "start_time": "2022-02-03T12:10:29.485939Z" + } + }, + "outputs": [], + "source": [ + "\"\"\"if isinstance(model, dLBM):\n", + " f, ax = plt.subplots(2, 2, figsize=(12, 6))\n", + "\n", + " pi_model = np.exp(model.best_parameters[0][1]['log_pi'])\n", + " reordered_pi_model = pi_model[np.ix_(indexes_Z, indexes_Z)]\n", + "\n", + " sns.heatmap(reordered_pi_model, ax=ax[0, 0], square=True)\n", + " sns.heatmap(pi, ax=ax[0, 1], square=True)\n", + "\n", + " ax[0, 0].set_title('Estimated pi');\n", + " ax[0, 1].set_title('True pi');\n", + "\n", + " rho_model = np.exp(model.best_parameters[0][1]['log_rho'])\n", + " reordered_rho_model = rho_model[np.ix_(indexes_W, indexes_W)]\n", + "\n", + " sns.heatmap(reordered_rho_model, ax=ax[1, 0], square=True)\n", + " sns.heatmap(rho, ax=ax[1, 1], square=True)\n", + "\n", + " ax[1, 0].set_title('Estimated rho');\n", + " ax[1, 1].set_title('True rho');\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1a6d1e41", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T12:10:51.320478Z", + "start_time": "2022-02-03T12:10:51.306552Z" + } + }, + "outputs": [], + "source": [ + "\"\"\"if isinstance(model, dLBM):\n", + " n_nodes = 20\n", + " f, ax = plt.subplots(2, 2, figsize=(2 * 8, 8))\n", + "\n", + " row_nodes = np.random.choice(N, size=n_nodes)\n", + " col_nodes = np.random.choice(D, size=n_nodes)\n", + "\n", + " ax[0, 0].plot(margins['mu'][:, row_nodes]);\n", + " ax[0, 1].plot(model.best_parameters[0][1]['mu'][:, row_nodes]);\n", + " ax[0, 0].set_title('True row margins mu');\n", + " ax[0, 1].set_title('Estimated row margins mu');\n", + "\n", + " ax[1, 0].plot(margins['nu'][:, col_nodes]);\n", + " ax[1, 1].plot(model.best_parameters[0][1]['nu'][:, col_nodes]);\n", + " ax[1, 0].set_title('True row margins nu');\n", + " ax[1, 1].set_title('Estimated row margins nu');\n", + "\"\"\"" + ] + }, + { + "cell_type": "markdown", + "id": "3cc92c2b", + "metadata": {}, + "source": [ + "# Debug : parameters during inference" + ] + }, + { + "cell_type": "markdown", + "id": "0bd09351", + "metadata": {}, + "source": [ + "Get parameter values during the iterations of the algorithm. The parameters we wish to analyze must be given as strings in given in model.debug_list. The parameters are written in the directory `dcblockmodels/model_debug_output`, which should be emptied from time to time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e40eab9f", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T12:11:18.472738Z", + "start_time": "2022-02-03T12:11:18.177298Z" + } + }, + "outputs": [], + "source": [ + "debug_dic = model.get_debug()\n", + "debug_dic" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a64d40fd", + "metadata": { + "ExecuteTime": { + "end_time": "2022-01-25T12:11:33.462494Z", + "start_time": "2022-01-25T12:11:33.457146Z" + } + }, + "outputs": [], + "source": [ + "# debug_dic['param'][init][iter] : returns the value of the parameter \n", + "# 'param' that was given in self.debug_list\n", + "# for the initialization init\n", + "# and for the iteration iter\n", + "debug_dic['gamma'][0][10]" + ] + }, + { + "cell_type": "markdown", + "id": "1fcac6d5", + "metadata": {}, + "source": [ + "## Alpha and beta" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "393c479c", + "metadata": { + "ExecuteTime": { + "end_time": "2022-01-25T12:11:34.378583Z", + "start_time": "2022-01-25T12:11:33.464514Z" + } + }, + "outputs": [], + "source": [ + "plot.plot_alphas_during_optim(debug_dic['log_alpha'])" + ] + }, + { + "cell_type": "markdown", + "id": "226c1bcb", + "metadata": {}, + "source": [ + "## Pi and rho" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8455add", + "metadata": { + "ExecuteTime": { + "end_time": "2022-01-24T15:59:48.656940Z", + "start_time": "2022-01-24T15:59:47.486307Z" + } + }, + "outputs": [], + "source": [ + "plot.plot_pi_rho_during_optim(debug_dic['log_pi'])" + ] + }, + { + "cell_type": "markdown", + "id": "78f83fdf", + "metadata": {}, + "source": [ + "## Gamma" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2faa8244", + "metadata": { + "ExecuteTime": { + "end_time": "2022-01-24T15:59:58.797714Z", + "start_time": "2022-01-24T15:59:57.843486Z" + }, + "scrolled": false + }, + "outputs": [], + "source": [ + "plot.plot_gamma_during_optim(debug_dic['gamma'])" + ] + }, + { + "cell_type": "markdown", + "id": "784d1a3e", + "metadata": {}, + "source": [ + "## Mu and nu" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a81749c5", + "metadata": { + "ExecuteTime": { + "end_time": "2022-01-24T16:00:12.572968Z", + "start_time": "2022-01-24T16:00:10.193151Z" + } + }, + "outputs": [], + "source": [ + "plot.plot_mu_nu_during_optim(debug_dic['mu'], indexes=np.random.choice(N, size=(2)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6de8df8f", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "hide_input": false, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.15" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": { + "height": "calc(100% - 180px)", + "left": "10px", + "top": "150px", + "width": "486.4px" + }, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/hlbm_example.ipynb b/notebooks/hlbm_example.ipynb new file mode 100644 index 0000000..c7e3f87 --- /dev/null +++ b/notebooks/hlbm_example.ipynb @@ -0,0 +1,1170 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "8ab48679", + "metadata": {}, + "source": [ + "**This notebook allows to simulate data, classify it using a HLBM model and evaluate the model in a controlled environnement. The model is a semi-supervised (or constrained) LBM `HLBM` using pairwise constraints in both row and column space.**" + ] + }, + { + "cell_type": "markdown", + "id": "3f254231", + "metadata": {}, + "source": [ + "# Imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "63dc66e2", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T14:12:31.205763Z", + "start_time": "2022-02-03T14:12:31.190085Z" + } + }, + "outputs": [], + "source": [ + "import sys, os\n", + "os.path.dirname(sys.executable), sys.version, sys.path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1f35a7d1", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T14:12:32.532417Z", + "start_time": "2022-02-03T14:12:31.799891Z" + }, + "scrolled": true + }, + "outputs": [], + "source": [ + "%config Completer.use_jedi = False\n", + "#!jt -t onedork -fs 100 -altp -tfs 11 -nfs 100 -cellw 60% -T -N\n", + "%pip list" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "86c3d00f", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-10T09:23:31.751414Z", + "start_time": "2022-02-10T09:23:28.903594Z" + } + }, + "outputs": [], + "source": [ + "import sys\n", + "import pathlib\n", + "import numpy as np\n", + "import scipy as sp\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "%matplotlib inline\n", + "\n", + "from sklearn.metrics import confusion_matrix\n", + "\n", + "import warnings\n", + "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n", + "warnings.filterwarnings(\"ignore\", message='Deprecation')\n", + "\n", + "from dcblockmodels.models.hlbm import HLBM\n", + "\n", + "from dcblockmodels import metrics, plot, data\n", + "from dcblockmodels.models.utils import similarity_matrices, general, init\n", + "from dcblockmodels.models.utils.smoothing_schedule import SmoothingSchedule" + ] + }, + { + "cell_type": "markdown", + "id": "aafeaf5d", + "metadata": {}, + "source": [ + "# Data" + ] + }, + { + "cell_type": "markdown", + "id": "562c9e8e", + "metadata": {}, + "source": [ + "## Sampling the data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c3793686", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-10T09:23:31.757910Z", + "start_time": "2022-02-10T09:23:31.753531Z" + } + }, + "outputs": [], + "source": [ + "# whether we sample from a SBM or LBM\n", + "model_type = 'LBM'\n", + "# in case of SBM, whether the graph is directed\n", + "directed = True\n", + "# number of time steps\n", + "T = 10\n", + "# nb of row nodes, nb of column nodes\n", + "N, D = 100, 200\n", + "# nb row clusters, nb of column clusters \n", + "Kz, Kw = 3, 4" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ddd4da4c", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-10T09:23:31.773339Z", + "start_time": "2022-02-10T09:23:31.759331Z" + } + }, + "outputs": [], + "source": [ + "level_alpha = 'medium'\n", + "level_beta = 'medium'\n", + "level_pi = 'medium'\n", + "level_rho = 'medium'\n", + "\n", + "alphas_dirichlet = {\n", + " 'very_easy': 10,\n", + " 'easy': 8,\n", + " 'medium': 6,\n", + " 'hard': 4\n", + "}\n", + "diag_vals = {\n", + " 'diag': 0,\n", + " 'easy': .9,\n", + " 'medium': .75,\n", + " 'hard': .6\n", + "}\n", + "\n", + "alpha = data.generate_initial_proportions(Kz, alphas_dirichlet[level_alpha])\n", + "beta = data.generate_initial_proportions(Kw, alphas_dirichlet[level_beta])\n", + "prior_init = {'alpha': alpha, 'beta': beta}\n", + "\n", + "pi = data.generate_diag_transition_matrix(Kz, diag_vals[level_pi]) \n", + "rho = data.generate_diag_transition_matrix(Kw, diag_vals[level_rho])\n", + "prior_trans = {'pi': pi, 'rho': rho}\n", + "\n", + "alpha, pi, beta, rho" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "155d99f1", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-10T09:23:32.496038Z", + "start_time": "2022-02-10T09:23:32.332025Z" + } + }, + "outputs": [], + "source": [ + "with_margins = True # True, False\n", + "constant_margins = False # True, False\n", + "start, stop, step = 1, 50, .1\n", + "order_power_law = -1.5 # margins ~ Unif(start, stop)^order_power_law\n", + "ar_margins, a_ar, sigma2_ar = True, 1.1, .05 # mu ~ AR1 : mu_{t+1} = N(a mu_t + c, sigma2) (c s.t. mu increasing if sigma2 = 0)\n", + "\n", + "if with_margins:\n", + " mu, nu = data.generate_margins(\n", + " T, N, D, constant_margins, start, stop, step,\n", + " directed, order_power_law,\n", + " ar_margins, a_ar, sigma2_ar\n", + " )\n", + " margins = {'mu': mu, 'nu': nu}\n", + "else:\n", + " margins = None\n", + " \n", + "margins" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e962e8fd", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-10T09:23:49.335157Z", + "start_time": "2022-02-10T09:23:49.310317Z" + } + }, + "outputs": [], + "source": [ + "with_absent_nodes = False\n", + "min_proba_t = .0\n", + "max_proba_t = .2\n", + "proba_absent = None\n", + "\n", + "if with_absent_nodes:\n", + " absent_row_nodes = data.sample_absent_nodes(\n", + " T, N,\n", + " min_proba_t=min_proba_t,\n", + " max_proba_t=max_proba_t,\n", + " proba_absent=proba_absent\n", + " )\n", + " if not directed:\n", + " absent_col_nodes = absent_row_nodes.copy()\n", + " else:\n", + " absent_col_nodes = data.sample_absent_nodes(\n", + " T, D,\n", + " min_proba_t=min_proba_t,\n", + " max_proba_t=max_proba_t,\n", + " proba_absent=proba_absent\n", + " )\n", + "else:\n", + " absent_row_nodes, absent_col_nodes = [], []\n", + "\n", + "absent_nodes = {\n", + " 'absent_row_nodes': absent_row_nodes,\n", + " 'absent_col_nodes': absent_col_nodes\n", + "}\n", + "\n", + "absent_nodes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1f1669f", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-10T09:23:56.082978Z", + "start_time": "2022-02-10T09:23:56.062983Z" + } + }, + "outputs": [], + "source": [ + "# scaling factor for the matrix gamma : determines the separability level\n", + "# lower is harder and more sparse\n", + "gamma_0 = .01\n", + "\n", + "# defines the sparsity in a block\n", + "# block_sparsity_matrix[t, k, l] is the proba of a zero\n", + "# in block (k, l) at time t\n", + "# corresponds to the \\beta_{kl}^t of Matias\n", + "block_sparsity_matrix = None\n", + "# block_sparsity_matrix = 0.1 * np.ones((Kz, Kw), dtype='float')\n", + "\n", + "# if we add gaussian noise to the sampled graph\n", + "# not advised since it can make models with lower\n", + "# complete data log likelihood give better classification results\n", + "# than model with higher complete data log likelihood\n", + "noise_level_ = 0.\n", + "\n", + "\n", + "if Kz == 3 and Kw == 4:\n", + " gamma = gamma_0 * np.array([\n", + " [1, 2, 3, 1 ],\n", + " [3, 1, 2, 3 ],\n", + " [2, 3, 1, 4 ]\n", + " ])\n", + "elif Kz == 3 and Kw == 3:\n", + " gamma = gamma_0 * np.array([\n", + " [1, 2, 3],\n", + " [3, 1, 2],\n", + " [2, 3, 1]\n", + " ])\n", + "else:\n", + " raise ValueError\n", + "\n", + "if T > 1:\n", + " gamma = np.stack([gamma for _ in range(T)], axis=0)\n", + " if block_sparsity_matrix is not None:\n", + " block_sparsity_matrix = np.stack([block_sparsity_matrix for _ in range(T)], axis=0)\n", + " \n", + "gamma" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1ab60b66", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-10T09:24:00.099758Z", + "start_time": "2022-02-10T09:23:58.117497Z" + } + }, + "outputs": [], + "source": [ + "dimensions = {'N': N, 'D': D}\n", + "n_clusters = {'Kz': Kz, 'Kw': Kw}\n", + "\n", + "self_loops = True\n", + "dtype = 'int32'\n", + "\n", + "X, Z, W = data.generate_data(\n", + " T,\n", + " model_type,\n", + " dimensions,\n", + " n_clusters,\n", + " prior_init,\n", + " prior_trans,\n", + " gamma,\n", + " with_margins,\n", + " margins,\n", + " self_loops,\n", + " directed,\n", + " noise_level_,\n", + " with_absent_nodes,\n", + " absent_nodes,\n", + " dtype,\n", + " block_sparsity_matrix\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "a2f6984a", + "metadata": {}, + "source": [ + "## Plot" + ] + }, + { + "cell_type": "markdown", + "id": "621122f7", + "metadata": {}, + "source": [ + "### Block view & link with matrix factorization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "851d9f4c", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T11:40:22.239985Z", + "start_time": "2022-02-03T11:40:21.551891Z" + } + }, + "outputs": [], + "source": [ + "t_plot = 0\n", + "X_ = X[t_plot] if X.ndim == 3 else X\n", + "Z_ = Z[t_plot] if X.ndim == 3 else Z\n", + "W_ = W[t_plot] if X.ndim == 3 else W\n", + "gamma_ = gamma[t_plot] if X.ndim == 3 else gamma\n", + "\n", + "row_indices = np.argsort(Z_.astype(int))\n", + "col_indices = np.argsort(W_.astype(int))\n", + "\n", + "cmap = sns.cubehelix_palette(light=1., as_cmap=True)\n", + "f, ax = plt.subplots(1, 4, figsize=(4 * 5, 5))\n", + "\n", + "sns.heatmap(X_, ax=ax[0], cbar=False, square=False, xticklabels=False, yticklabels=False, cmap=cmap)\n", + "ax[0].set_title('Raw data')\n", + "\n", + "sns.heatmap(X_[row_indices, :], ax=ax[1], cbar=False, square=False, xticklabels=False, yticklabels=False, cmap=cmap)\n", + "ax[1].set_title('Row-reorganized data')\n", + "\n", + "sns.heatmap(X_[np.ix_(row_indices, col_indices)], ax=ax[2], cbar=False, square=False, xticklabels=False, yticklabels=False, cmap=cmap)\n", + "ax[2].set_title('Row and column-reorganized data')\n", + "\n", + "Z_encoded = general.encode(Z_, Kz)\n", + "W_encoded = general.encode(W_, Kw)\n", + "X_approx = Z_encoded.dot(gamma_).dot(W_encoded.T)\n", + "sns.heatmap(X_approx[np.ix_(row_indices, col_indices)], ax=ax[3], cbar=False, square=False, xticklabels=False, yticklabels=False, cmap=cmap)\n", + "ax[3].set_title('Connectivity-approximized data')\n", + "\n", + "plt.tight_layout()" + ] + }, + { + "cell_type": "markdown", + "id": "6f3b3e3c", + "metadata": {}, + "source": [ + "### Dimensionality reduction with Correspondence Analysis" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4ac827f", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T11:40:39.150051Z", + "start_time": "2022-02-03T11:40:34.318621Z" + } + }, + "outputs": [], + "source": [ + "t_plot = 0\n", + "t_plot = [t for t in range(T)]\n", + "t_plot = [0, T//2, T - 1]\n", + "\n", + "if X.ndim == 2:\n", + " plot.CA_plot(X, Z, W)\n", + "else:\n", + " if type(t_plot) == int:\n", + " t_plot = [t_plot]\n", + "\n", + " n_plots = len(t_plot)\n", + " f, ax = plt.subplots(n_plots, 2, figsize=(10, 5 * n_plots))\n", + " for i, t in enumerate(t_plot):\n", + " W_plot = W[t] if W is not None else None\n", + "\n", + " absent_row = [tup[1] for tup in absent_row_nodes if tup[0] == t]\n", + " absent_col = [tup[1] for tup in absent_col_nodes if tup[0] == t]\n", + "\n", + " plot.CA_plot(\n", + " X[t],\n", + " Z[t], W_plot,\n", + " absent_row, absent_col,\n", + " ax=ax[i]\n", + " )\n", + " ax[i, 0].set_title(f't = {t}')\n", + " ax[i, 1].set_title(f't = {t}')\n" + ] + }, + { + "cell_type": "markdown", + "id": "e3c8e4a2", + "metadata": {}, + "source": [ + "### True margins" + ] + }, + { + "cell_type": "markdown", + "id": "51bdaee3", + "metadata": {}, + "source": [ + "Plot margins over time, in the dynamic case" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "80020334", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T11:40:44.260515Z", + "start_time": "2022-02-03T11:40:44.011289Z" + } + }, + "outputs": [], + "source": [ + "n_nodes = 20\n", + "f, ax = plt.subplots(1, 2, figsize=(2 * 6, 4))\n", + "\n", + "ax[0].plot(margins['mu'][:, np.random.choice(N, size=n_nodes)]);\n", + "ax[0].set_title('True row margins mu');\n", + "\n", + "ax[1].plot(margins['nu'][:, np.random.choice(D, size=n_nodes)]);\n", + "ax[1].set_title('True col margins nu');" + ] + }, + { + "cell_type": "markdown", + "id": "b54caf79", + "metadata": {}, + "source": [ + "### Factorial Discriminant Analysis" + ] + }, + { + "cell_type": "markdown", + "id": "6cb7cfe0", + "metadata": {}, + "source": [ + "Measures the level of linear separability of the classes after projection onto R^N using correspondence analysis\n", + "\n", + "See Discriminative Factorial Analysis : http://www.math.u-bordeaux.fr/~mchave100p/wordpress/wp-content/uploads/2013/10/AFD.pdf" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd360456", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T11:40:49.139055Z", + "start_time": "2022-02-03T11:40:47.753899Z" + } + }, + "outputs": [], + "source": [ + "t_plot = 0\n", + "t_plot = [t for t in range(T)]\n", + "t_plot = [0, T//2, -1]\n", + "\n", + "n_components = 3\n", + "\n", + "f, ax = plt.subplots(len(t_plot), 2, squeeze=False, sharex=True, sharey=True, figsize=(5 * len(t_plot), 8))\n", + "xs = np.arange(n_components, dtype='int')\n", + "\n", + "if X.ndim == 3:\n", + " for i, t in enumerate(t_plot):\n", + " res = metrics.AFD_CA_linear_separation(\n", + " X[t], Z[t], W[t],\n", + " n_components=n_components,\n", + " absent_row_nodes=absent_row_nodes,\n", + " absent_col_nodes=absent_col_nodes\n", + " )\n", + "\n", + " ax[i, 0].bar(xs, res[0])\n", + " ax[i, 1].bar(xs, res[1])\n", + " ax[i, 0].set_xlabel('factorial axis')\n", + " ax[i, 1].set_xlabel('factorial axis')\n", + " ax[i, 0].set_title(f'Rows, T = {t}')\n", + " ax[i, 1].set_title(f'Cols, T = {t}')\n", + "else:\n", + " res = metrics.AFD_CA_linear_separation(\n", + " X, Z, W,\n", + " n_components=n_components,\n", + " absent_row_nodes=absent_row_nodes,\n", + " absent_col_nodes=absent_col_nodes\n", + " )\n", + " ax[0, 0].bar(xs, res[0])\n", + " ax[0, 1].bar(xs, res[1])\n", + " ax[0, 0].set_xlabel('factorial axis')\n", + " ax[0, 1].set_xlabel('factorial axis')\n", + " ax[0, 0].set_title(f'Rows')\n", + " ax[0, 1].set_title(f'Cols')\n", + " \n", + "plt.suptitle('CA AFD linear separability', y=1);\n", + "plt.tight_layout()" + ] + }, + { + "cell_type": "markdown", + "id": "4c8c6ee6", + "metadata": {}, + "source": [ + "### Distribution of the values of the cells of the data matrix" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "52fbeac0", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T11:41:00.800991Z", + "start_time": "2022-02-03T11:41:00.136830Z" + } + }, + "outputs": [], + "source": [ + "t_plot = 0\n", + "t_plot = [t for t in range(T)]\n", + "t_plot = [0, T//2, -1]\n", + "\n", + "bins = 50\n", + "val_min = 1\n", + "val_max = 100 # int or None\n", + "\n", + "f, ax = plt.subplots(len(t_plot), 1, sharex=True, sharey=True, figsize=(10, 1.5 * len(t_plot)))\n", + "\n", + "for i, t in enumerate(t_plot):\n", + " values = X[t].flatten()\n", + " values = values[values >= val_min]\n", + " if val_max is not None:\n", + " values = values[values < val_max]\n", + " ax[i].hist(values, bins=bins)\n", + " ax[i].set_title(f'time t = {t}')\n", + " \n", + "f.suptitle('Histogram of the values of the cells of the data matrix over time');\n", + "plt.tight_layout();" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "70363856", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T11:41:02.120634Z", + "start_time": "2022-02-03T11:41:01.567850Z" + } + }, + "outputs": [], + "source": [ + "t_plot = 0\n", + "t_plot = [t for t in range(T)]\n", + "t_plot = [0, T//2, -1]\n", + "\n", + "bins = 50\n", + "val_min = 0\n", + "val_max = None #int or None\n", + "\n", + "f, ax = plt.subplots(len(t_plot), 1, sharex=True, sharey=True, figsize=(10, 1.5 * len(t_plot)))\n", + "\n", + "for i, t in enumerate(t_plot):\n", + " values = X[t].sum(0).flatten()\n", + " values = values[values >= val_min]\n", + " if val_max is not None:\n", + " values = values[values < val_max]\n", + " ax[i].hist(values, bins=bins)\n", + " ax[i].set_title(f'time t = {t}')\n", + " \n", + "f.suptitle('Histogram of the degrees of the nodes over time');\n", + "plt.tight_layout();" + ] + }, + { + "cell_type": "markdown", + "id": "258338e3", + "metadata": {}, + "source": [ + "# Models" + ] + }, + { + "cell_type": "markdown", + "id": "35d91484", + "metadata": {}, + "source": [ + "## HLBM" + ] + }, + { + "cell_type": "markdown", + "id": "be241d93", + "metadata": {}, + "source": [ + "### Algo params" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "931527d8", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T12:16:34.960156Z", + "start_time": "2022-02-03T12:16:34.952768Z" + } + }, + "outputs": [], + "source": [ + "t_ = 0\n", + "if X.ndim == 3:\n", + " X_ = X[t_]\n", + " Z_, W_ = Z[t_], W[t_]\n", + "else:\n", + " X_ = X\n", + " Z_, W_ = Z, W\n", + "\n", + "sparse_X = True\n", + "\n", + "frac_r, frac_c = .1, .1\n", + "frac_noise = 0.\n", + "\n", + "lambda_0 = 3.\n", + "damping_factor = .7\n", + "\n", + "n_init = 20\n", + "model_type = 'with_margins' # 'with_margins', 'without_margins'\n", + "estimated_margins = True # True, False\n", + "# 'skmeans' requires spherecluster & Python 3.7\n", + "init_type = 'kmeans' # 'skmeans', 'kmeans'\n", + "regularize_row, regularize_col = False, False\n", + "regularization_mode = 'all' # 'all' 'mixture'\n", + "em_type = 'CEM' # 'VEM', 'CEM'\n", + "dtype = 'float64'\n", + "\n", + "n_init_clustering = 1\n", + "node_perturbation_rate = .1\n", + "multiplicative_init_rows = False # True, False\n", + "multiplicative_init_cols = False # True, False\n", + "power_multiplicative_init = 1 # True, False\n", + "given_Z, given_W = None, None\n", + "n_jobs = -1\n", + "random_state = None # or np.random.RandomState(42) \n", + "\n", + "max_iter = 200\n", + "tol_iter = 1e-8\n", + "\n", + "min_float = 1e-15\n", + "min_proba_Z, min_proba_W = .05, .05\n", + "min_proba_mixture_proportions = 1e-2 # to avoid empty clusters\n", + "min_margin = 1e-10\n", + "min_gamma = 1e-8\n", + "threshold_absent_nodes = -1\n", + "\n", + "# debug_list contains the names of the parameters fo the models\n", + "# or of the variational distribution that we wish to monitor\n", + "# during the fitting of the model\n", + "# This is done by writing the values of the model to disk\n", + "# so it takes time and space. Providing an empty list\n", + "# is the normal behavior\n", + "debug_list = []\n", + "debug_output = pathlib.Path(r'../dcblockmodels/model_debug_output')\n", + "verbose = 1\n", + "model_id = 1" + ] + }, + { + "cell_type": "markdown", + "id": "3975831e", + "metadata": {}, + "source": [ + "### Initialization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "abd3c0f0", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T12:16:38.260411Z", + "start_time": "2022-02-03T12:16:38.246110Z" + } + }, + "outputs": [], + "source": [ + "if sparse_X:\n", + " if not sp.sparse.issparse(X_):\n", + " X_ = general.to_sparse(X_)\n", + "else:\n", + " if sp.sparse.issparse(X):\n", + " X_ = general.to_dense(X_)\n", + "\n", + "S_r = similarity_matrices.build_S(Z_, frac_r, frac_noise)\n", + "S_c = similarity_matrices.build_S(W_, frac_c, frac_noise)\n", + "\n", + "S_r = sp.sparse.csr.csr_matrix(S_r)\n", + "S_c = sp.sparse.csr.csr_matrix(S_c)" + ] + }, + { + "cell_type": "markdown", + "id": "ec0474ce", + "metadata": {}, + "source": [ + "### Fitting the model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "702fc4d2", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T12:16:44.073088Z", + "start_time": "2022-02-03T12:16:39.589722Z" + } + }, + "outputs": [], + "source": [ + "regularize_row, regularize_col = (lambda_0 != 0), (lambda_0 != 0)\n", + "lambda_r, lambda_c = lambda_0, lambda_0\n", + "\n", + "model = HLBM(\n", + " Kz=Kz, Kw=Kw,\n", + " model_type=model_type,\n", + " estimated_margins=estimated_margins,\n", + " regularization_mode=regularization_mode,\n", + " regularize_row=regularize_row,\n", + " regularize_col=regularize_col,\n", + " n_init=n_init,\n", + " max_iter=max_iter,\n", + " em_type=em_type,\n", + " damping_factor=damping_factor,\n", + " multiplicative_init_rows=multiplicative_init_rows,\n", + " multiplicative_init_cols=multiplicative_init_cols,\n", + " power_multiplicative_init=power_multiplicative_init,\n", + " min_float=min_float,\n", + " min_proba_Z=min_proba_Z,\n", + " min_proba_W=min_proba_W,\n", + " min_proba_mixture_proportions=min_proba_mixture_proportions,\n", + " min_margin=min_margin,\n", + " min_gamma=min_gamma,\n", + " init_type=init_type,\n", + " n_init_clustering=n_init_clustering,\n", + " node_perturbation_rate=node_perturbation_rate,\n", + " model_id=model_id,\n", + " dtype=dtype,\n", + " threshold_absent_nodes=threshold_absent_nodes,\n", + " blockmodel_params=None,\n", + " random_state=random_state,\n", + " tol_iter=tol_iter,\n", + " n_jobs=n_jobs,\n", + " verbose=verbose, \n", + " debug_list=debug_list,\n", + " debug_output=debug_output\n", + ")\n", + "\n", + "# Fit model\n", + "model.fit(\n", + " X_,\n", + " given_Z=given_Z,\n", + " given_W=given_W, \n", + " S_r=S_r, lambda_r=lambda_r,\n", + " S_c=S_c, lambda_c=lambda_c\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "91cd4cc9", + "metadata": {}, + "source": [ + "## Load/save model" + ] + }, + { + "cell_type": "markdown", + "id": "426e83bd", + "metadata": {}, + "source": [ + "### Save" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35c23fc1", + "metadata": { + "ExecuteTime": { + "end_time": "2022-01-24T16:04:42.325906Z", + "start_time": "2022-01-24T16:04:42.312022Z" + } + }, + "outputs": [], + "source": [ + "model.save(path='../saved_models', modelname='my_model')" + ] + }, + { + "cell_type": "markdown", + "id": "25b79fd3", + "metadata": {}, + "source": [ + "### Load" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "895ce56c", + "metadata": { + "ExecuteTime": { + "end_time": "2022-01-24T16:04:56.102664Z", + "start_time": "2022-01-24T16:04:56.094125Z" + } + }, + "outputs": [], + "source": [ + "model = general.load_model('../saved_models/my_model')\n", + "model" + ] + }, + { + "cell_type": "markdown", + "id": "4f5ab881", + "metadata": {}, + "source": [ + "# Metrics & visualizations" + ] + }, + { + "cell_type": "markdown", + "id": "0787b309", + "metadata": {}, + "source": [ + "## Partitions and criterion" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "48d5ec9c", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T14:15:04.857333Z", + "start_time": "2022-02-03T14:15:04.628455Z" + } + }, + "outputs": [], + "source": [ + "Z_model, W_model = model.best_partition(mode='likelihood', n_first=1)[0]\n", + "#Z_model, W_model = model.best_partition(mode='consensus: hbgf', n_first=(model.n_init) // 2)[0]\n", + "\n", + "plot.plot_criterions(\n", + " model,\n", + " thr_decrease=1000,\n", + " i_start=0, i_end=-1,\n", + " legend=True\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "386fad5b", + "metadata": {}, + "source": [ + "## HLBM" + ] + }, + { + "cell_type": "markdown", + "id": "99b7b664", + "metadata": {}, + "source": [ + "### Metrics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d4b178ac", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T11:56:32.389344Z", + "start_time": "2022-02-03T11:56:32.369821Z" + } + }, + "outputs": [], + "source": [ + "Z_model, W_model = model.best_partition(mode='likelihood', n_first=1)[0]\n", + "\n", + "metrics.print_metrics(\n", + " Z_model, W_model, Z_, W_,\n", + " absent_nodes=None,\n", + " print_each_timestep=False\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "fbc6bd87", + "metadata": {}, + "source": [ + "### Distribution of the metrics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c9d61f67", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T11:58:50.615439Z", + "start_time": "2022-02-03T11:58:50.273701Z" + } + }, + "outputs": [], + "source": [ + "f, ax = plt.subplots(figsize=(8, 4.5))\n", + "\n", + "caris = np.array([\n", + " metrics.get_metrics(\n", + " Z_model, W_model,\n", + " Z_, W_,\n", + " absent_nodes=None\n", + " )['cari']\n", + " for Z_model, W_model in model.best_partition(mode='likelihood', n_first=model.n_init)\n", + "])\n", + "sns.kdeplot(data=caris, ax=ax, bw=.2, clip=(caris.min() - .1, caris.max() + .1));\n", + "ax.set_title(f'{model.__class__.__name__}: max CARI = {100 * caris.max():.2f}');\n", + "ax.set_xlabel('global CARI values');" + ] + }, + { + "cell_type": "markdown", + "id": "761ec98f", + "metadata": {}, + "source": [ + "### Confusion matrix" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "63beaa4b", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T12:01:51.629518Z", + "start_time": "2022-02-03T12:01:51.614201Z" + } + }, + "outputs": [], + "source": [ + "print(' Z')\n", + "print(\n", + " metrics.cmat_clustering(\n", + " confusion_matrix(Z_model, Z_)\n", + " ),\n", + " end='\\n\\n'\n", + ")\n", + "print(' W')\n", + "print(\n", + " metrics.cmat_clustering(\n", + " confusion_matrix(W_model, W_)\n", + " ),\n", + " end='\\n\\n'\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "c706dfcf", + "metadata": {}, + "source": [ + "## Model parameters" + ] + }, + { + "cell_type": "markdown", + "id": "871a033d", + "metadata": {}, + "source": [ + "`model.best_parameters` : a list of length the number of initializations of the model. Each element of the list is a tuple in the form `(crit, param_dic)`, where `crit` is the best value of the objective criterion of the model of the given init and `param_dic` contains the parameters of the model that gave this `crit` " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7ec82d62", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T12:01:59.387254Z", + "start_time": "2022-02-03T12:01:59.379744Z" + }, + "scrolled": true + }, + "outputs": [], + "source": [ + "model.best_parameters[0]" + ] + }, + { + "cell_type": "markdown", + "id": "092ed77c", + "metadata": {}, + "source": [ + "Mapping the indexes of the found clusters to the indexes of the true clusters using the Kuhn Munkres/Hungarian algorithm on the confusion matrix" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3696d6ab", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T12:09:17.156827Z", + "start_time": "2022-02-03T12:09:17.148111Z" + } + }, + "outputs": [], + "source": [ + "from scipy.optimize import linear_sum_assignment\n", + "from sklearn.metrics import confusion_matrix\n", + "\n", + "Z_model, W_model = model.best_partition(mode='likelihood', n_first=1)[0]\n", + "\n", + "cmat_Z = confusion_matrix(Z_model, Z_)\n", + "cmat_W = confusion_matrix(W_model, W_)\n", + " \n", + "indexes_Z = linear_sum_assignment(-cmat_Z)[1]\n", + "indexes_W = linear_sum_assignment(-cmat_W)[1]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8e3122c6", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T12:09:18.353259Z", + "start_time": "2022-02-03T12:09:17.993155Z" + }, + "scrolled": true + }, + "outputs": [], + "source": [ + "f, ax = plt.subplots(1, 2, figsize=(12, 6))\n", + "\n", + "gamma_model = model.best_parameters[0][1]['gamma']\n", + "print(gamma_model)\n", + "print(indexes_W)\n", + "print(indexes_Z)\n", + "print(Z_model)\n", + "print(Z_)\n", + "print(W_model)\n", + "print(W_)\n", + "print(cmat_Z)\n", + "print(cmat_W)\n", + "\n", + "reordered_gamma_model = gamma_model[np.ix_(indexes_Z, indexes_W)]\n", + "\n", + "sns.heatmap(reordered_gamma_model, ax=ax[0], square=True)\n", + "sns.heatmap(gamma[0], ax=ax[1], square=True)\n", + "\n", + "ax[0].set_title('Estimated gamma');\n", + "ax[1].set_title('True gamma');" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e6f478f8", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "hide_input": false, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.15" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": { + "height": "calc(100% - 180px)", + "left": "10px", + "top": "150px", + "width": "486.4px" + }, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/hlbm_on_text_data.ipynb b/notebooks/hlbm_on_text_data.ipynb new file mode 100644 index 0000000..a679540 --- /dev/null +++ b/notebooks/hlbm_on_text_data.ipynb @@ -0,0 +1,961 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "ed4f90bd", + "metadata": {}, + "source": [ + "**This notebook applies the semi-supervised (or constrained) LBM `HLBM` on high-dimensional text data.**" + ] + }, + { + "cell_type": "markdown", + "id": "98daf658", + "metadata": {}, + "source": [ + "# Imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cbd9af3b", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T14:23:14.066696Z", + "start_time": "2022-02-03T14:23:11.977193Z" + } + }, + "outputs": [], + "source": [ + "import os\n", + "import numpy as np\n", + "import scipy as sp\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "import seaborn as sns\n", + "import pathlib\n", + "\n", + "from dcblockmodels.models.hlbm import HLBM\n", + "from dcblockmodels import metrics, plot, data\n", + "from dcblockmodels.models.utils import similarity_matrices, general, init\n", + "\n", + "import os\n", + "import sys\n", + "sys.stderr = open(os.devnull, \"w\")" + ] + }, + { + "cell_type": "markdown", + "id": "683348b4", + "metadata": {}, + "source": [ + "# Data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0f4d4dc9", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T13:14:26.095465Z", + "start_time": "2022-02-03T13:14:25.865931Z" + } + }, + "outputs": [], + "source": [ + "import re\n", + "import nltk \n", + "from sklearn.feature_extraction.text import CountVectorizer\n", + "from nltk.corpus import stopwords\n", + "nltk.download('stopwords')\n", + "\n", + "def handle_spe_char(s):\n", + " \"\"\"\n", + " Removes some special characters considered uniinformative\n", + " The \"''\" character has to be treated separately\n", + " If there is a number in a token, it is replaced\n", + " by =number and spaces are added around it\n", + " \"\"\"\n", + " spe_char = \".,;()'-/:=[]`*+\\_^|\" #‘’\n", + " table = str.maketrans(dict.fromkeys(spe_char))\n", + " s = s.translate(table)\n", + " s = re.sub(\"''\", \"\", s)\n", + " #s = re.sub('|', ' ', s)\n", + " s = re.sub('\\n', ' ', s)\n", + " s = re.sub('\\d+', ' =number ', s)\n", + " return s\n", + "\n", + "class StemTokenizer:\n", + " stop_words = stopwords.words('english')\n", + " \n", + " def __init__(self, min_word_length):\n", + " self.stemmer = nltk.stem.PorterStemmer()\n", + " self.tokenizer = nltk.word_tokenize\n", + " self.min_wl = min_word_length\n", + " \n", + " def __call__(self, doc):\n", + " res_doc = []\n", + " for token in self.tokenizer(doc):\n", + " token_ = self.stemmer.stem(token)\n", + " if len(token_) >= self.min_wl:\n", + " if token_ not in self.stop_words:\n", + " res_doc.append(token_)\n", + " return res_doc\n", + " \n", + "\n", + "vectorizer = CountVectorizer(\n", + " input='content', encoding='utf-8',\n", + " decode_error='strict', strip_accents=None,\n", + " lowercase=True, preprocessor=handle_spe_char,\n", + " tokenizer=StemTokenizer(min_word_length=3),\n", + " stop_words=None, token_pattern=r\"(?u)\\b\\w\\w+\\b\",\n", + " ngram_range=(1, 1), analyzer='word',\n", + " max_df=1.0, min_df=1, max_features=None,\n", + " vocabulary=None, binary=False, dtype=np.int64\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "id": "444e58ee", + "metadata": {}, + "source": [ + "## 20 Newsgroup" + ] + }, + { + "cell_type": "markdown", + "id": "6aef9c1c", + "metadata": {}, + "source": [ + "### Build" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7c94c45c", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T12:19:39.822609Z", + "start_time": "2022-02-03T12:19:39.388314Z" + } + }, + "outputs": [], + "source": [ + "from sklearn.datasets import fetch_20newsgroups\n", + "\n", + "corpus = fetch_20newsgroups(\n", + " data_home=None, subset='all',\n", + " categories=None, shuffle=True,\n", + " random_state=42, remove=(),\n", + " download_if_missing=True)\n", + "\n", + "corpus.target, corpus.target_names, corpus" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "39b94d74", + "metadata": {}, + "outputs": [], + "source": [ + "nltk.download('punkt')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "605a7122", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T12:22:13.874435Z", + "start_time": "2022-02-03T12:19:40.726561Z" + }, + "scrolled": true + }, + "outputs": [], + "source": [ + "X0 = vectorizer.fit_transform(corpus.data)\n", + "y_ = corpus.target\n", + "wf = np.squeeze(np.asarray(X0.sum(0)))\n", + "\n", + "X0.shape, vectorizer.get_feature_names()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b672bc73", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T12:22:14.103107Z", + "start_time": "2022-02-03T12:22:13.876815Z" + } + }, + "outputs": [], + "source": [ + "plt.plot([np.where(wf >= k)[0].shape[0] for k in range(20)]);\n", + "plt.plot([19949 for k in range(20)]) # 26214\n", + "plt.xlabel('word frequency threshold')\n", + "plt.ylabel('X.shape[1]')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "79336310", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T12:22:14.519733Z", + "start_time": "2022-02-03T12:22:14.104609Z" + } + }, + "outputs": [], + "source": [ + "n_docs = 10000 # None\n", + "min_word_frequency = 9\n", + "\n", + "features = np.array(vectorizer.get_feature_names())\n", + "selected_features_ind = np.where(wf >= min_word_frequency)[0]\n", + "selected_features = features[selected_features_ind]\n", + "unselected_features = features[~selected_features_ind]\n", + "\n", + "X = X0[:, selected_features_ind]\n", + "\n", + "if n_docs is not None:\n", + " docs = np.random.choice(X.shape[0], size=n_docs, replace=False)\n", + " X = X[docs]\n", + " y_ = y_[docs]\n", + " del docs\n", + "\n", + "del X0, features, wf, corpus\n", + "\n", + "X.shape, list(unselected_features)" + ] + }, + { + "cell_type": "markdown", + "id": "d736c296", + "metadata": {}, + "source": [ + "### Save/load" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3affb526", + "metadata": { + "ExecuteTime": { + "end_time": "2022-01-28T10:55:00.672514Z", + "start_time": "2022-01-28T10:55:00.588742Z" + } + }, + "outputs": [], + "source": [ + "sp.sparse.save_npz('data_ng20', X)\n", + "np.save('labels_ng20', y_)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a59d6fb", + "metadata": { + "ExecuteTime": { + "end_time": "2022-01-28T11:04:10.716543Z", + "start_time": "2022-01-28T11:04:10.683613Z" + } + }, + "outputs": [], + "source": [ + "X = sp.sparse.load_npz('data_ng20.npz')\n", + "y_ = np.load('labels_ng20.npy')" + ] + }, + { + "cell_type": "markdown", + "id": "076b894f", + "metadata": {}, + "source": [ + "## Reuters NLTK" + ] + }, + { + "cell_type": "markdown", + "id": "1e681d2c", + "metadata": {}, + "source": [ + "### Build" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8e9f6ace", + "metadata": {}, + "outputs": [], + "source": [ + "import nltk\n", + "nltk.download('reuters')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "467e6b33", + "metadata": { + "ExecuteTime": { + "end_time": "2022-01-28T13:38:05.458909Z", + "start_time": "2022-01-28T13:38:03.752041Z" + } + }, + "outputs": [], + "source": [ + "from nltk.corpus import reuters\n", + "\n", + "fileids = reuters.fileids()\n", + "\n", + "corpus = []\n", + "y = []\n", + "for fileid in fileids[:]:\n", + " labels = reuters.categories(fileid)\n", + " if len(labels) == 1:\n", + " corpus.append(reuters.raw(fileid))\n", + " y.append(labels[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4568bf1a", + "metadata": { + "ExecuteTime": { + "end_time": "2022-01-28T13:38:05.473853Z", + "start_time": "2022-01-28T13:38:05.460877Z" + } + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "n_largest_classes = 10\n", + "classes, counts = np.unique(np.asarray(y), return_counts=True)\n", + "df_classes = pd.DataFrame({'classes': classes, 'counts': counts}).sort_values(by='counts', ascending=False)\n", + "categories = df_classes['classes'].values[:n_largest_classes]\n", + "categories" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "79b5de5a", + "metadata": { + "ExecuteTime": { + "end_time": "2022-01-28T13:38:05.964017Z", + "start_time": "2022-01-28T13:38:05.475299Z" + } + }, + "outputs": [], + "source": [ + "fileids = reuters.fileids(categories=categories)\n", + "\n", + "corpus = []\n", + "y = []\n", + "for fileid in fileids[:]:\n", + " labels = reuters.categories(fileid)\n", + " if len(labels) == 1:\n", + " corpus.append(reuters.raw(fileid))\n", + " y.append(labels[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "438156c1", + "metadata": { + "ExecuteTime": { + "end_time": "2022-01-28T13:38:30.324862Z", + "start_time": "2022-01-28T13:38:05.965851Z" + }, + "scrolled": true + }, + "outputs": [], + "source": [ + "X0 = vectorizer.fit_transform(corpus)\n", + "\n", + "y = np.array(y)\n", + "classes = np.unique(y)\n", + "map_classes = lambda c : np.argmax(classes == c)\n", + "vmap_classes = np.vectorize(map_classes)\n", + "y_ = vmap_classes(y)\n", + "wf = np.squeeze(np.asarray(X0.sum(0)))\n", + "\n", + "X0.shape, vectorizer.get_feature_names()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "62c1725d", + "metadata": { + "ExecuteTime": { + "end_time": "2022-01-28T13:38:30.533417Z", + "start_time": "2022-01-28T13:38:30.326426Z" + } + }, + "outputs": [], + "source": [ + "plt.plot([np.where(wf >= k)[0].shape[0] for k in range(20)]);\n", + "plt.plot([18900 for k in range(20)])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b36dfcd5", + "metadata": { + "ExecuteTime": { + "end_time": "2022-01-28T13:38:30.589577Z", + "start_time": "2022-01-28T13:38:30.535001Z" + } + }, + "outputs": [], + "source": [ + "min_word_frequency = 5\n", + "\n", + "features = np.array(vectorizer.get_feature_names())\n", + "selected_features_ind = np.where(wf >= min_word_frequency)[0]\n", + "selected_features = features[selected_features_ind]\n", + "unselected_features = features[~selected_features_ind]\n", + "\n", + "X = X0[:, selected_features_ind]\n", + "\n", + "X.shape, list(unselected_features)" + ] + }, + { + "cell_type": "markdown", + "id": "077adbbc", + "metadata": {}, + "source": [ + "### Save/load" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "570aba3b", + "metadata": { + "ExecuteTime": { + "end_time": "2022-01-28T13:38:30.803326Z", + "start_time": "2022-01-28T13:38:30.591513Z" + } + }, + "outputs": [], + "source": [ + "sp.sparse.save_npz('data_reuters', X)\n", + "np.save('labels_reuters', y_)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4b0145cc", + "metadata": {}, + "outputs": [], + "source": [ + "X = sp.sparse.load_npz('data_reuters.npz')\n", + "y_ = np.load('labels_reuters.npy')" + ] + }, + { + "cell_type": "markdown", + "id": "63cd7af3", + "metadata": {}, + "source": [ + "## Classic" + ] + }, + { + "cell_type": "markdown", + "id": "4412be51", + "metadata": {}, + "source": [ + "### Build" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fbcc32ad", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T14:23:18.696625Z", + "start_time": "2022-02-03T14:23:17.648900Z" + } + }, + "outputs": [], + "source": [ + "import zipfile\n", + "\n", + "dataset_dir = '../datasets/'\n", + "if 'classic' not in os.listdir(dataset_dir):\n", + " with zipfile.ZipFile(dataset_dir + 'classic.zip', 'r') as zip_ref:\n", + " zip_ref.extractall(dataset_dir)\n", + "os.listdir(dataset_dir)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d87cb71a", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T13:14:53.423619Z", + "start_time": "2022-02-03T13:14:33.044322Z" + } + }, + "outputs": [], + "source": [ + "path = dataset_dir + 'classic/'\n", + "\n", + "def get_X_classic(path, vectorizer):\n", + " files = os.listdir(path)\n", + " corpus = []\n", + " y = []\n", + " for file in files:\n", + " y.append(file.split('.')[0]) # cluster in file name\n", + " with open(path + file) as f:\n", + " doc = ''.join(f.readlines())\n", + " corpus.append(doc)\n", + "\n", + " X0 = vectorizer.fit_transform(corpus)\n", + " return X0, y\n", + "\n", + "X0, y = get_X_classic(path, vectorizer)\n", + "\n", + "y = np.array(y)\n", + "classes = np.unique(y)\n", + "map_classes = lambda c : np.argmax(classes == c)\n", + "vmap_classes = np.vectorize(map_classes)\n", + "y_ = vmap_classes(y)\n", + "\n", + "X0.shape, vectorizer.get_feature_names()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b7ce53ac", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T13:14:53.507717Z", + "start_time": "2022-02-03T13:14:53.425845Z" + }, + "scrolled": true + }, + "outputs": [], + "source": [ + "min_word_frequency = 4 #4\n", + "wf = np.squeeze(np.asarray(X0.sum(0)))\n", + "\n", + "features = np.array(vectorizer.get_feature_names())\n", + "selected_features_ind = np.where(wf >= min_word_frequency)[0]\n", + "selected_features = features[selected_features_ind]\n", + "unselected_features = features[~selected_features_ind]\n", + "\n", + "X = X0[:, selected_features_ind]\n", + "\n", + "X.shape, list(unselected_features)" + ] + }, + { + "cell_type": "markdown", + "id": "4c9e5cad", + "metadata": {}, + "source": [ + "### Save/load" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2c7da18a", + "metadata": { + "ExecuteTime": { + "end_time": "2021-07-21T15:57:22.082456Z", + "start_time": "2021-07-21T15:57:19.406120Z" + } + }, + "outputs": [], + "source": [ + "sp.sparse.save_npz('data_classic', X)\n", + "np.save('labels_classic', y_)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e6e16f9e", + "metadata": {}, + "outputs": [], + "source": [ + "X = sp.sparse.load_npz('data_classic.npz')\n", + "y_ = np.load('labels_classic.npy')" + ] + }, + { + "cell_type": "markdown", + "id": "b519acfe", + "metadata": {}, + "source": [ + "# Model" + ] + }, + { + "cell_type": "markdown", + "id": "8dd6b845", + "metadata": {}, + "source": [ + "## Model params" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ac416815", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T13:15:02.119890Z", + "start_time": "2022-02-03T13:15:02.115223Z" + } + }, + "outputs": [], + "source": [ + "X.shape, np.unique(y_).shape[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c90f3fac", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T13:15:02.423069Z", + "start_time": "2022-02-03T13:15:02.409397Z" + } + }, + "outputs": [], + "source": [ + "np.unique(X.data, return_counts=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dbf8dec7", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T13:15:02.997189Z", + "start_time": "2022-02-03T13:15:02.641958Z" + } + }, + "outputs": [], + "source": [ + "f, ax = plt.subplots(figsize=(10, 10))\n", + "ax.spy(X, markersize=.1, precision=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e57d244d", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T13:15:03.134598Z", + "start_time": "2022-02-03T13:15:03.125542Z" + } + }, + "outputs": [], + "source": [ + "Kz = np.unique(y_).shape[0]\n", + "Kw = 10\n", + "\n", + "max_iter = 100\n", + "tol_iter = 1e-5\n", + "\n", + "frac_r, frac_c = .01, None\n", + "frac_noise = 0.\n", + "\n", + "n_init = 10\n", + "model_type = 'with_margins' # 'with_margins', 'without_margins'\n", + "estimated_margins = False # True, False\n", + "init_type = 'kmeans' #'skmeans' # 'skmeans', 'kmeans'\n", + "regularize_row, regularize_col = True, False\n", + "regularization_mode = 'all' # 'all' 'mixture'\n", + "em_type = 'VEM' # 'VEM', 'CEM'\n", + "compute_regularization = True\n", + "\n", + "lambda_r = 1.\n", + "lambda_c = None\n", + "damping_factor = None if em_type == 'CEM' else .7\n", + "\n", + "multiplicative_init_rows, multiplicative_init_cols = False, False # True, False\n", + "power_multiplicative_init = 1\n", + "given_Z, given_W = None, None\n", + "\n", + "min_float = 1e-15\n", + "min_proba_Z, min_proba_W = .005, .005\n", + "min_proba_mixture_proportions = .1 * (1 / Kz) # to avoid empty clusters\n", + "min_margin = 1e-12\n", + "min_gamma = 1e-12\n", + "threshold_absent_nodes = 0\n", + "dtype = 'float32'\n", + "debug_output = pathlib.Path(r'../dcblockmodels/model_debug_output')\n", + "\n", + "n_init_clustering = 7 * 1\n", + "node_perturbation_rate = .2" + ] + }, + { + "cell_type": "markdown", + "id": "ef7916de", + "metadata": {}, + "source": [ + "## Similarity" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1b50fc4d", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T13:15:14.298175Z", + "start_time": "2022-02-03T13:15:04.292928Z" + } + }, + "outputs": [], + "source": [ + "if regularize_row:\n", + " S_r = similarity_matrices.build_S_sparse(y_, frac_r, stratified=False)\n", + "else:\n", + " S_r = None\n", + "S_c = None\n", + "\n", + "S_r, S_c" + ] + }, + { + "cell_type": "markdown", + "id": "6c1be724", + "metadata": {}, + "source": [ + "## Fitting the model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fd998f3b", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T13:17:54.921904Z", + "start_time": "2022-02-03T13:15:14.300575Z" + }, + "scrolled": true + }, + "outputs": [], + "source": [ + "model = HLBM(\n", + " Kz=Kz, Kw=Kw,\n", + " model_type=model_type,\n", + " estimated_margins=estimated_margins,\n", + " regularization_mode=regularization_mode,\n", + " regularize_row=regularize_row, regularize_col=regularize_col,\n", + " n_init=n_init,\n", + " max_iter=max_iter,\n", + " em_type=em_type,\n", + " damping_factor=damping_factor,\n", + " multiplicative_init_rows=multiplicative_init_rows,\n", + " multiplicative_init_cols=multiplicative_init_cols,\n", + " power_multiplicative_init=power_multiplicative_init,\n", + " min_float=min_float,\n", + " min_proba_Z=min_proba_Z,\n", + " min_proba_W=min_proba_W,\n", + " min_proba_mixture_proportions=min_proba_mixture_proportions,\n", + " min_margin=min_margin,\n", + " min_gamma=min_gamma,\n", + " init_type=init_type,\n", + " n_init_clustering=n_init_clustering,\n", + " node_perturbation_rate=node_perturbation_rate,\n", + " compute_regularization=compute_regularization,\n", + " model_id=1,\n", + " dtype=dtype,\n", + " threshold_absent_nodes=threshold_absent_nodes,\n", + " blockmodel_params=None,\n", + " random_state=None, #np.random.RandomState(42) \n", + " tol_iter=tol_iter,\n", + " n_jobs=-1,\n", + " verbose=1, debug_list=[], #'Z', 'W'\n", + " debug_output=debug_output\n", + ")\n", + "model.fit(\n", + " X,\n", + " given_Z=given_Z,\n", + " given_W=given_W, \n", + " S_r=S_r, lambda_r=lambda_r,\n", + " S_c=S_c, lambda_c=lambda_c\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "471faf4d", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T13:17:55.204754Z", + "start_time": "2022-02-03T13:17:54.923929Z" + } + }, + "outputs": [], + "source": [ + "plot.plot_criterions(\n", + " model,\n", + " thr_decrease=1000,\n", + " i_start=0, i_end=-1,\n", + " legend=True\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "292eb55f", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T13:17:55.223934Z", + "start_time": "2022-02-03T13:17:55.207349Z" + } + }, + "outputs": [], + "source": [ + "Z_model, W_model = model.best_partition(mode='likelihood', n_first=1)[0]\n", + "\n", + "metrics.print_metrics(\n", + " Z_model, W_model, y_, None,\n", + " absent_nodes=None,\n", + " print_each_timestep=False\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31cd63ba", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T13:17:55.695263Z", + "start_time": "2022-02-03T13:17:55.226095Z" + } + }, + "outputs": [], + "source": [ + "from sklearn.metrics import confusion_matrix\n", + "\n", + "Z_model, W_model = model.best_partition(mode='likelihood', n_first=1)[0]\n", + "cmat = metrics.cmat_clustering(confusion_matrix(Z_model, y_))\n", + "\n", + "f, ax = plt.subplots(figsize=(10, 10))\n", + "sns.heatmap(cmat, annot=True, fmt='.0f', ax=ax, square=True, cmap=sns.light_palette(\"red\"))\n", + "ax.set_title('confusion matrix');\n", + "ax.set_xlabel('predicted');\n", + "ax.set_ylabel('true');" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2e4e37b", + "metadata": { + "ExecuteTime": { + "end_time": "2022-02-03T13:17:56.792808Z", + "start_time": "2022-02-03T13:17:55.697194Z" + } + }, + "outputs": [], + "source": [ + "lw_cluster = 2.\n", + "\n", + "X_reorg = X.toarray()[np.ix_(np.argsort(Z_model), np.argsort(W_model))]\n", + "\n", + "f, ax = plt.subplots(figsize=(10, 10))\n", + "ax.spy(X_reorg, markersize=.1, precision=1)\n", + "\n", + "# plots the lines that separates the blocks\n", + "row_clusters, unique_row_indices = np.unique(Z_model, return_counts=True)\n", + "x_indices = np.cumsum(unique_row_indices)\n", + "for x in x_indices[:-1]:\n", + " ax.axhline(x, linewidth=lw_cluster)\n", + "\n", + "col_clusters, unique_col_indices = np.unique(W_model, return_counts=True)\n", + "y_indices = np.cumsum(unique_col_indices)\n", + "for x in y_indices[:-1]:\n", + " ax.axvline(x, linewidth=lw_cluster)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ecf49d8a", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "hide_input": false, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.15" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": { + "height": "calc(100% - 180px)", + "left": "10px", + "top": "150px", + "width": "480px" + }, + "toc_section_display": true, + "toc_window_display": true + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..cd84611 --- /dev/null +++ b/setup.py @@ -0,0 +1,64 @@ +import sys +import warnings +from setuptools import setup, find_packages + +with open("README.md", "r", encoding="utf-8") as fh: + long_description = fh.read() + +interactive = [ + 'notebook==5.7.10', + 'jupyter_contrib_nbextensions', + 'jupyter_nbextensions_configurator', + 'matplotlib', + 'networkx', + 'seaborn', + 'plotly', + 'pandas', + 'prince', # for Correspondence Analysis + 'nltk' # for notebook of text processing +] +metrics = ['sparsebm'] # for Co-clustering ARI (CARI) +tests = ['pytest', 'jinja2<3.1'] + +if (sys.version_info.major, sys.version_info.minor) != (3, 7): + warnings.warn( + 'Python version is different from 3.7 -> spherecluster package ' + 'will not be installed, so initializations with spherical k-means ' + 'will not be possible.' + ) + initialization = ['scikit-learn'] + base = [ + 'numpy', + 'scipy', + 'numba' + ] +else: + warnings.warn( + 'Python version is 3.7 -> spherecluster package can be installed ' + 'so initializations with spherical k-means are possible' + ) + initialization = ['spherecluster', 'scikit-learn==0.20'] + base = [ + 'numpy==1.21', + 'scipy', + 'numba' + ] + +all_extras = interactive + initialization + metrics + tests + +setup( + name='dcblockmodels', + version='1.0.0', + description='Dynamic and constrained block models', + install_requires=base, + extras_require={ + 'interactive': interactive, + 'intialization': initialization, + 'metrics': metrics, + 'all': all_extras + }, + long_description=long_description, + long_description_content_type='text/markdown', + tests_require=tests, + packages=find_packages() +)