kmeans

Author: quangh

Machine Learning/K-means/K-means Algorithm Mathematics base.md

@@ -0,0 +1,51 @@
+##### K-means Algorithm Mathematics base
+
+###### Problem:
+
+Given a data set $\{x_1, ..., x_n\} \in \mathbb{R^d}$ and a integer number K<=N. Our goal is to partition the data set into K clusters.
+
+-------------------------
+
+Let $\{u_i\} | u_i \in \mathbb{R^d}, i = 1,..,K$ be the set of center point of each cluster. Our goad is then to find an assignment of data points to clusters such that sum of the squares of the distances of each data point to its closest vector $u_k$ is minimum.
+
+Find min: 
+
+​										$||x_i - u_k||_2^2$
+
+For each data point $x_i$, we introduce a set $\{y_{ij} \in \{0,1\}\}$ where j = 1,...,K. If data point $x_i$ is assigned to cluster k then $y_{ik} = 1, and\: y_{ij} = 0\:for\: i\neq j $. Now, we can define an objective function:
+
+​								$\mathcal{L}(\mathbf{Y}, \mathbf{U}) = \sum_{i=1}^N\sum_{j=1}^K y_{ij} \|\mathbf{x}_i - \mathbf{u}_j\|_2^2$
+
+We can minimize this function through an iterative procedure in which each iteration
+involves two successive steps corresponding to successive optimizations with respect
+to the $y_{ij}$ and $u_k$. 
+
+- fixed U, find Y:
+
+  ​							$\mathbf{y}_i = \arg\min_{\mathbf{y}_i} \sum_{j=1}^K y_{ij}\|\mathbf{x}_i - \mathbf{u}_j\|_2^2 ~~~ (3)$
+
+  ​							$\text{subject to:} ~~ y_{ij} \in \{0, 1\}~~ \forall j;~~~ \sum_{j = 1}^K y_{ij} = 1$
+
+  ​							$ <=> j = \arg\min_{j} \|\mathbf{x}_i - \mathbf{u}_j\|_2^2$
+
+  In other words, we simply assign point $x_i$ to the closest cluster center.
+
+- fixed Y, find U:
+
+  ​							$\mathbf{u}_j = \arg\min_{\mathbf{u}_j} \sum_{i = 1}^{N} y_{ij}\|\mathbf{x}_i - \mathbf{u}_j \|_2^2.$
+
+  Objective function is a quadratic function of $u_j$ so it can be minimized by setting its derivative to 0:
+
+  ​							$\frac{\partial l(\mathbf{u}_j)}{\partial \mathbf{u}_j} = 2\sum_{i=1}^N y_{ij}(\mathbf{u}_j - \mathbf{x}_i) = 0$
+
+  ​							$\Rightarrow \mathbf{u}_j = \frac{ \sum_{i=1}^N y_{ij} \mathbf{x}_i}{\sum_{i=1}^N y_{ij}}$
+
+  ​
+
+  The denominator in this expression is equal to the number of points assigned to cluster k, the numerator is sum of all points of cluster j. So this result has a simple interpretation, namely set $u_j$ equal to the mean of all of the data points $x_i$ assigned to cluster j.
+
+  --------------------------
+
+  Because each phase reduces the value of the objective function, convergence of the algorithm is assured. However, it may converge to a local rather than global minimum. 
+
+  ​

Machine Learning/K-means/K-means Algorithm Mathematics base.pdf

@@ -0,0 +1,80 @@
+"""
+1.
+
+O()
+
+n : number of points
+K : number of clusters
+I : number of iterations
+d : number of attributes
+
+"""
+
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.spatial.distance import cdist
+
+means = [[0,0], [1,8], [5,2]]
+N = 500
+cov = [[1,0], [0,1]]
+X0 = np.random.multivariate_normal(means[0], cov, N)
+X1 = np.random.multivariate_normal(means[1], cov, N)
+X2 = np.random.multivariate_normal(means[2], cov, N)
+X = np.concatenate((X0,X1,X2),axis=0)
+
+
+def kmeans_display(X, label):
+    X0 = X[label == 0, :]
+    X1 = X[label == 1, :]
+    X2 = X[label == 2, :]
+
+    plt.plot(X0[:, 0], X0[:, 1], 'b^', markersize=4, alpha=.8)
+    plt.plot(X1[:, 0], X1[:, 1], 'go', markersize=4, alpha=.8)
+    plt.plot(X2[:, 0], X2[:, 1], 'rs', markersize=4, alpha=.8)
+
+    plt.axis('equal')
+    plt.plot()
+    plt.show()
+
+def init_center(X, k):
+    return X[np.random.choice(X.shape[0], k, replace=False)]
+
+def assign_labels(X, centers):
+    D = cdist(X, centers)
+    return np.argmin(D, axis=1)
+
+def update_centers(X, labels, K):
+    centers = np.zeros((K, X.shape[1]))
+    for k in range(K):
+        Xk = X[labels == k,:]
+        centers[k,:] = np.mean(Xk,axis=0)
+
+    return centers
+
+def is_converged(centers, new_centers):
+    return(set([tuple(a) for a in centers]) ==
+           set([tuple(a) for a in new_centers]))
+
+def kmeans(X,K):
+    centers = init_center(X,K)
+    labels = []
+    count = 0
+    while True:
+        labels = assign_labels(X, centers)
+        new_centers = update_centers(X, labels, K)
+        if is_converged(centers, new_centers):
+            break
+
+        count +=1
+        centers = new_centers
+
+    return centers, labels, count
+
+def bisecting_kmeans(X,K):
+    centers = [np.mean(X, axis=0).tolist()]
+    while len(centers) < K:
+        for cluster_center in centers:
+
+
+bisecting_kmeans(X,3)
+

Machine Learning/K-means/bisecting-k-means.py

@@ -0,0 +1,81 @@
+"""
+1. choose K random points as init centers
+2. assign each points to its closest center
+3. re-compute the centers
+4. if centers are unchanged, stop the algorithm
+5. back to step 2
+
+O(n * K * I * d)
+
+n : number of points
+K : number of clusters
+I : number of iterations
+d : number of attributes
+
+"""
+
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.spatial.distance import cdist
+
+means = [[0,0], [1,8], [5,2]]
+N = 500
+cov = [[1,0], [0,1]]
+X0 = np.random.multivariate_normal(means[0], cov, N)
+X1 = np.random.multivariate_normal(means[1], cov, N)
+X2 = np.random.multivariate_normal(means[2], cov, N)
+X = np.concatenate((X0,X1,X2),axis=0)
+
+
+def kmeans_display(X, label):
+    X0 = X[label == 0, :]
+    X1 = X[label == 1, :]
+    X2 = X[label == 2, :]
+
+    plt.plot(X0[:, 0], X0[:, 1], 'b^', markersize=4, alpha=.8)
+    plt.plot(X1[:, 0], X1[:, 1], 'go', markersize=4, alpha=.8)
+    plt.plot(X2[:, 0], X2[:, 1], 'rs', markersize=4, alpha=.8)
+
+    plt.axis('equal')
+    plt.plot()
+    plt.show()
+
+def init_center(X, k):
+    return X[np.random.choice(X.shape[0], k, replace=False)]
+
+def assign_labels(X, centers):
+    D = cdist(X, centers)
+    return np.argmin(D, axis=1)
+
+def update_centers(X, labels, K):
+    centers = np.zeros((K, X.shape[1]))
+    for k in range(K):
+        Xk = X[labels == k,:]
+        centers[k,:] = np.mean(Xk,axis=0)
+
+    return centers
+
+def is_converged(centers, new_centers):
+    return(set([tuple(a) for a in centers]) ==
+           set([tuple(a) for a in new_centers]))
+
+def kmean(X,K):
+    centers = init_center(X,K)
+    labels = []
+    count = 0
+    while True:
+        labels = assign_labels(X, centers)
+        new_centers = update_centers(X, labels, K)
+        if is_converged(centers, new_centers):
+            break
+
+        count +=1
+        centers = new_centers
+
+    return centers, labels, count
+
+centers , labels, count = kmean(X,3)
+
+print count
+kmeans_display(X, labels)
+