Added compatibility with persistent Storage Numpys from Hecuba

.travis.yml

@@ -5,7 +5,7 @@ sudo: required
 
 branches:
   only:
-    - master
+    - test_compss
     - /^release-.*/
 
 services:
@@ -14,24 +14,27 @@ env:
   global:
     - REGISTRY_USER=compss
     - secure: ""
+    - TEST_CASSANDRA_VERSION=3.11.4
 
 before_script:
-    - docker build --tag bscwdc/dislib .
-    - docker run $(bash <(curl -s https://codecov.io/env)) -d --name dislib bscwdc/dislib
-
-script: "docker exec dislib /dislib/run_ci_checks.sh"
-
-after_script:
-  - docker images
-  - docker exec dislib /dislib/bin/print_tests_logs.sh
-
-before_deploy:
-  - docker login -u "$REGISTRY_USER" -p "$REGISTRY_PASS"
-  - docker tag bscwdc/dislib bscwdc/dislib:latest
-deploy:
-  provider: script
-  script: docker push bscwdc/dislib:latest
-  on:
-    branch: master
+    - source launch_cassandra.sh
+    - docker build --tag emebemb/dislib_hecuba_compss_production:0.2 .
+    - docker run -it --network cassandra_bridge -d --name dislib emebemb/dislib_hecuba_compss_production:0.2
+
+
+script: "docker exec -e CONTACT_NAMES='cassandra_container' -e NODE_PORT=9042 dislib /dislib/run_tests.sh"
+
+#after_script:
+#  - docker images
+#  - docker exec dislib /dislib/bin/print_tests_logs.sh
+#
+#before_deploy:
+#  - docker login -u "$REGISTRY_USER" -p "$REGISTRY_PASS"
+#  - docker tag bscwdc/dislib bscwdc/dislib:latest
+#deploy:
+#  provider: script
+#  script: docker push bscwdc/dislib:latest
+#  on:
+#    branch: master
 
 

Dockerfile

@@ -1,6 +1,17 @@
-FROM bscwdc/dislib-base:latest
+#FROM bscwdc/dislib-base:latest
+FROM adrianespejo/dislib_hecuba:0.1
 MAINTAINER COMPSs Support <[email protected]>
 
+#RUN apt-get update -y && apt-get update
+#RUN apt-get install -y cmake python3-dev libpython3-dev gcc-4.8 libtool python3-numpy python3-pip python3-setuptools
+#RUN curl -L https://github.com/bsc-dd/hecuba/archive/NumpyWritePartitions.tar.gz | tar -xz
+
+#WORKDIR hecuba-NumpyWritePartitions
+#RUN python3 -m pip install -r requirements.txt
+#RUN python3 setup.py install
+WORKDIR /
+
+#RUN rm -rf dislib/
 COPY . dislib/
 
 ENV PYTHONPATH=$PYTHONPATH:/dislib

counter

@@ -0,0 +1 @@
+2

dislib/__init__.py

@@ -1,7 +1,7 @@
 import os
 
 from dislib.data.array import random_array, apply_along_axis, array, zeros, \
-    full, identity
+    full, load_from_hecuba, identity
 from dislib.data.io import load_svmlight_file, load_npy_file, load_txt_file
 from dislib.math import kron, svd
 

dislib/cluster/kmeans/base.py

@@ -1,6 +1,6 @@
 import numpy as np
 from pycompss.api.api import compss_wait_on
-from pycompss.api.parameter import COLLECTION_IN, Depth, Type
+from pycompss.api.parameter import INOUT, COLLECTION_IN, Depth, Type
 from pycompss.api.task import task
 from scipy.sparse import csr_matrix
 from sklearn.base import BaseEstimator
@@ -10,10 +10,8 @@
 
 from dislib.data.array import Array
 
-
 class KMeans(BaseEstimator):
     """ Perform K-means clustering.
-
     Parameters
     ----------
     n_clusters : int, optional (default=8)
@@ -22,7 +20,6 @@ class KMeans(BaseEstimator):
     init : {'random', nd-array or sparse matrix}, optional (default='random')
         Method of initialization, defaults to 'random', which generates
         random centers at the beginning.
-
         If an nd-array or sparse matrix is passed, it should be of shape
         (n_clusters, n_features) and gives the initial centers.
     max_iter : int, optional (default=10)
@@ -37,14 +34,12 @@ class KMeans(BaseEstimator):
         for centroid initialization.
     verbose: boolean, optional (default=False)
         Whether to print progress information.
-
     Attributes
     ----------
     centers : ndarray
         Computed centroids.
     n_iter : int
         Number of iterations performed.
-
     Examples
     --------
     >>> from dislib.cluster import KMeans
@@ -73,14 +68,12 @@ def __init__(self, n_clusters=8, init='random', max_iter=10, tol=1e-4,
 
     def fit(self, x, y=None):
         """ Compute K-means clustering.
-
         Parameters
         ----------
         x : ds-array
             Samples to cluster.
         y : ignored
             Not used, present here for API consistency by convention.
-
         Returns
         -------
         self : KMeans
@@ -95,6 +88,7 @@ def fit(self, x, y=None):
             old_centers = self.centers.copy()
             partials = []
 
+
             for row in x._iterator(axis=0):
                 partial = _partial_sum(row._blocks, old_centers)
                 partials.append(partial)
@@ -108,31 +102,26 @@ def fit(self, x, y=None):
 
     def fit_predict(self, x, y=None):
         """ Compute cluster centers and predict cluster index for each sample.
-
         Parameters
         ----------
         x : ds-array
             Samples to cluster.
         y : ignored
             Not used, present here for API consistency by convention.
-
         Returns
         -------
         labels : ds-array, shape=(n_samples, 1)
             Index of the cluster each sample belongs to.
         """
-
         self.fit(x)
         return self.predict(x)
 
     def predict(self, x):
         """ Predict the closest cluster each sample in the data belongs to.
-
         Parameters
         ----------
         x : ds-array
             New data to predict.
-
         Returns
         -------
         labels : ds-array, shape=(n_samples, 1)
@@ -193,7 +182,6 @@ def _init_centers(self, n_features, sparse):
 def _partial_sum(blocks, centers):
     partials = np.zeros((centers.shape[0], 2), dtype=object)
     arr = Array._merge_blocks(blocks)
-
     close_centers = pairwise_distances(arr, centers).argmin(axis=1)
 
     for center_idx, _ in enumerate(centers):
@@ -204,6 +192,8 @@ def _partial_sum(blocks, centers):
     return partials
 
 
+
+
 @task(returns=dict)
 def _merge(*data):
     accum = data[0].copy()
@@ -217,4 +207,4 @@ def _merge(*data):
 @task(blocks={Type: COLLECTION_IN, Depth: 2}, returns=np.array)
 def _predict(blocks, centers):
     arr = Array._merge_blocks(blocks)
-    return pairwise_distances(arr, centers).argmin(axis=1).reshape(-1, 1)
+    return pairwise_distances(arr, centers).argmin(axis=1).reshape(-1, 1)

dislib/data/__init__.py

@@ -1,6 +1,6 @@
 from dislib.data.array import array, random_array, apply_along_axis, zeros, \
-    full, identity
+    full, load_from_hecuba, identity
 from dislib.data.io import load_txt_file, load_npy_file, load_svmlight_file
 
 __all__ = ['load_txt_file', 'load_svmlight_file', 'array', 'random_array',
-           'apply_along_axis', 'load_npy_file', 'zeros', 'full', 'identity']
+           'apply_along_axis', 'load_from_hecuba', 'load_npy_file', 'zeros', 'full', 'identity']