Michelp/v4.0.0draft5 (#76)

* hypersparse dnn checkpoint.

* better to_image function

* test fixups.

* hypersparse dnn checkpoint.

* update lagraph and 4.0.0 rework.

* a few test failures yet but most of 4.0.0 including positional ops and new lagraph.

* fix bad error call and add get_implemetation/spec_version() and setup local script.

* 4.0.1

* positional binary and unary ops

* fixup notebook docker.

* tracking more 4.0.1 changes, added conda environment.

* tracking more 4.0.1 changes, added conda environment.

* env for binder.

* GxB_HYPER_SWITCH

* add getter/setters for options and remove old functions, black

* new options, new official 4.0.1 header file.

* version updates

* add implementation and spec tuples.

* travis ci fixes.

* more travis fixes.

.travis.yml

@@ -2,5 +2,5 @@ gitservices:
   - docker
 
 script:
-  - SS_COMPACT=1 ./docker_build.sh v3.3.3 minimal
+  - SS_COMPACT=1 ./docker_build.sh v4.0.1 test minimal
   - ./test.sh

Dockerfile-manim

@@ -19,7 +19,7 @@ RUN apt-get update \
 		dvisvgm \
     && rm -rf /var/lib/apt/lists/*
 
-ARG SS_RELEASE=3.3.3
+ARG SS_RELEASE=v4.0.1
 ARG SS_BURBLE=0
 ARG SS_COMPACT=0
 

Dockerfile-minimal

@@ -2,17 +2,16 @@ ARG BASE_CONTAINER=ubuntu:20.04
 FROM ${BASE_CONTAINER}
 ENV DEBIAN_FRONTEND=noninteractive
 
-RUN apt-get update && apt-get install -yq cmake make wget libpython3-dev python3-pip libreadline-dev llvm-10-dev
+RUN apt-get update && apt-get install -yq cmake make wget libpython3-dev python3-pip libreadline-dev llvm-10-dev git
 
-ARG SS_RELEASE=3.3.3
+ARG SS_RELEASE=v4.0.1
 ARG SS_BURBLE=0
 ARG SS_COMPACT=0
 
 WORKDIR /build
-RUN wget --quiet https://github.com/DrTimothyAldenDavis/GraphBLAS/archive/v$SS_RELEASE.tar.gz
-RUN tar -xf v$SS_RELEASE.tar.gz
+RUN git clone https://github.com/DrTimothyAldenDavis/GraphBLAS.git --depth 1 --branch $SS_RELEASE
 
-WORKDIR /build/GraphBLAS-$SS_RELEASE/build
+WORKDIR /build/GraphBLAS/build
 RUN cmake .. -DGB_BURBLE=${SS_BURBLE} -DGBCOMPACT=${SS_COMPACT} && make -j8 && make install
 RUN ldconfig
 RUN /bin/rm -Rf /build
@@ -23,4 +22,4 @@ WORKDIR /pygraphblas
 RUN pip3 install -r minimal-requirements.txt
 RUN python3 setup.py install
 RUN ldconfig
-RUN pytest
+# RUN pytest

Dockerfile-notebook

@@ -6,29 +6,35 @@ USER root
 
 RUN apt-get update && apt-get install -yq cmake make wget libpython3-dev python3-pip libreadline-dev
 
-ARG SS_RELEASE=3.3.3
+ARG SS_RELEASE=v4.0.1
 ARG SS_BURBLE=0
 ARG SS_COMPACT=0
 
 WORKDIR /build
-RUN wget --quiet https://github.com/DrTimothyAldenDavis/GraphBLAS/archive/v$SS_RELEASE.tar.gz
-RUN tar -xf v$SS_RELEASE.tar.gz
+RUN git clone https://github.com/DrTimothyAldenDavis/GraphBLAS.git --depth 1 --branch $SS_RELEASE
 
-WORKDIR /build/GraphBLAS-$SS_RELEASE/build
-RUN cmake .. -DGB_BURBLE=${SS_BURBLE} -DGBCOMPACT=${SS_COMPACT} && make JOBS=8 && make install
+WORKDIR /build/GraphBLAS/build
+RUN cmake .. -DGB_BURBLE=${SS_BURBLE} -DGBCOMPACT=${SS_COMPACT} && make -j8 && make install
+RUN ldconfig
 RUN /bin/rm -Rf /build
 
 RUN conda install -y graphviz
 
+WORKDIR /build
+RUN git clone https://github.com/fcdimitr/fglt.git
+WORKDIR /build/fglt
+RUN pip install -r requirements.txt
+RUN meson build && cd build && ninja install
+
 ADD . /home/jovyan
 WORKDIR /home/jovyan
 
 RUN python setup.py develop
 RUN pip install -r notebook-requirements.txt
-RUN jupyter nbextension install rise --py --sys-prefix
-RUN jupyter nbextension enable rise --py --sys-prefix
+#RUN jupyter nbextension install rise --py --sys-prefix
+#RUN jupyter nbextension enable rise --py --sys-prefix
 RUN chown -R jovyan /home/jovyan
 
 RUN ldconfig
 USER jovyan
-RUN pytest
+# RUN pytest

dnn.sh

@@ -37,5 +37,5 @@ cd -
 
 docker run --rm --env DEST=/$1 --env NEURONS=$2 --env NLAYERS=$3 \
        -v `pwd`/$1:/$1 \
-       -v `pwd`/pygraphblas:/pygraphblas/pygraphblas \
-       -it graphblas/pygraphblas-minimal ipython -i -m pygraphblas.demo.dnn
+       -v `pwd`/pygraphblas:/home/jovyan/pygraphblas \
+       -it graphblas/pygraphblas-notebook ipython -i -m pygraphblas.demo.dnn

docker_build.sh

@@ -1,7 +1,6 @@
 if [ $# -eq 0 ]
     then
-        echo "Usage: ./docker_build.sh SS_RELEASE BASE_NAME BRANCH [LOCATION PUSH]"
-        echo "Note: BASE_NAME=minimal image does not use SS_RELEASE parameter"
+        echo "Usage: ./docker_build.sh SS_RELEASE PY_RELEASE BASE_NAME BRANCH [LOCATION PUSH]"
         echo
         echo "Example: ./docker_build.sh v3.3.3 v3.4.0 notebook master clone push"
         exit 1

environment.yml

@@ -0,0 +1,30 @@
+name: pygraphblas
+channels:
+  - conda-forge
+dependencies:
+  - pygraphblas
+  - numpy
+  - cffi
+  - numba
+  - scipy
+  - pip
+  - pip:
+    - pytest
+    - pytest-cov
+    - ipdb
+    - graphviz
+    - pandas
+    - networkx
+    - py_d3
+    - git+https://github.com/taynaud/python-louvain.git
+    - pypng
+    - numpngw
+    - ipycytoscape
+    - Pillow
+    - matplotlib
+    - colorcet
+    - imageio
+    - imageio-ffmpeg
+    - pygifsicle
+
+

install-ubuntu.sh

@@ -8,7 +8,7 @@ sudo apt install -y curl m4 g++
 
 # Install the required version of SuiteSparse:GraphBLAS
 
-SS_RELEASE=v3.3.1
+SS_RELEASE=v4.0.1
 SS_BURBLE=0
 SS_COMPACT=0
 

minimal-requirements.txt

@@ -1,6 +1,8 @@
 numpy
+scipy
 numba
 pytest
 pytest-cov
 setuptools
 ipython
+contextvars

notebook-requirements.txt

@@ -8,3 +8,13 @@ networkx
 scipy
 py_d3
 git+https://github.com/taynaud/python-louvain.git
+pypng
+numpngw
+ipycytoscape
+Pillow
+matplotlib
+colorcet
+imageio
+imageio-ffmpeg
+pygifsicle
+

pygraphblas/base.py

@@ -22,12 +22,29 @@
     "IndexOutOfBound",
     "Panic",
     "options_set",
+    "options_get",
+    "GxB_IMPLEMENTATION",
 ]
 
 NULL = ffi.NULL
 
+GxB_IMPLEMENTATION = (
+    lib.GxB_IMPLEMENTATION_MAJOR,
+    lib.GxB_IMPLEMENTATION_MINOR,
+    lib.GxB_IMPLEMENTATION_SUB,
+)
 
-def options_set(nthreads=None, chunk=None, burble=None):
+GxB_SPEC = (lib.GxB_SPEC_MAJOR, lib.GxB_SPEC_MINOR, lib.GxB_SPEC_SUB)
+
+
+def options_set(
+    nthreads=None,
+    chunk=None,
+    burble=None,
+    hyper_switch=None,
+    bitmap_switch=None,
+    format=None,
+):
     if nthreads is not None:
         nthreads = ffi.cast("int", nthreads)
         _check(lib.GxB_Global_Option_set(lib.GxB_GLOBAL_NTHREADS, nthreads))
@@ -37,6 +54,39 @@ def options_set(nthreads=None, chunk=None, burble=None):
     if burble is not None:
         burble = ffi.cast("int", burble)
         _check(lib.GxB_Global_Option_set(lib.GxB_BURBLE, burble))
+    if hyper_switch is not None:
+        hyper_switch = ffi.cast("double", hyper_switch)
+        _check(lib.GxB_Global_Option_set(lib.GxB_HYPER_SWITCH, hyper_switch))
+    if bitmap_switch is not None:
+        bitmap_switch = ffi.new("double[8]", bitmap_switch)
+        _check(lib.GxB_Global_Option_set(lib.GxB_BITMAP_SWITCH, bitmap_switch))
+    if format is not None:
+        format = ffi.cast("GxB_Format_Value*", format)
+        _check(lib.GxB_Global_Option_set(lib.GxB_FORMAT, format))
+
+
+def options_get():
+    nthreads = ffi.new("int*")
+    _check(lib.GxB_Global_Option_get(lib.GxB_GLOBAL_NTHREADS, nthreads))
+    chunk = ffi.new("double*")
+    _check(lib.GxB_Global_Option_get(lib.GxB_GLOBAL_CHUNK, chunk))
+    burble = ffi.new("int*")
+    _check(lib.GxB_Global_Option_get(lib.GxB_BURBLE, burble))
+    hyper_switch = ffi.new("double*")
+    _check(lib.GxB_Global_Option_get(lib.GxB_HYPER_SWITCH, hyper_switch))
+    bitmap_switch = ffi.new("double[8]")
+    _check(lib.GxB_Global_Option_get(lib.GxB_BITMAP_SWITCH, bitmap_switch))
+    format = ffi.new("GxB_Format_Value*")
+    _check(lib.GxB_Global_Option_get(lib.GxB_FORMAT, format))
+
+    return dict(
+        nthreads=nthreads[0],
+        chunk=chunk[0],
+        burble=burble[0],
+        hyper_switch=hyper_switch[0],
+        bitmap_switch=list(bitmap_switch),
+        format=format[0],
+    )
 
 
 class GraphBLASException(Exception):
@@ -116,7 +166,7 @@ def _check(res, raise_no_val=False):
     if res != lib.GrB_SUCCESS:
         if raise_no_val and res == lib.GrB_NO_VALUE:
             raise KeyError
-        raise _error_codes[res](ffi.string(lib.GrB_error()))
+        raise _error_codes[res]()
 
 
 def _build_range(rslice, stop_val):

pygraphblas/binaryop.py

@@ -87,6 +87,7 @@ def __exit__(self, *errors):
 
 grb_binop_re = re.compile(
     "^(GrB|GxB)_(FIRST|SECOND|MIN|MAX|PLUS|MINUS|RMINUS|TIMES|DIV|RDIV|"
+    "FIRSTI|FIRSTI1|FIRSTJ|FIRSTJ1|SECONDI|SECONDI1|SECONDJ|SECONDJ1|"
     "PAIR|ANY|POW|EQ|NE|GT|LT|GE|LE|LOR|LAND|LXOR|BOR|BAND|BXOR|BXNOR|"
     "ATAN2|HYPOT|FMOD|REMAINDER|LDEXP|COPYSIGN|BGET|BSET|BCLR|BSHIFT|CMPLX)_"
     "(BOOL|UINT8|UINT16|UINT32|UINT64|INT8|INT16|INT32|INT64|FP32|FP64)$"

pygraphblas/build.py

@@ -9,6 +9,8 @@ def build_ffi():
         #include <math.h>
         #include <stdint.h>
         #include <complex.h>
+
+        #define LAGRAPH_EXPERIMENTAL_ASK_BEFORE_BENCHMARKING
         """
     p = Path("pygraphblas/cdef")
     l = Path("pygraphblas/cdef/LAGraph")
@@ -48,10 +50,10 @@ def build_ffi():
         ],
     )
 
-    with open(p / "GraphBLAS-3.3.3.h") as gb_cdef:
+    with open(p / "GraphBLAS-4.0.1.h") as gb_cdef:
         ffibuilder.cdef(gb_cdef.read())
 
-    with open(p / "la_a6fcf0_cdef.h") as la_cdef:
+    with open(p / "lagraph_c2c0cf_cdef.h") as la_cdef:
         ffibuilder.cdef(la_cdef.read())
 
     with open(p / "extra.h") as ex_cdef: