From 493e55ee867f7ba88107d80af3e92e29859ba0f2 Mon Sep 17 00:00:00 2001 From: mhmerrill Date: Sat, 19 Oct 2019 10:49:34 -0400 Subject: [PATCH 1/5] added some initial READMEs for subdirectories --- benchmarks/README.md | 5 +++++ converter/README.md | 4 ++++ test/README.md | 4 ++++ tests/README.md | 5 +++++ toys/README.md | 4 ++++ 5 files changed, 22 insertions(+) create mode 100644 benchmarks/README.md create mode 100644 converter/README.md create mode 100644 test/README.md create mode 100644 tests/README.md create mode 100644 toys/README.md diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000000..59ffa4ff95 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,5 @@ +# Arkouda Python Benchmarks + + * depending on your setup you may need to set `PYTHONPATH` environment variable to find the `arkouda.py` module + * the `run_all.sh` script runs them against an `arkouda_server` running on `localhost:5555` + * check the benchmark source for runtime options diff --git a/converter/README.md b/converter/README.md new file mode 100644 index 0000000000..a3e967343c --- /dev/null +++ b/converter/README.md @@ -0,0 +1,4 @@ +# Python converter from CSV to HDF5 + + * example of conversion of the open LANL netflow data from CSV to HDF5 + diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000000..836f443bf8 --- /dev/null +++ b/test/README.md @@ -0,0 +1,4 @@ +# Some unit testing against the arkouda chapel modules + + * still under construction + diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000000..53dfb4fa28 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,5 @@ +# Python Tests against the arkouda_server + + * depending on your setup you may need to set `PYTHONPATH` environment variable to find the `arkouda.py` module + * check the python source for runtime options + \ No newline at end of file diff --git a/toys/README.md b/toys/README.md new file mode 100644 index 0000000000..ef71e0d8e4 --- /dev/null +++ b/toys/README.md @@ -0,0 +1,4 @@ +# Experiments and toys related to the development of Arkouda + + * provided as examples ONLY and are not guaranteed to still work + \ No newline at end of file From a3ed01ab4fa26ce2f8369ff6051c1ab64073ad22 Mon Sep 17 00:00:00 2001 From: mhmerrill Date: Sat, 19 Oct 2019 10:51:29 -0400 Subject: [PATCH 2/5] moved check.py to tests/ --- check.py => tests/check.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename check.py => tests/check.py (100%) diff --git a/check.py b/tests/check.py similarity index 100% rename from check.py rename to tests/check.py From 1d4c6bcfb1196aa320a221126313a2cdeaecdb3e Mon Sep 17 00:00:00 2001 From: mhmerrill Date: Sat, 19 Oct 2019 10:54:25 -0400 Subject: [PATCH 3/5] fixed typo in the comments of ak-based rmat generator --- toys/ak_bfs_conn_comp.py | 2 +- toys/ak_rmat.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/toys/ak_bfs_conn_comp.py b/toys/ak_bfs_conn_comp.py index 067e892c1a..ad2fdcca0d 100755 --- a/toys/ak_bfs_conn_comp.py +++ b/toys/ak_bfs_conn_comp.py @@ -49,7 +49,7 @@ def gen_rmat_edges(lgNv, Ne_per_v, p, perm=False): # # maybe: remove edges which are self-loops??? # - # return pair of ndarrays + # return pair of pdarrays return (ii,jj) # src and dst pdarrays hold the edge list diff --git a/toys/ak_rmat.py b/toys/ak_rmat.py index d4e81ce37f..67494ecc94 100755 --- a/toys/ak_rmat.py +++ b/toys/ak_rmat.py @@ -41,7 +41,7 @@ def gen_rmat_edges(lgNv, Ne_per_v, p, perm=False): # # maybe: remove edges which are self-loops??? # - # return pair of ndarrays + # return pair of pdarrays return (ii,jj) From fc87e5744612a161bbc0ae0f1137ccadc4d3e8d4 Mon Sep 17 00:00:00 2001 From: mhmerrill Date: Sat, 19 Oct 2019 14:29:40 -0400 Subject: [PATCH 4/5] initial implementation of expansion by bool index vector --- src/IndexingMsg.chpl | 72 ++++++++++++++++++++++++++++++++++++------ tests/check.py | 74 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 131 insertions(+), 15 deletions(-) diff --git a/src/IndexingMsg.chpl b/src/IndexingMsg.chpl index 2190bf4fa9..bb3b3bae39 100644 --- a/src/IndexingMsg.chpl +++ b/src/IndexingMsg.chpl @@ -262,9 +262,10 @@ module IndexingMsg var gIV: borrowed GenSymEntry = st.lookup(iname); if (gIV == nil) {return unknownSymbolError(pn,iname);} - proc idxToValHelper(type Xtype, type IVtype, type dtype): string { + // scatter indexing by integer index vector + proc ivInt64Helper(type Xtype, type dtype): string { var e = toSymEntry(gX,Xtype); - var iv = toSymEntry(gIV,IVtype); + var iv = toSymEntry(gIV,int); var ivMin = min reduce iv.a; var ivMax = max reduce iv.a; if ivMin < 0 {return try! "Error: %s: OOBindex %i < 0".format(pn,ivMin);} @@ -280,18 +281,42 @@ module IndexingMsg [i in iva] unorderedCopy(ea[i],val); return try! "%s success".format(pn); } - - // add check for IV to be dtype of int64 or bool + // expansion boolean indexing by bool index vector + proc ivBoolHelper(type Xtype, type dtype): string { + var e = toSymEntry(gX,Xtype); + var truth = toSymEntry(gIV,bool); + if (e.size != truth.size) {return try! "Error: %s: bool iv must be same size %i != %i".format(pn,e.size,truth.size);} + if isBool(dtype) { + value = value.replace("True","true"); // chapel to python bool + value = value.replace("False","false"); // chapel to python bool + } + var val = try! value:dtype; + ref ead = e.aD; + ref ea = e.a; + ref trutha = truth.a; + [i in ead] if (trutha[i] == true) {unorderedCopy(ea[i],val);} + return try! "%s success".format(pn); + } + select(gX.dtype, gIV.dtype, dtype) { when (DType.Int64, DType.Int64, DType.Int64) { - return idxToValHelper(int, int, int); + return ivInt64Helper(int, int); + } + when (DType.Int64, DType.Bool, DType.Int64) { + return ivBoolHelper(int, int); } when (DType.Float64, DType.Int64, DType.Float64) { - return idxToValHelper(real, int, real); + return ivInt64Helper(real, real); + } + when (DType.Float64, DType.Bool, DType.Float64) { + return ivBoolHelper(real, real); } when (DType.Bool, DType.Int64, DType.Bool) { - return idxToValHelper(bool, int, bool); + return ivInt64Helper(bool, bool); + } + when (DType.Bool, DType.Bool, DType.Bool) { + return ivBoolHelper(bool, bool); } otherwise {return notImplementedError(pn, "("+dtype2str(gX.dtype)+","+dtype2str(gIV.dtype)+","+dtype2str(dtype)+")");} @@ -317,11 +342,12 @@ module IndexingMsg var gY: borrowed GenSymEntry = st.lookup(yname); if (gY == nil) {return unknownSymbolError(pn,yname);} - // add check to make syre IV and Y are same size - if (gIV.size != gY.size) {return try! "Error: %s: size mismatch %i %i".format(pn,gIV.size, gY.size);} // add check for IV to be dtype of int64 or bool + // scatter indexing by an integer index vector proc ivInt64Helper(type t) { + // add check to make syre IV and Y are same size + if (gIV.size != gY.size) {return try! "Error: %s: size mismatch %i %i".format(pn,gIV.size,gY.size);} var e = toSymEntry(gX,t); var iv = toSymEntry(gIV,int); var ivMin = min reduce iv.a; @@ -337,16 +363,44 @@ module IndexingMsg return try! "%s success".format(pn); } + // expansion indexing by a bool index vector + proc ivBoolHelper(type t) { + // add check to make syre IV and Y are same size + if (gIV.size != gX.size) {return try! "Error: %s: size mismatch %i %i".format(pn,gIV.size,gX.size);} + var e = toSymEntry(gX,t); + var truth = toSymEntry(gIV,bool); + var iv: [truth.aD] int = (+ scan truth.a); + var pop = iv[iv.size-1]; + if v {writeln("pop = ",pop,"last-scan = ",iv[iv.size-1]);try! stdout.flush();} + var y = toSymEntry(gY,t); + if (y.size != pop) {return try! "Error: %s: pop size mismatch %i %i".format(pn,pop,y.size);} + ref ya = y.a; + ref ead = e.aD; + ref ea = e.a; + ref trutha = truth.a; + [i in ead] if (trutha[i] == true) {unorderedCopy(ea[i],ya[iv[i]-1]);} + return try! "%s success".format(pn); + } + select(gX.dtype, gIV.dtype, gY.dtype) { when (DType.Int64, DType.Int64, DType.Int64) { return ivInt64Helper(int); } + when (DType.Int64, DType.Bool, DType.Int64) { + return ivBoolHelper(int); + } when (DType.Float64, DType.Int64, DType.Float64) { return ivInt64Helper(real); } + when (DType.Float64, DType.Bool, DType.Float64) { + return ivBoolHelper(real); + } when (DType.Bool, DType.Int64, DType.Bool) { return ivInt64Helper(bool); } + when (DType.Bool, DType.Bool, DType.Bool) { + return ivBoolHelper(bool); + } otherwise {return notImplementedError(pn, "("+dtype2str(gX.dtype)+","+dtype2str(gIV.dtype)+","+dtype2str(gY.dtype)+")");} } diff --git a/tests/check.py b/tests/check.py index c9211ae9f4..cc0ebc1557 100755 --- a/tests/check.py +++ b/tests/check.py @@ -86,20 +86,35 @@ def check_get_slice(N): print("check get slice [::2] :", check_get_slice(N)) +def check_set_slice_value(N): + # create np version + a = np.ones(N) + a[::2] = -1 + a = ak.array(a) + # create ak version + b = ak.ones(N) + b[::2] = -1 + # print(a,b) + c = a == b + # print(type(c),c) + return pass_fail(c.all()) + +print("check set slice [::2] = value:", check_set_slice_value(N)) + def check_set_slice(N): # create np version a = np.ones(N) - a[::2] = 0 + a[::2] = a[::2] * -1 a = ak.array(a) # create ak version b = ak.ones(N) - b[::2] = 0 + b[::2] = b[::2] * -1 # print(a,b) c = a == b # print(type(c),c) return pass_fail(c.all()) -print("check set slice [::2] :", check_set_slice(N)) +print("check set slice [::2] = pda:", check_set_slice(N)) def check_get_bool_iv(N): # create np version @@ -116,6 +131,36 @@ def check_get_bool_iv(N): print("check (compressing) get bool iv :", check_get_bool_iv(N)) +def check_set_bool_iv_value(N): + # create np version + a = np.arange(N) + a[a < N//2] = -1 + a = ak.array(a) + # create ak version + b = ak.arange(N) + b[b < N//2] = -1 + # print(a,b) + c = a == b + # print(type(c),c) + return pass_fail(c.all()) + +print("check (expanding) set bool iv = value:", check_set_bool_iv_value(N)) + +def check_set_bool_iv(N): + # create np version + a = np.arange(N) + a[a < N//2] = a[:N//2] * -1 + a = ak.array(a) + # create ak version + b = ak.arange(N) + b[b < N//2] = b[:N//2] * -1 + # print(a,b) + c = a == b + # print(type(c),c) + return pass_fail(c.all()) + +print("check (expanding) set bool iv = pda:", check_set_bool_iv(N)) + def check_get_integer_iv(N): # create np version a = np.arange(N) @@ -131,7 +176,24 @@ def check_get_integer_iv(N): # print(type(c),c) return pass_fail(c.all()) -print("check (gather) get integer iv :", check_get_integer_iv(N)) +print("check (gather) get integer iv:", check_get_integer_iv(N)) + +def check_set_integer_iv_value(N): + # create np version + a = np.arange(N) + iv = np.arange(N//2) + a[iv] = -1 + a = ak.array(a) + # create ak version + b = ak.arange(N) + iv = ak.arange(N//2) + b[iv] = -1 + # print(a,b) + c = a == b + # print(type(c),c) + return pass_fail(c.all()) + +print("check (scatter) set integer iv = value:", check_set_integer_iv_value(N)) def check_set_integer_iv(N): # create np version @@ -148,7 +210,7 @@ def check_set_integer_iv(N): # print(type(c),c) return pass_fail(c.all()) -print("check (scatter) set integer iv :", check_set_integer_iv(N)) +print("check (scatter) set integer iv = pda:", check_set_integer_iv(N)) def check_get_integer_idx(N): # create np version @@ -172,7 +234,7 @@ def check_set_integer_idx(N): v2 = b[N//2] return pass_fail(v1 == v2) -print("check set integer idx :", check_set_integer_idx(N)) +print("check set integer idx = value:", check_set_integer_idx(N)) def check_argsort(N): # create np version From ec3ed0ba80664a505a6a3dee270366a14b3db58d Mon Sep 17 00:00:00 2001 From: mhmerrill Date: Sat, 19 Oct 2019 15:13:56 -0400 Subject: [PATCH 5/5] updated check.py --- tests/check.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/check.py b/tests/check.py index cc0ebc1557..39e3a2f0b9 100755 --- a/tests/check.py +++ b/tests/check.py @@ -71,6 +71,25 @@ def check_zeros(N): print("check zeros :", check_zeros(N)) +def check_argsort(N): + # create np version + a = np.arange(N) + a = a[::-1] + iv = np.argsort(a) + a = a[iv] + a = ak.array(a) + # create ak version + b = ak.arange(N) + b = b[::-1] + iv = ak.argsort(b) + b = b[iv] + # print(a,b) + c = a == b + # print(type(c),c) + return pass_fail(c.all()) + +print("check argsort :", check_argsort(N)) + def check_get_slice(N): # create np version a = np.ones(N) @@ -236,24 +255,5 @@ def check_set_integer_idx(N): print("check set integer idx = value:", check_set_integer_idx(N)) -def check_argsort(N): - # create np version - a = np.arange(N) - a = a[::-1] - iv = np.argsort(a) - a = a[iv] - a = ak.array(a) - # create ak version - b = ak.arange(N) - b = b[::-1] - iv = ak.argsort(b) - b = b[iv] - # print(a,b) - c = a == b - # print(type(c),c) - return pass_fail(c.all()) - -print("check argsort :", check_argsort(N)) - #ak.disconnect() ak.shutdown()