Release 7.033 && Fixes #434

techascent · Nov 16, 2024 · 12c9a96 · 12c9a96
1 parent 471b526
commit 12c9a96
Show file tree

Hide file tree

Showing 40 changed files with 67 additions and 52 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,7 @@
 # Changelog
+# 7.033
+ * [issue-434](https://github.com/techascent/tech.ml.dataset/issues/413) - bad transit encoding - packed instants are microseconds since epoch and have been for a while - not milliseconds since epoch.
+
 # 7.031
  * [issue-413](https://github.com/techascent/tech.ml.dataset/issues/413) - reduce with packed columns.
  * [issue-414](https://github.com/techascent/tech.ml.dataset/issues/414) - categorical maps are now integers.

diff --git a/deps.edn b/deps.edn
@@ -1,6 +1,6 @@
 {:paths ["src" "resources" "target/classes"]
  :deps {;;org.clojure/clojure {:mvn/version "1.11.1"}
-        cnuernber/dtype-next {:mvn/version "10.116"}
+        cnuernber/dtype-next {:mvn/version "10.120"}
         techascent/tech.io {:mvn/version "4.31"
                             :exclusions [org.apache.commons/commons-compress]}
         org.apache.datasketches/datasketches-java {:mvn/version "4.2.0"}
@@ -14,7 +14,7 @@
    :exec-fn codox.main/-main
    :exec-args {:group-id "techascent"
                :artifact-id "tech.ml.dataset"
-               :version "7.032"
+               :version "7.033"
                :name "TMD"
                :description "A Clojure high performance data processing system"
                :metadata {:doc/format :markdown}

diff --git a/docs/000-getting-started.html b/docs/000-getting-started.html
diff --git a/docs/100-walkthrough.html b/docs/100-walkthrough.html
diff --git a/docs/200-quick-reference.html b/docs/200-quick-reference.html
diff --git a/docs/columns-readers-and-datatypes.html b/docs/columns-readers-and-datatypes.html
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/nippy-serialization-rocks.html b/docs/nippy-serialization-rocks.html
diff --git a/docs/supported-datatypes.html b/docs/supported-datatypes.html
diff --git a/docs/tech.v3.dataset.categorical.html b/docs/tech.v3.dataset.categorical.html
diff --git a/docs/tech.v3.dataset.clipboard.html b/docs/tech.v3.dataset.clipboard.html
diff --git a/docs/tech.v3.dataset.column-filters.html b/docs/tech.v3.dataset.column-filters.html
diff --git a/docs/tech.v3.dataset.column.html b/docs/tech.v3.dataset.column.html
diff --git a/docs/tech.v3.dataset.html b/docs/tech.v3.dataset.html
diff --git a/docs/tech.v3.dataset.io.csv.html b/docs/tech.v3.dataset.io.csv.html
diff --git a/docs/tech.v3.dataset.io.datetime.html b/docs/tech.v3.dataset.io.datetime.html
diff --git a/docs/tech.v3.dataset.io.string-row-parser.html b/docs/tech.v3.dataset.io.string-row-parser.html
diff --git a/docs/tech.v3.dataset.io.univocity.html b/docs/tech.v3.dataset.io.univocity.html
diff --git a/docs/tech.v3.dataset.join.html b/docs/tech.v3.dataset.join.html
diff --git a/docs/tech.v3.dataset.math.html b/docs/tech.v3.dataset.math.html
diff --git a/docs/tech.v3.dataset.metamorph.html b/docs/tech.v3.dataset.metamorph.html
diff --git a/docs/tech.v3.dataset.modelling.html b/docs/tech.v3.dataset.modelling.html
diff --git a/docs/tech.v3.dataset.print.html b/docs/tech.v3.dataset.print.html
diff --git a/docs/tech.v3.dataset.reductions.apache-data-sketch.html b/docs/tech.v3.dataset.reductions.apache-data-sketch.html
diff --git a/docs/tech.v3.dataset.reductions.html b/docs/tech.v3.dataset.reductions.html
diff --git a/docs/tech.v3.dataset.rolling.html b/docs/tech.v3.dataset.rolling.html
diff --git a/docs/tech.v3.dataset.set.html b/docs/tech.v3.dataset.set.html
diff --git a/docs/tech.v3.dataset.tensor.html b/docs/tech.v3.dataset.tensor.html
diff --git a/docs/tech.v3.dataset.zip.html b/docs/tech.v3.dataset.zip.html
diff --git a/docs/tech.v3.libs.arrow.html b/docs/tech.v3.libs.arrow.html
diff --git a/docs/tech.v3.libs.clj-transit.html b/docs/tech.v3.libs.clj-transit.html
diff --git a/docs/tech.v3.libs.fastexcel.html b/docs/tech.v3.libs.fastexcel.html
diff --git a/docs/tech.v3.libs.guava.cache.html b/docs/tech.v3.libs.guava.cache.html
diff --git a/docs/tech.v3.libs.parquet.html b/docs/tech.v3.libs.parquet.html
diff --git a/docs/tech.v3.libs.poi.html b/docs/tech.v3.libs.poi.html
diff --git a/docs/tech.v3.libs.smile.data.html b/docs/tech.v3.libs.smile.data.html
diff --git a/docs/tech.v3.libs.tribuo.html b/docs/tech.v3.libs.tribuo.html
diff --git a/scripts/deploy b/scripts/deploy
@@ -2,7 +2,7 @@
 
 set -e
 
-scripts/run-tests
+scripts/run-tests-m1
 rm -rf pom.xml
 clojure -T:build jar
 cp target/classes/META-INF/maven/techascent/tech.ml.dataset/pom.xml .

diff --git a/src/tech/v3/libs/clj_transit.clj b/src/tech/v3/libs/clj_transit.clj
@@ -141,7 +141,7 @@
        (#{:packed-local-date :local-date} col-dt)
        (obj-col->numeric-b64 col :int32 dtype-dt/local-date->days-since-epoch)
        (#{:packed-instant :instant} col-dt)
-       (obj-col->numeric-b64 col :int64 dtype-dt/instant->milliseconds-since-epoch)
+       (obj-col->numeric-b64 col :int64 dtype-dt/instant->microseconds-since-epoch)
        :else ;;Punt!!
        (vec col))}))
 
@@ -260,12 +260,12 @@
 (def ^{:doc "Transit write handlers for java.time.LocalDate and java.time.Instant"}
   java-time-write-handlers
   {LocalDate (t/write-handler "java.time.LocalDate" dtype-dt/local-date->days-since-epoch)
-   Instant (t/write-handler "java.time.Instant" dtype-dt/instant->milliseconds-since-epoch)})
+   Instant (t/write-handler "java.time.Instant" dtype-dt/instant->microseconds-since-epoch)})
 
 (def ^{:doc "Transit read handlers for java.time.LocalDate and java.time.Instant"}
   java-time-read-handlers
   {"java.time.LocalDate" (t/read-handler dtype-dt/days-since-epoch->local-date)
-   "java.time.Instant" (t/read-handler dtype-dt/milliseconds-since-epoch->instant)})
+   "java.time.Instant" (t/read-handler dtype-dt/microseconds-since-epoch->instant)})
 
 
 (defn dataset->transit
@@ -315,17 +315,18 @@
 (comment
   (defn master-ds
     []
-    (ds/->dataset {:a (mapv double (range 5))
-                   :b (repeat 5 :a)
-                   :c (repeat 5 "hey")
-                   :d (repeat 5 {:a 1 :b 2})
-                   :e (repeat 4 [1 2 3])
-                   :f (repeat 5 (dtype-dt/local-date))
-                   :g (repeat 5 (dtype-dt/instant))
-                   :h [true false true true false]
-                   :i (repeat 5 "text")
-                   :j [1 nil 2 nil 3]}
+    (ds/->dataset (array-map :a (mapv double (range 5))
+                             :b (repeat 5 :a)
+                             :c (repeat 5 "hey")
+                             :d (repeat 5 {:a 1 :b 2})
+                             :e (repeat 4 [1 2 3])
+                             :f (repeat 5 (dtype-dt/local-date))
+                             :g (repeat 5 (dtype-dt/instant))
+                             :h [true false true true false]
+                             :i (repeat 5 "text")
+                             :j [1 nil 2 nil 3])
                   {:parser-fn {:i :text}}))
+
 
 
   (-> (master-ds)

diff --git a/test/tech/v3/dataset/parse_test.clj b/test/tech/v3/dataset/parse_test.clj
@@ -502,6 +502,17 @@
     (is (= (ds :b) (nds :b)))))
 
 
+(deftest issue-434-transit-support
+  (let [ds (ds/->dataset {:a [1 2 3]
+                          :b [:one :two :three]
+                          :c [(java.time.Instant/now) (java.time.Instant/now)]})
+        str-data (ds-transit/dataset->transit-str ds)
+        nds (ds-transit/transit-str->dataset str-data)]
+    (is (= (ds :a) (nds :a)))
+    (is (= (ds :b) (nds :b)))
+    (is (= (ds :c) (nds :c)))))
+
+
 (deftest issue-414-json-parser-fn
   (is (= [1 2 3] (get (ds/->dataset "test/data/local_date.json"
                                     {:parser-fn {:time-period :local-date}})