diff --git a/src/tmducken/duckdb.clj b/src/tmducken/duckdb.clj index bca6938..c091252 100644 --- a/src/tmducken/duckdb.clj +++ b/src/tmducken/duckdb.clj @@ -343,7 +343,7 @@ tmducken.duckdb> (get-config-options) ;;stack resource context is mainly for the string values. (resource/stack-resource-context (let [row-offset (* chunk chunk-size) - row-count (rem n-rows chunk-size) + row-count (min chunk-size (- n-rows row-offset)) n-valid (quot (+ row-count 63) 64) string-allocs (ArrayList.) reduce-groups (ArrayList.)] @@ -666,7 +666,8 @@ tmducken.duckdb> (get-config-options) (defn- destroy-chunk [chunk] - (duckdb-ffi/duckdb_destroy_data_chunk (dt-ffi/make-ptr :pointer (.address ^Pointer chunk)))) + (when (and chunk (not (== 0 (.address ^Pointer chunk)))) + (duckdb-ffi/duckdb_destroy_data_chunk (dt-ffi/make-ptr :pointer (.address ^Pointer chunk))))) (defn- reduce-chunk @@ -752,7 +753,9 @@ tmducken.duckdb> (get-config-options) (not (== 0 (.address ^Pointer chunk))) (not (reduced? acc))) (recur (reduce-chunk chunk realize-chunk reduce-type rfn acc)) - (if (reduced? acc) @acc acc))))) + (do + (destroy-chunk chunk) + (if (reduced? acc) @acc acc)))))) Iterable (iterator [this] (SupplierIter. this (.get this))) Seqable diff --git a/test/tmducken/duckdb_test.clj b/test/tmducken/duckdb_test.clj index f98da37..34d8841 100644 --- a/test/tmducken/duckdb_test.clj +++ b/test/tmducken/duckdb_test.clj @@ -1,6 +1,7 @@ (ns tmducken.duckdb-test (:require [clojure.test :refer [deftest is]] [tmducken.duckdb :as duckdb] + [tmducken.duckdb.ffi :as duckdb-ffi] [tech.v3.dataset :as ds] [tech.v3.datatype.functional :as dfn] [tech.v3.datatype.datetime :as dtype-dt] @@ -135,23 +136,33 @@ (catch Throwable e nil))))) (deftest insert-test - #_(println "========================================") - #_(println "insert-test") (let [cn 4 rn 1024 ds-fn #(-> (into {} (for [i (range cn)] [(str "c" i) (for [_ (range rn)] (str (random-uuid)))])) (ds/->dataset {:dataset-name "t"}) (ds/select-columns (for [i (range cn)] (str "c" i))))] - #_(println "drop-table!") (try (duckdb/drop-table! @conn* "t") (catch Throwable e nil)) (duckdb/create-table! @conn* (ds-fn)) - #_(println "insert-dataset! (first)") (duckdb/insert-dataset! @conn* (ds-fn)) - #_(println "insert-dataset! (second)") (duckdb/insert-dataset! @conn* (ds-fn)) - #_(println "insert-dataset! (sql->dataset)") + (is (= (* 2 rn) (-> (duckdb/sql->dataset @conn* "from t") + (ds/row-count)))))) + +(deftest insert-chunk-size-test + (let [cn 4 + rn (duckdb-ffi/duckdb_vector_size) + ds-fn #(-> (into {} (for [i (range cn)] [(str "c" i) + (for [_ (range rn)] (random-uuid))])) + (ds/->dataset {:dataset-name "t"}) + (ds/select-columns (for [i (range cn)] (str "c" i))))] + (try + (duckdb/drop-table! @conn* "t") + (catch Throwable e nil)) + (duckdb/create-table! @conn* (ds-fn)) + (duckdb/insert-dataset! @conn* (ds-fn)) + (duckdb/insert-dataset! @conn* (ds-fn)) (is (= (* 2 rn) (-> (duckdb/sql->dataset @conn* "from t") (ds/row-count))))))