[WIP] Implement the delete-history Command

.github/scripts/delete.sh

@@ -11,11 +11,11 @@ test_empty "content type header" "$(echo "$HEADERS" | grep -i content-type | tr
 
 PATIENT_HISTORY=$(curl -s "$BASE/Patient/$PATIENT_ID/_history")
 
-TOTAL=$(echo "$PATIENT_HISTORY" | jq .total)
-if [ "$TOTAL" = "1" ]; then
+NUM_ENTRIES=$(echo "$PATIENT_HISTORY" | jq '.entry | length')
+if [ "$NUM_ENTRIES" = "1" ]; then
   echo "OK 👍: patient history has one entry"
 else
-  echo "Fail 😞: patient history has $TOTAL entries"
+  echo "Fail 😞: patient history has $NUM_ENTRIES entries"
   exit 1
 fi
 

modules/coll/src/blaze/coll/core.clj

@@ -21,7 +21,7 @@
   [coll]
   (identical? ::empty (reduce #(reduced %2) ::empty coll)))
 
-(defn- inc-rf [sum _] (inc ^long sum))
+(defn inc-rf [sum _] (inc ^long sum))
 
 (defn eduction
   "Like `clojure.core/eduction` but implements Counted instead of Iterable."
@@ -67,4 +67,7 @@
      IReduceInit
      (reduce [_ rf# init#]
        (with-open ~bindings
-         (reduce rf# init# ~coll)))))
+         (reduce rf# init# ~coll)))
+     Counted
+     (count [coll#]
+       (.reduce coll# inc-rf 0))))

modules/db-protocols/src/blaze/db/impl/protocols.clj

@@ -37,8 +37,6 @@
 
   (-instance-history [db tid id start-t])
 
-  (-total-num-of-instance-changes [_ tid id since])
-
   (-type-history [db type start-t start-id])
 
   (-total-num-of-type-changes [db type since])

modules/db-tx-log/src/blaze/db/tx_log/spec.clj

@@ -19,7 +19,7 @@
   queue?)
 
 (s/def :blaze.db.tx-cmd/op
-  #{"create" "put" "keep" "delete" "conditional-delete"})
+  #{"create" "put" "keep" "delete" "conditional-delete" "delete-history"})
 
 (s/def :blaze.db.tx-cmd/type
   :fhir.resource/type)
@@ -87,6 +87,11 @@
                    :blaze.db.tx-cmd/check-refs
                    :blaze.db.tx-cmd/allow-multiple]))
 
+(defmethod tx-cmd "delete-history" [_]
+  (s/keys :req-un [:blaze.db.tx-cmd/op
+                   :blaze.db.tx-cmd/type
+                   :blaze.resource/id]))
+
 (s/def :blaze.db/tx-cmd
   (s/multi-spec tx-cmd :op))
 

modules/db-tx-log/test/blaze/db/tx_log/spec_test.clj

@@ -11,7 +11,7 @@
    [taoensso.timbre :as log]))
 
 (st/instrument)
-(log/set-level! :trace)
+(log/set-min-level! :trace)
 
 (test/use-fixtures :each tu/fixture)
 
@@ -36,23 +36,49 @@
        :id "0"
        :hash observation-hash-0
        :refs [["Patient" "0"]]}
+      {:op "put"
+       :type "Patient"
+       :id "0"
+       :hash patient-hash-0}
       {:op "put"
        :type "Patient"
        :id "0"
        :hash patient-hash-0
        :if-match 1}
+      {:op "keep"
+       :type "Patient"
+       :id "0"
+       :hash patient-hash-0}
       {:op "delete"
        :type "Patient"
        :id "0"
        :if-match 1}
       {:op "delete"
        :type "Patient"
        :id "0"
-       :check-refs true}))
+       :check-refs true}
+      {:op "conditional-delete"
+       :type "Patient"}
+      {:op "delete-history"
+       :type "Patient"
+       :id "0"}))
 
   (testing "invalid"
     (are [tx-cmd] (not (s/valid? :blaze.db/tx-cmd tx-cmd))
+      nil
+      1
+      {:op "create"
+       :type "Patient"
+       :id "0"}
+      {:op "put"
+       :type "Patient"
+       :id "0"}
+      {:op "delete"
+       :type "Patient"}
       {:op "delete"
        :type "Patient"
        :id "0"
-       :check-refs "i should be a boolean"})))
+       :check-refs "i should be a boolean"}
+      {:op "conditional-delete"}
+      {:op "delete-history"
+       :type "Patient"})))

modules/db/src/blaze/db/api.clj

@@ -349,19 +349,6 @@
   ([db type id start-t]
    (p/-instance-history db (codec/tid type) (codec/id-byte-string id) start-t)))
 
-(defn total-num-of-instance-changes
-  "Returns the total number of changes (versions) of the resource with the given
-  `type` and `id` starting as-of `db`.
-
-  Optionally a `since` instant can be given to define a point in the past where
-  the calculation should start."
-  ([db type id]
-   (p/-total-num-of-instance-changes db (codec/tid type)
-                                     (codec/id-byte-string id) nil))
-  ([db type id since]
-   (p/-total-num-of-instance-changes db (codec/tid type)
-                                     (codec/id-byte-string id) since)))
-
 ;; ---- Type-Level History Functions ------------------------------------------
 
 (defn type-history

modules/db/src/blaze/db/api_spec.clj

@@ -174,13 +174,6 @@
                :start-t (s/? (s/nilable :blaze.db/t)))
   :ret (cs/coll-of :blaze.db/resource-handle))
 
-(s/fdef d/total-num-of-instance-changes
-  :args (s/cat :db :blaze.db/db
-               :type :fhir.resource/type
-               :id :blaze.resource/id
-               :since (s/? (s/nilable inst?)))
-  :ret nat-int?)
-
 ;; ---- Type-Level History Functions ------------------------------------------
 
 (s/fdef d/type-history

modules/db/src/blaze/db/impl/batch_db.clj

@@ -146,17 +146,13 @@
 
   (-instance-history [_ tid id start-t]
     (let [start-t (if (some-> start-t (<= t)) start-t t)]
-      (rao/instance-history snapshot tid id start-t)))
-
-  (-total-num-of-instance-changes [_ tid id since]
-    (let [end-t (or (some->> since (t-by-instant/t-by-instant snapshot)) 0)]
-      (rao/num-of-instance-changes snapshot tid id t end-t)))
+      (rao/instance-history snapshot tid id t start-t)))
 
   ;; ---- Type-Level History Functions ----------------------------------------
 
   (-type-history [_ tid start-t start-id]
     (let [start-t (if (some-> start-t (<= t)) start-t t)]
-      (tao/type-history snapshot tid start-t start-id)))
+      (tao/type-history snapshot tid t start-t start-id)))
 
   (-total-num-of-type-changes [_ type since]
     (let [tid (codec/tid type)
@@ -168,7 +164,7 @@
 
   (-system-history [_ start-t start-tid start-id]
     (let [start-t (if (some-> start-t (<= t)) start-t t)]
-      (sao/system-history snapshot start-t start-tid start-id)))
+      (sao/system-history snapshot t start-t start-tid start-id)))
 
   (-total-num-of-system-changes [_ since]
     (let [end-t (some->> since (t-by-instant/t-by-instant snapshot))]

modules/db/src/blaze/db/impl/db.clj

@@ -113,10 +113,6 @@
     (with-open-coll [batch-db (batch-db/new-batch-db node basis-t t)]
       (p/-instance-history batch-db tid id start-t)))
 
-  (-total-num-of-instance-changes [_ tid id since]
-    (with-open [batch-db (batch-db/new-batch-db node basis-t t)]
-      (p/-total-num-of-instance-changes batch-db tid id since)))
-
   ;; ---- Type-Level History Functions ----------------------------------------
 
   (-type-history [_ tid start-t start-id]

modules/db/src/blaze/db/impl/index/resource_as_of.clj

@@ -21,9 +21,12 @@
 (def ^:private ^:const ^long max-key-size
   (+ except-id-key-size codec/max-id-size))
 
-(def ^:const ^long value-size
+(def ^:const ^long min-value-size
   (+ hash/size codec/state-size))
 
+(def ^:const ^long max-value-size
+  (+ hash/size codec/state-size codec/t-size))
+
 (defn- focus-id!
   "Reduces the limit of `kb` in order to hide the t and focus on id solely."
   [kb]
@@ -295,15 +298,16 @@
     (rh/resource-handle! tid id (codec/descending-long (bb/get-long! kb tid-id-size)) vb)))
 
 (defn instance-history
-  "Returns a reducible collection of all versions of the resource with `tid` and
-  `id` starting at `start-t`.
-
-  Versions are resource handles."
-  [snapshot tid id start-t]
+  "Returns a reducible collection of all historic resource handles of the
+  resource with `tid` and `id` of the database with the point in time `t`
+  starting at `start-t`."
+  [snapshot tid id t start-t]
   (let [tid-id-size (+ codec/tid-size (bs/size id))]
-    (i/prefix-entries snapshot :resource-as-of-index
-                      (map (decoder tid (codec/id-string id) tid-id-size))
-                      tid-id-size (start-key tid id start-t))))
+    (i/prefix-entries
+     snapshot :resource-as-of-index
+     (comp (map (decoder tid (codec/id-string id) tid-id-size))
+           (take-while #(< (long t) (rh/purged-at %))))
+     tid-id-size (start-key tid id start-t))))
 
 (defn- resource-handle* [iter target-buf key-buf value-buf tid id t]
   (let [tid-id-size (+ codec/tid-size (bs/size id))
@@ -347,7 +351,7 @@
   [snapshot tid id t]
   (let [target-buf (bb/allocate max-key-size)
         key-buf (bb/allocate max-key-size)
-        value-buf (bb/allocate value-size)]
+        value-buf (bb/allocate max-value-size)]
     (with-open [iter (kv/new-iterator snapshot :resource-as-of-index)]
       (resource-handle* iter target-buf key-buf value-buf tid id t))))
 
@@ -369,7 +373,7 @@
   [snapshot t]
   (let [target-buf (bb/allocate max-key-size)
         key-buf (bb/allocate max-key-size)
-        value-buf (bb/allocate value-size)
+        value-buf (bb/allocate max-value-size)
         iter (kv/new-iterator snapshot :resource-as-of-index)]
     (comp
      (keep
@@ -389,7 +393,7 @@
   ([snapshot t tid id-extractor matcher]
    (let [target-buf (bb/allocate max-key-size)
          key-buf (bb/allocate max-key-size)
-         value-buf (bb/allocate value-size)
+         value-buf (bb/allocate max-value-size)
          iter (kv/new-iterator snapshot :resource-as-of-index)]
      (comp
       (keep
@@ -399,10 +403,3 @@
            (when (matcher input handle)
              handle))))
       (closer iter)))))
-
-(defn num-of-instance-changes
-  "Returns the number of changes between `start-t` (inclusive) and `end-t`
-  (inclusive) of the resource with `tid` and `id` in `snapshot`."
-  [snapshot tid id start-t end-t]
-  (- (long (:num-changes (resource-handle snapshot tid id start-t) 0))
-     (long (:num-changes (resource-handle snapshot tid id end-t) 0))))

modules/db/src/blaze/db/impl/index/resource_handle.clj

@@ -11,7 +11,7 @@
 (set! *warn-on-reflection* true)
 (set! *unchecked-math* :warn-on-boxed)
 
-(deftype ResourceHandle [^int tid id ^long t hash ^long num-changes op]
+(deftype ResourceHandle [^int tid id ^long t hash ^long num-changes op ^long purged-at]
   p/FhirType
   (-type [_]
    ;; TODO: maybe cache this
@@ -29,6 +29,7 @@
       :hash hash
       :num-changes num-changes
       :op op
+      :purged-at purged-at
       not-found))
 
   Object
@@ -61,6 +62,11 @@
       :delete
       :put)))
 
+(defn- get-purged-at! [vb]
+  (if (<= 8 (bb/remaining vb))
+    (bb/get-long! vb)
+    Long/MAX_VALUE))
+
 (defn resource-handle!
   "Creates a new resource handle.
 
@@ -74,7 +80,8 @@
      t
      hash
      (state->num-changes state)
-     (state->op state))))
+     (state->op state)
+     (get-purged-at! vb))))
 
 (defn resource-handle? [x]
   (instance? ResourceHandle x))
@@ -116,6 +123,11 @@
   [rh]
   (.-op ^ResourceHandle rh))
 
+(defn purged-at
+  {:inline (fn [rh] `(.-purged-at ~(with-meta rh {:tag `ResourceHandle})))}
+  [rh]
+  (.-purged-at ^ResourceHandle rh))
+
 (defn reference [rh]
   (str (codec/tid->type (tid rh)) "/" (id rh)))
 

modules/db/src/blaze/db/impl/index/rts_as_of.clj

@@ -21,14 +21,27 @@
     (identical? :create op) (Numbers/setBit 1)
     (identical? :delete op) (Numbers/setBit 0)))
 
-(defn encode-value [hash num-changes op]
-  (-> (bb/allocate rao/value-size)
-      (hash/into-byte-buffer! hash)
-      (bb/put-long! (state num-changes op))
-      bb/array))
+(defn- encode-value
+  ([hash num-changes op]
+   (-> (bb/allocate rao/min-value-size)
+       (hash/into-byte-buffer! hash)
+       (bb/put-long! (state num-changes op))
+       bb/array))
+  ([hash num-changes op purged-at]
+   (-> (bb/allocate rao/max-value-size)
+       (hash/into-byte-buffer! hash)
+       (bb/put-long! (state num-changes op))
+       (bb/put-long! purged-at)
+       bb/array)))
 
-(defn index-entries [tid id t hash num-changes op]
-  (let [value (encode-value hash num-changes op)]
-    [[:resource-as-of-index (rao/encode-key tid id t) value]
-     [:type-as-of-index (tao/encode-key tid t id) value]
-     [:system-as-of-index (sao/encode-key t tid id) value]]))
+(defn index-entries
+  ([tid id t hash num-changes op]
+   (let [value (encode-value hash num-changes op)]
+     [[:resource-as-of-index (rao/encode-key tid id t) value]
+      [:type-as-of-index (tao/encode-key tid t id) value]
+      [:system-as-of-index (sao/encode-key t tid id) value]]))
+  ([tid id t hash num-changes op purged-at]
+   (let [value (encode-value hash num-changes op purged-at)]
+     [[:resource-as-of-index (rao/encode-key tid id t) value]
+      [:type-as-of-index (tao/encode-key tid t id) value]
+      [:system-as-of-index (sao/encode-key t tid id) value]])))

modules/db/src/blaze/db/impl/index/system_as_of.clj

@@ -63,14 +63,15 @@
     (Longs/toByteArray (codec/descending-long ^long start-t))))
 
 (defn system-history
-  "Returns a reducible collection of all versions between `start-t` (inclusive),
-  `start-tid` (optional, inclusive) and `start-id` (optional, inclusive) of all
-  resources.
-
-  Versions are resource handles."
-  [snapshot start-t start-tid start-id]
-  (i/entries snapshot :system-as-of-index (map (decoder))
-             (bs/from-byte-array (start-key start-t start-tid start-id))))
+  "Returns a reducible collection of all historic resource handles of the
+  database with the point in time `t` between `start-t` (inclusive), `start-tid`
+  (optional, inclusive) and `start-id` (optional, inclusive)."
+  [snapshot t start-t start-tid start-id]
+  (i/entries
+   snapshot :system-as-of-index
+   (comp (map (decoder))
+         (filter #(< (long t) (rh/purged-at %))))
+   (bs/from-byte-array (start-key start-t start-tid start-id))))
 
 (defn changes
   "Returns a reducible collection of all resource handles changed at `t`."