From 4f78f45c2cd58e580396b3a6401505d507f2ef31 Mon Sep 17 00:00:00 2001 From: Alexander Kiel Date: Sun, 12 Jun 2022 22:30:53 +0200 Subject: [PATCH] [WIP] Switch to Internal Resource Identifier in the Index --- docs/implementation/database.md | 112 +++-- docs/performance/fhir-search.md | 2 +- modules/byte-buffer/src/blaze/byte_buffer.clj | 29 +- .../test/blaze/byte_buffer_test.clj | 50 +- .../src/blaze/db/impl/protocols.clj | 4 +- modules/db-stub/src/blaze/db/api_stub.clj | 1 + modules/db-tx-log/deps.edn | 3 +- .../db-tx-log/src/blaze/db/tx_log/spec.clj | 17 +- .../test/blaze/db/tx_log/spec_test.clj | 2 +- modules/db/NOTES.md | 125 ----- modules/db/src/blaze/db/api.clj | 41 +- modules/db/src/blaze/db/impl/batch_db.clj | 72 +-- modules/db/src/blaze/db/impl/codec.clj | 54 +-- modules/db/src/blaze/db/impl/db.clj | 8 +- modules/db/src/blaze/db/impl/index.clj | 14 +- .../db/impl/index/compartment/resource.clj | 70 ++- .../search_param_value_resource.clj | 65 ++- .../blaze/db/impl/index/resource_as_of.clj | 315 ++++++------- .../blaze/db/impl/index/resource_handle.clj | 38 +- .../src/blaze/db/impl/index/resource_id.clj | 34 ++ .../index/resource_search_param_value.clj | 79 ++-- .../db/src/blaze/db/impl/index/rts_as_of.clj | 22 +- .../index/search_param_value_resource.clj | 128 +++-- .../search_param_value_resource/impl.clj | 15 - .../src/blaze/db/impl/index/system_as_of.clj | 60 ++- .../src/blaze/db/impl/index/system_stats.clj | 8 +- .../db/src/blaze/db/impl/index/type_as_of.clj | 58 ++- .../db/src/blaze/db/impl/index/type_stats.clj | 10 +- modules/db/src/blaze/db/impl/search_param.clj | 37 +- .../blaze/db/impl/search_param/chained.clj | 9 +- .../db/impl/search_param/composite/common.clj | 10 +- .../search_param/composite/token_quantity.clj | 10 +- .../search_param/composite/token_token.clj | 10 +- .../src/blaze/db/impl/search_param/date.clj | 54 +-- .../db/src/blaze/db/impl/search_param/has.clj | 24 +- .../src/blaze/db/impl/search_param/list.clj | 51 +- .../src/blaze/db/impl/search_param/number.clj | 12 +- .../blaze/db/impl/search_param/quantity.clj | 100 ++-- .../src/blaze/db/impl/search_param/string.clj | 25 +- .../src/blaze/db/impl/search_param/token.clj | 63 +-- .../src/blaze/db/impl/search_param/util.clj | 20 +- modules/db/src/blaze/db/node.clj | 18 +- .../db/src/blaze/db/node/resource_indexer.clj | 74 +-- .../src/blaze/db/node/tx_indexer/verify.clj | 70 +-- .../db/test-perf/blaze/db/api_test_perf.clj | 3 +- .../impl/index/resource_handle_test_perf.clj | 51 -- modules/db/test/blaze/db/api_test.clj | 41 +- modules/db/test/blaze/db/impl/codec/spec.clj | 18 +- modules/db/test/blaze/db/impl/codec_spec.clj | 18 +- modules/db/test/blaze/db/impl/codec_test.clj | 15 +- modules/db/test/blaze/db/impl/db_spec.clj | 1 + .../impl/index/compartment/resource_spec.clj | 4 +- .../index/compartment/resource_test_util.clj | 7 +- .../search_param_value_resource_spec.clj | 2 +- .../search_param_value_resource_test_util.clj | 29 +- .../db/impl/index/resource_as_of_spec.clj | 12 +- .../db/impl/index/resource_as_of_test.clj | 25 + .../impl/index/resource_as_of_test_util.clj | 35 +- .../db/impl/index/resource_handle_spec.clj | 11 +- .../db/impl/index/resource_handle_test.clj | 58 ++- .../blaze/db/impl/index/resource_id_spec.clj | 17 + .../db/impl/index/resource_id_test_util.clj | 20 + .../resource_search_param_value_spec.clj | 20 +- .../resource_search_param_value_test_util.clj | 9 +- .../blaze/db/impl/index/rts_as_of_spec.clj | 14 +- .../search_param_value_resource/impl_test.clj | 35 -- .../search_param_value_resource_spec.clj | 19 +- .../search_param_value_resource_test.clj | 32 +- .../search_param_value_resource_test_util.clj | 13 +- .../blaze/db/impl/index/system_as_of_spec.clj | 2 +- .../db/impl/index/system_as_of_test_util.clj | 33 +- .../db/impl/index/system_stats_test_util.clj | 21 +- .../blaze/db/impl/index/type_as_of_spec.clj | 2 +- .../db/impl/index/type_as_of_test_util.clj | 33 +- .../db/impl/index/type_stats_test_util.clj | 23 +- modules/db/test/blaze/db/impl/index_spec.clj | 2 +- .../db/impl/search_param/composite_test.clj | 19 +- .../blaze/db/impl/search_param/date_test.clj | 34 +- .../db/impl/search_param/number_test.clj | 17 +- .../db/impl/search_param/quantity_spec.clj | 2 +- .../db/impl/search_param/quantity_test.clj | 57 +-- .../test/blaze/db/impl/search_param/spec.clj | 2 +- .../db/impl/search_param/string_test.clj | 33 +- .../blaze/db/impl/search_param/token_spec.clj | 2 +- .../blaze/db/impl/search_param/token_test.clj | 146 +++--- .../blaze/db/impl/search_param/util_spec.clj | 6 + .../test/blaze/db/impl/search_param_spec.clj | 6 +- .../test/blaze/db/impl/search_param_test.clj | 66 +-- .../blaze/db/node/resource_indexer_test.clj | 64 ++- .../blaze/db/node/tx_indexer/verify_spec.clj | 3 +- .../blaze/db/node/tx_indexer/verify_test.clj | 441 ++++++++++-------- modules/db/test/blaze/db/node_test.clj | 6 +- modules/db/test/blaze/db/test_util.clj | 4 +- .../blaze/interaction/search/include_test.clj | 8 +- .../blaze/interaction/search_type_test.clj | 4 +- profiling/blaze/profiling.clj | 1 + resources/blaze.edn | 13 + 97 files changed, 1742 insertions(+), 1840 deletions(-) delete mode 100644 modules/db/NOTES.md create mode 100644 modules/db/src/blaze/db/impl/index/resource_id.clj delete mode 100644 modules/db/src/blaze/db/impl/index/search_param_value_resource/impl.clj delete mode 100644 modules/db/test-perf/blaze/db/impl/index/resource_handle_test_perf.clj create mode 100644 modules/db/test/blaze/db/impl/index/resource_as_of_test.clj create mode 100644 modules/db/test/blaze/db/impl/index/resource_id_spec.clj create mode 100644 modules/db/test/blaze/db/impl/index/resource_id_test_util.clj delete mode 100644 modules/db/test/blaze/db/impl/index/search_param_value_resource/impl_test.clj diff --git a/docs/implementation/database.md b/docs/implementation/database.md index dc5425598..73161c913 100644 --- a/docs/implementation/database.md +++ b/docs/implementation/database.md @@ -38,20 +38,25 @@ There are two different sets of indices, ones which depend on the database value ### Indices depending on t -| Name | Key Parts | Value | -|--------------|-----------|-------------------------------| -| ResourceAsOf | type id t | content-hash, num-changes, op | -| TypeAsOf | type t id | content-hash, num-changes, op | -| SystemAsOf | t type id | content-hash, num-changes, op | -| TxSuccess | t | instant | -| TxError | t | anomaly | -| TByInstant | instant | t | -| TypeStats | type t | total, num-changes | -| SystemStats | t | total, num-changes | +| Name | Key Parts | Value | +|--------------|-----------|-----------------------------------| +| ResourceId | tid id | did | +| ResourceAsOf | tid did t | content-hash, num-changes, op, id | +| TypeAsOf | tid t did | content-hash, num-changes, op, id | +| SystemAsOf | t tid did | content-hash, num-changes, op, id | +| TxSuccess | t | instant | +| TxError | t | anomaly | +| TByInstant | instant | t | +| TypeStats | tid t | total, num-changes | +| SystemStats | t | total, num-changes | + +#### ResourceId + +The `ResourceId` index maps the external resource identifier represented by the tuple `(tid, id)`, where `tid` is a 4-byte hash of the resource type and `id` is the [logical id][8] of the resource, to the id part of the internal resource identifier `did`. The term `did` stands for database identifier. #### ResourceAsOf -The `ResourceAsOf` index is the primary index which maps the resource identifier `(type, id)` together with the `t` to the `content-hash` of the resource version. In addition to that, the index contains the number of changes `num-changes` to the resource and the operator `op` of the change leading to the index entry. +The `ResourceAsOf` index is the primary index which maps the internal resource identifier `(tid, did)` together with the `t` to the `content-hash` of the resource version. In addition to that, the index contains the number of changes `num-changes` to the resource and the operator `op` of the change leading to the index entry. The `ResourceAsOf` index is used to access the version of a resource at a particular point in time `t`. In other words, given a point in time `t`, the database value with that `t`, allows to access the resource version at that point in time by its identifier. Because the index only contains entries with `t` values of changes to each resource, the most current resource version is determined by querying the index for the greatest `t` less or equal to the `t` of the database value. @@ -59,16 +64,16 @@ The `ResourceAsOf` index is used to access the version of a resource at a partic The following `ResourceAsOf` index: -| Key (type, id, t) | Value (content-hash, num-changes, op) | -|-------------------|---------------------------------------| -| Patient, 0, 4 | -, 3, delete | -| Patient, 0, 3 | b7e3e5f8, 2, update | -| Patient, 0, 1 | ba9c9b24, 1, create | -| Patient, 1, 2 | 6744ed32, 1, create | +| Key (tid, did, t) | Value (content-hash, num-changes, op, id) | +|-------------------|-------------------------------------------| +| Patient, 0, 4 | --------, 3, delete, 0 | +| Patient, 0, 3 | b7e3e5f8, 2, update, 0 | +| Patient, 0, 1 | ba9c9b24, 1, create, 0 | +| Patient, 1, 2 | 6744ed32, 1, create, 1 | provides the basis for the following database values: -| t | type | id | content-hash | +| t | type | did | content-hash | |-----|---------|-----|--------------| | 1 | Patient | 0 | ba9c9b24 | | 2 | Patient | 0 | ba9c9b24 | @@ -77,17 +82,17 @@ provides the basis for the following database values: | 3 | Patient | 1 | 6744ed32 | | 4 | Patient | 1 | 6744ed32 | -The database value with `t=1` contains one patient with `id=0` and content hash `ba9c9b24`, because the second patient was created later at `t=2`. The index access algorithm will not find an entry for the patient with `id=1` on a database value with `t=1` because there is no index key with `type=Patient`, `id=1` and `t<=1`. However, the database value with `t=2` will contain the patient with `id=1` and additionally contains the patient with `id=0` because there is a key with `type=Patient`, `id=0` and `t<=2`. Next, the database value with `t=3` still contains the same content hash for the patient with `id=1` and reflects the update on patient with `id=0` because the key `(Patient, 0, 3)` is now the one with the greatest `t<=3`, resulting in the content hash `b7e3e5f8`. Finally, the database value with `t=4` doesn't contain the patient with `id=0` anymore, because it was deleted. As can be seen in the index, deleting a resource is done by adding the information that it was deleted at some point in time. +The database value with `t=1` contains one patient with `did=0` and content hash `ba9c9b24`, because the second patient was created later at `t=2`. The index access algorithm will not find an entry for the patient with `did=1` on a database value with `t=1` because there is no index key with `type=Patient`, `did=1` and `t<=1`. However, the database value with `t=2` will contain the patient with `did=1` and additionally contains the patient with `did=0` because there is a key with `type=Patient`, `did=0` and `t<=2`. Next, the database value with `t=3` still contains the same content hash for the patient with `did=1` and reflects the update on patient with `did=0` because the key `(Patient, 0, 3)` is now the one with the greatest `t<=3`, resulting in the content hash `b7e3e5f8`. Finally, the database value with `t=4` doesn't contain the patient with `did=0` anymore, because it was deleted. As can be seen in the index, deleting a resource is done by adding the information that it was deleted at some point in time. In addition to direct resource lookup, the `ResourceAsOf` index is used for listing all versions of a particular resource, listing all resources of a particular type and listing all resources at all. Listings are done by scanning through the index and for the non-history case, skipping versions not appropriate for the `t` of the database value. #### TypeAsOf -The `TypeAsOf` index contains the same information as the `ResourceAsOf` index with the difference that the components of the key are ordered `type`, `t` and `id` instead of `type`, `id` and `t`. The index is used for listing all versions of all resources of a particular type. Such history listings start with the `t` of the database value going into the past. This is done by not only choosing the resource version with the latest `t` less or equal the database values `t` but instead using all older versions. Such versions even include deleted versions because in FHIR it is allowed to bring back a resource to a new life after it was already deleted. The listing is done by simply scanning through the index in reverse. Because the key is ordered by `type`, `t` and `id`, the entries will be first ordered by time, newest first, and second by resource identifier. +The `TypeAsOf` index contains the same information as the `ResourceAsOf` index with the difference that the components of the key are ordered `type`, `t` and `did` instead of `type`, `did` and `t`. The index is used for listing all versions of all resources of a particular type. Such history listings start with the `t` of the database value going into the past. This is done by not only choosing the resource version with the latest `t` less or equal the database values `t` but instead using all older versions. Such versions even include deleted versions because in FHIR it is allowed to bring back a resource to a new life after it was already deleted. The listing is done by simply scanning through the index in reverse. Because the key is ordered by `type`, `t` and `did`, the entries will be first ordered by time, newest first, and second by resource identifier. #### SystemAsOf -In the same way the `TypeAsOf` index uses a different key ordering in comparison to the `ResourceAsOf` index, the `SystemAsOf` index will use the key order `t`, `type` and `id` in order to provide a global time axis order by resource type and by identifier secondarily. +In the same way the `TypeAsOf` index uses a different key ordering in comparison to the `ResourceAsOf` index, the `SystemAsOf` index will use the key order `t`, `type` and `did` in order to provide a global time axis order by resource type and by identifier secondarily. #### TxSuccess @@ -115,23 +120,26 @@ The `SystemStats` index keeps track of the total number of resources, and the nu The indices not depending on `t` directly point to the resource versions by their content hash. -| Name | Key Parts | Value | -|-------------------------------------|--------------------------------------------------------------|-------| -| SearchParamValueResource | search-param, type, value, id, content-hash | - | -| ResourceSearchParamValue | type, id, content-hash, search-param, value | - | -| CompartmentSearchParamValueResource | co-c-hash, co-res-id, sp-c-hash, tid, value, id, hash-prefix | - | -| CompartmentResource | co-c-hash, co-res-id, tid, id | - | -| SearchParam | code, tid | id | -| ActiveSearchParams | id | - | +| Name | Key Parts | Value | +|-------------------------------------|----------------------------------------------------------------|-------| +| SearchParamValueResource | sp-c-hash, tid, value, did, hash-prefix | - | +| ResourceSearchParamValue | tid, did, hash-prefix, sp-c-hash, value | - | +| CompartmentSearchParamValueResource | co-c-hash, co-res-did, sp-c-hash, tid, value, did, hash-prefix | - | +| CompartmentResource | co-c-hash, co-res-did, tid, did | - | +| SearchParam | code, tid | id | +| ActiveSearchParams | id | - | #### SearchParamValueResource -The `SearchParamValueResource` index contains all values from resources that are reachable from search parameters. The components of its key are: -* `search-param` - a 4-byte hash of the search parameters code used to identify the search parameter -* `type` - a 4-byte hash of the resource type -* `value` - the encoded value of the resource reachable by the search parameters FHIRPath expression. The encoding depends on the search parameters type. -* `id` - the logical id of the resource -* `content-hash` - a 4-byte prefix of the content-hash of the resource version +The `SearchParamValueResource` index contains all values from resources that are reachable from search parameters. + +The components of its key are: + +* `sp-c-hash` - a 4-byte hash of the search parameters code used to identify the search parameter +* `tid` - a 4-byte hash of the resource type +* `value` - the encoded value of the resource reachable by the search parameters FHIRPath expression. The encoding depends on the search parameters type +* `did` - the internal id (database id) of the resource +* `hash-prefix` - a 4-byte prefix of the content-hash of the resource version The way the `SearchParamValueResource` index is used, depends on the type of the search parameter. The following sections will explain this in detail for each type: @@ -169,19 +177,19 @@ In order to facilitate different forms of searches specified in the [FHIR Spec][ * `|code` - the code if the resource doesn't specify a system * `system|` - the system independent of the code, used to find all resources with any code in that system -After concatenation, the strings are hashed with the [Murmur3][7] algorithm in its 32-bit variant, yielding a 4-byte wide value. The hashing is done to save space and ensure that all values are of the same length. +After concatenation, the strings are hashed with [FarmHash's Fingerprint64][7] algorithm, yielding an 8-byte wide value. The hashing is done to save space and ensure that all values are of the same length. ###### Example For this example, we don't use the hashed versions of the key parts except for the content-hash. -| Key (search-param, type, value, id, content-hash) | -|---| -| gender, Patient, female, 1, 6744ed32 | -| gender, Patient, female, 2, b7e3e5f8 | -| gender, Patient, male, 0, ba9c9b24 | +| Key (sp-c-hash, tid, value, did, hash-prefix) | +|-----------------------------------------------| +| gender, Patient, female, 1, 6744ed32 | +| gender, Patient, female, 2, b7e3e5f8 | +| gender, Patient, male, 0, ba9c9b24 | -In case one searches for female patients, Blaze will seek into the index with the key prefix (gender, Patient, female) and scan over it while the prefix stays the same. The result will be the `[id, hash]` tuples: +In case one searches for female patients, Blaze will seek into the index with the key prefix (gender, Patient, female) and scan over it while the prefix stays the same. The result will be the `[did, hash]` tuples: * `[1, 6744ed32]` and * `[2, b7e3e5f8]`. @@ -207,11 +215,25 @@ That tuples are further processed against the `ResourceAsOf` index in order to c **TODO: continue...** +#### CompartmentSearchParamValueResource + +Same as the `SearchParamValueResource` index but prefixed with a compartment the resource belongs to. This index is used in [variant searches][9] and in CQL evaluation within the Patient context. In the CQL Patient context all retrieves are relative to one patient. Using that patient as compartment in the `CompartmentSearchParamValueResource` index allows for efficient implementation of that retrieves. + +The components of its key are: + +* `co-c-hash` - a 4-byte hash of the code of the compartment +* `co-res-did` - the internal id (database id) of the resource of the compartment +* `sp-c-hash` - a 4-byte hash of the search parameters code used to identify the search parameter +* `tid` - a 4-byte hash of the resource type +* `value` - the encoded value of the resource reachable by the search parameters FHIRPath expression. The encoding depends on the search parameters type +* `did` - the internal id (database id) of the resource +* `hash-prefix` - a 4-byte prefix of the content-hash of the resource version + ## Transaction Handling * a transaction bundle is POST'ed to one arbitrary node * this node submits the transaction commands to the central transaction log -* all nodes (inkl. the transaction submitter) receive the transaction commands from the central transaction log +* all nodes (incl. the transaction submitter) receive the transaction commands from the central transaction log **TODO: continue...** @@ -221,4 +243,6 @@ That tuples are further processed against the `ResourceAsOf` index in order to c [4]: [5]: [6]: -[7]: +[7]: +[8]: +[9]: diff --git a/docs/performance/fhir-search.md b/docs/performance/fhir-search.md index 46b1693b9..7e4d751af 100644 --- a/docs/performance/fhir-search.md +++ b/docs/performance/fhir-search.md @@ -97,7 +97,7 @@ The result is a dataset which consists only of the resource types Patient, Obser ## Controlling and Monitoring the Caches -The size of the resource cache and the resource handle cache can be set by their respective environment variables `DB_RESOURCE_CACHE_SIZE` and `DB_RESOURCE_HANDLE_CACHE_SIZE`. The size denotes the number of resources / resource handles. Because one has to specify a number of resources / resource handles, it's important to know how many bytes a resource / resource handle allocates on the heap. For resource handles, it can be said that they allocate between 272 and 328 bytes depending on the size of the resource id. For resources, the size varies widely. Monitoring of the heap usage is critical. +The size of the resource cache and the resource handle cache can be set by their respective environment variables `DB_RESOURCE_CACHE_SIZE` and `DB_RESOURCE_HANDLE_CACHE_SIZE`. The size denotes the number of resources / resource handles. Because one has to specify a number of resources / resource handles, it's important to know how many bytes a resource / resource handle allocates on the heap. For resource handles, it can be said that they allocate between 152 and 208 bytes depending on the size of the resource id. For resources, the size varies widely. Monitoring of the heap usage is critical. ### Monitoring diff --git a/modules/byte-buffer/src/blaze/byte_buffer.clj b/modules/byte-buffer/src/blaze/byte_buffer.clj index d4d618ff8..7e81210a7 100644 --- a/modules/byte-buffer/src/blaze/byte_buffer.clj +++ b/modules/byte-buffer/src/blaze/byte_buffer.clj @@ -78,6 +78,12 @@ (.putInt ^ByteBuffer byte-buffer x)) +(defn put-5-byte-long! + [byte-buffer ^long x] + (put-byte! byte-buffer (bit-shift-right (unchecked-long x) 32)) + (put-int! byte-buffer x)) + + (defn put-long! {:inline (fn [byte-buffer x] @@ -220,6 +226,11 @@ (.getInt ^ByteBuffer byte-buffer)) +(defn get-5-byte-long! [byte-buffer] + (+ (bit-shift-left (bit-and (get-byte! byte-buffer) 0xFF) 32) + (bit-and (get-int! byte-buffer) 0xFFFFFFFF))) + + (defn get-long! {:inline (fn [byte-buffer] @@ -243,24 +254,6 @@ (.get ^ByteBuffer byte-buffer ^bytes byte-array offset length))) -(defn size-up-to-null [byte-buffer] - (when (pos? (remaining byte-buffer)) - (mark! byte-buffer) - (loop [byte (bit-and (long (get-byte! byte-buffer)) 0xFF) - size 0] - (cond - (zero? byte) - (do (reset! byte-buffer) - size) - - (pos? (remaining byte-buffer)) - (recur (bit-and (long (get-byte! byte-buffer)) 0xFF) (inc size)) - - :else - (do (reset! byte-buffer) - nil))))) - - (defn mismatch "Finds and returns the relative index of the first mismatch between `a` and `b`. diff --git a/modules/byte-buffer/test/blaze/byte_buffer_test.clj b/modules/byte-buffer/test/blaze/byte_buffer_test.clj index 77caa7a64..2b4c350cc 100644 --- a/modules/byte-buffer/test/blaze/byte_buffer_test.clj +++ b/modules/byte-buffer/test/blaze/byte_buffer_test.clj @@ -3,7 +3,7 @@ [blaze.byte-buffer :as bb] [blaze.test-util :refer [satisfies-prop]] [clojure.spec.test.alpha :as st] - [clojure.test :as test :refer [deftest is testing]] + [clojure.test :as test :refer [are deftest]] [clojure.test.check.generators :as gen] [clojure.test.check.properties :as prop])) @@ -26,41 +26,19 @@ (= capacity (bb/limit (bb/allocate capacity)))))) -(deftest size-up-to-null-test - (testing "empty buffer" - (let [buf (bb/allocate 0)] - (is (nil? (bb/size-up-to-null buf))))) +(defn- transcode-5-byte-long [x] + (let [buf (bb/allocate 5)] + (bb/put-5-byte-long! buf x) + (bb/flip! buf) + (= x (bb/get-5-byte-long! buf)))) - (testing "buffer with only one null byte" - (let [buf (bb/allocate 1)] - (bb/put-byte! buf 0) - (bb/flip! buf) - (is (zero? (bb/size-up-to-null buf))))) - (testing "buffer with only one non-null byte" - (let [buf (bb/allocate 1)] - (bb/put-byte! buf 1) - (bb/flip! buf) - (is (nil? (bb/size-up-to-null buf))))) +(deftest transcode-5-byte-long-test + (are [x] (transcode-5-byte-long x) + 0 + 0xFFFFFFFF + 0xFFFFFFFFFF) - (testing "buffer with one non-null and one null byte" - (let [buf (bb/allocate 2)] - (bb/put-byte! buf 1) - (bb/put-byte! buf 0) - (bb/flip! buf) - (is (= 1 (bb/size-up-to-null buf))))) - - (testing "buffer with two null bytes" - (let [buf (bb/allocate 2)] - (bb/put-byte! buf 0) - (bb/put-byte! buf 0) - (bb/flip! buf) - (is (zero? (bb/size-up-to-null buf))))) - - (testing "buffer with two non-null and one null byte" - (let [buf (bb/allocate 3)] - (bb/put-byte! buf 1) - (bb/put-byte! buf 2) - (bb/put-byte! buf 0) - (bb/flip! buf) - (is (= 2 (bb/size-up-to-null buf)))))) + (satisfies-prop 100000 + (prop/for-all [x (gen/choose 0 0xFFFFFFFFFF)] + (transcode-5-byte-long x)))) diff --git a/modules/db-protocols/src/blaze/db/impl/protocols.clj b/modules/db-protocols/src/blaze/db/impl/protocols.clj index 61ddc38c5..37bb56f8c 100644 --- a/modules/db-protocols/src/blaze/db/impl/protocols.clj +++ b/modules/db-protocols/src/blaze/db/impl/protocols.clj @@ -76,8 +76,8 @@ (-compartment-keys [search-param context compartment tid compiled-value]) (-matches? [search-param context resource-handle modifier compiled-values]) (-compartment-ids [_ resolver resource]) - (-index-values [_ resolver resource]) - (-index-value-compiler [_])) + (-index-values [_ resource-id resolver resource]) + (-index-value-compiler [_ resource-id])) (defprotocol Pull diff --git a/modules/db-stub/src/blaze/db/api_stub.clj b/modules/db-stub/src/blaze/db/api_stub.clj index 33cc925ca..29a8fe809 100644 --- a/modules/db-stub/src/blaze/db/api_stub.clj +++ b/modules/db-stub/src/blaze/db/api_stub.clj @@ -58,6 +58,7 @@ :tx-success-index {:reverse-comparator? true} :tx-error-index nil :t-by-instant-index {:reverse-comparator? true} + :resource-id-index nil :resource-as-of-index nil :type-as-of-index nil :system-as-of-index nil diff --git a/modules/db-tx-log/deps.edn b/modules/db-tx-log/deps.edn index b89260e43..48d415bdc 100644 --- a/modules/db-tx-log/deps.edn +++ b/modules/db-tx-log/deps.edn @@ -31,4 +31,5 @@ {cloverage/cloverage {:mvn/version "1.2.4"}} - :main-opts ["-m" "cloverage.coverage" "--codecov" "-p" "src" "-s" "test"]}}} + :main-opts ["-m" "cloverage.coverage" "--codecov" "-p" "src" "-s" "test" + "-e" ".+spec"]}}} diff --git a/modules/db-tx-log/src/blaze/db/tx_log/spec.clj b/modules/db-tx-log/src/blaze/db/tx_log/spec.clj index a28c434e4..2e4b6c5a1 100644 --- a/modules/db-tx-log/src/blaze/db/tx_log/spec.clj +++ b/modules/db-tx-log/src/blaze/db/tx_log/spec.clj @@ -4,7 +4,8 @@ [blaze.db.tx-log :as tx-log] [blaze.fhir.spec] [blaze.spec] - [clojure.spec.alpha :as s])) + [clojure.spec.alpha :as s] + [clojure.spec.gen.alpha :as gen])) (s/def :blaze.db/tx-log @@ -27,8 +28,20 @@ (s/coll-of :blaze.fhir/local-ref-tuple)) +(def ^:private ^:const ^long max-t 0xFFFFFFFFFF) + + +;; With a 5 byte long `t`, we can create one transaction each millisecond +;; for 34 years. Alone the t-values would need a storage of 5 TB. +(comment + (let [millis-per-year (* 1000 3600 24 365) + five-bytes (long (Math/pow 2 40))] + (double (/ five-bytes millis-per-year)))) + + +;; The point in time `t` of a database value. (s/def :blaze.db/t - (s/and int? #(<= 0 % 0xFFFFFFFFFFFFFF))) + (s/with-gen (s/and int? #(<= 0 % max-t)) #(gen/choose 0 max-t))) (s/def :blaze.db.tx-cmd/if-none-exist diff --git a/modules/db-tx-log/test/blaze/db/tx_log/spec_test.clj b/modules/db-tx-log/test/blaze/db/tx_log/spec_test.clj index fb7ab9186..0c0da19e2 100644 --- a/modules/db-tx-log/test/blaze/db/tx_log/spec_test.clj +++ b/modules/db-tx-log/test/blaze/db/tx_log/spec_test.clj @@ -27,7 +27,7 @@ (are [x] (s/valid? :blaze.db/t x) 0 1 - 0xFFFFFFFFFFFFFF)) + 0xFFFFFFFFFF)) (def patient-hash-0 (hash/generate {:fhir/type :fhir/Patient :id "0"})) diff --git a/modules/db/NOTES.md b/modules/db/NOTES.md deleted file mode 100644 index dee7125e5..000000000 --- a/modules/db/NOTES.md +++ /dev/null @@ -1,125 +0,0 @@ -# Blaze - -## Features - -### Versioned read of resources - -* I need to get to a particular version of a resource -* versionId could be the content-hash - -### Normal read returning the last known Version of a Resource - -* need to get the content-hash of a resource given a t -* can be done with the ResourceAsOf index - -### Search - -* stable search also in recent past (at given t) -* - -## Principles - -* don't optimize for deleted resources because deleting a resource is not common - -## Indices - -### Independent from t - -| Name | Key Parts | Value | -|---|---|---| -| SVR | c-hash tid value id hash-prefix | - | -| RSV | tid id hash-prefix c-hash value | - | -| CSVR | co-c-hash co-res-id sp-c-hash tid value id hash-prefix | - | -| CompartmentResource | co-c-hash co-res-id tid id | - | -| SearchParam | code tid | id | -| ActiveSearchParams | id | - | - -### Depend on t - -| Name | Key Parts | Value | -|---|---|---| -| TxSuccess | t | transaction | -| TxError | t | anomaly | -| TByInstant | inst-ms (desc) | t | -| ResourceAsOf | tid id t | hash, state | -| TypeAsOf | tid t id | hash, state | -| SystemAsOf | t tid id | hash, state | -| TypeStats | tid t | total, num-changes | -| SystemStats | t | total, num-changes | - -We can make hashes in SearchParam indices shorter (4-bytes) because we only need to differentiate between the versions of a resource. The odds of a hash collision is 1 out of 10000 for about 1000 versions. In case of a hash collision we would produce a false positive query hit. So we would return more resources instead of less, which is considered fine in FHIR. - -### Search param Value Resource version (SVR) - -The key consists of: - -* c-hash - a 4-byte hash of the code of the search parameter -* tid - the 4-byte type id -* value - the value encoded depending on the search parameter -* id - the logical id of the resource -* hash-prefix - a 4-byte prefix of the resource content hash - -The total size of the key is 4 + 4 + value-size + id-size + 4 = 12 + value-size + id-size bytes. - -The value is empty. - -The key contains the id of the resource for two reasons, first we can skip to the next resource by seeking with max-hash, not having to test all versions of a resource against ResourceAsOf and second, going into ResourceAsOf will be local because it is sorted by id. - -The SVR index is comparable to the AVET index in Datomic. Search parameters are the equivalent of indexed attributes in Datomic. - -### Resource version Search param Value (RSV) - - - -### Compartment Search-param Value Resource (CSVR) - -Same as the SVR index but prefixed with a compartment the resource belongs to. This index is used in [variant searches][2] and in CQL evaluation within the Patient context. In the CQL Patient context all retrieves are relative to one patient. Using that patient as compartment in the CSVR index allows for efficient implementation of that retrieves. - -The key consists of: - -* co-c-hash - a 4-byte hash of the code of the compartment -* co-res-id - the logical id of the resource of the compartment -* c-hash - a 4-byte hash of the code of the search parameter -* tid - the 4-byte type id -* value - the value encoded depending on the search parameter -* id - the logical id of the resource -* hash-prefix - a 4-byte prefix of the resource content hash - -The total size of the key is 4 + co-res-id-size 4 + 4 + value-size + id-size + 4 = 16 + co-res-id-size 4 + value-size + id-size bytes. - -The value is empty. - -### TByInstant - -Provides access to t's by instant (point in time). It encodes the instant as milliseconds since epoch as descending long from Long/MAX_VALUE so that TODO WHY??? - -### ResourceAsOf - -The key consists of: - -* tid - the 4-byte type id -* id - the variable length id (max 64 byte) - - -### TxSuccess - -### TxError - -### TypeStats / SystemStats - -total = number of non-deleted resources -num-changes = total number of changes (creates, updates, deletes) - -## Search - -The FHIR search parameters have different types. The search implementation depends on that types. The following sections describe the implementation by type. - -### Date - -The date search parameter type is used for the data types date, dateTime, instant, Period and Timing. The search is always performed against a range. Both the value given in the search and the target value in resources have either an implicit or an explicit range. For example the range of a date like 2020-02-09 starts at 2020-02-09T00:00:00.000 and ends at 2020-02-09T23:59:59.999. - -By default the search is an equal search were the range of the search value have to fully contain the range of the target value. In addition to the equal search, other search operators are possible. - - -[1]: -[2]: diff --git a/modules/db/src/blaze/db/api.clj b/modules/db/src/blaze/db/api.clj index cd0c73302..81f3b87e2 100644 --- a/modules/db/src/blaze/db/api.clj +++ b/modules/db/src/blaze/db/api.clj @@ -105,7 +105,7 @@ Please use `pull` to obtain the full resource." [db type id] (log/trace "fetch resource handle of" (str type "/" id)) - (p/-resource-handle db (codec/tid type) (codec/id-byte-string id))) + (p/-resource-handle db (codec/tid type) id)) (defn resource-handle? [x] @@ -124,7 +124,7 @@ ([db type] (p/-type-list db (codec/tid type))) ([db type start-id] - (p/-type-list db (codec/tid type) (codec/id-byte-string start-id)))) + (p/-type-list db (codec/tid type) start-id))) (defn type-total @@ -186,7 +186,7 @@ ([db] (p/-system-list db)) ([db start-type start-id] - (p/-system-list db (codec/tid start-type) (codec/id-byte-string start-id)))) + (p/-system-list db (codec/tid start-type) start-id))) (defn system-total @@ -221,10 +221,6 @@ ;; ---- Compartment-Level Functions ------------------------------------------- -(defn- compartment [code id] - [(codec/c-hash code) (codec/id-byte-string id)]) - - (defn list-compartment-resource-handles "Returns a reducible collection of all resource handles of `type` in `db` linked to the compartment with `code` and `id`. @@ -240,10 +236,9 @@ Please use `pull-many` to obtain the full resources." ([db code id type] - (p/-compartment-resource-handles db (compartment code id) (codec/tid type))) + (p/-compartment-resource-handles db [code id] (codec/tid type))) ([db code id type start-id] - (p/-compartment-resource-handles db (compartment code id) (codec/tid type) - (codec/id-byte-string start-id)))) + (p/-compartment-resource-handles db [code id] (codec/tid type) start-id))) (defn compartment-query @@ -321,13 +316,11 @@ History entries are resource handles. Please use `pull-many` to obtain the full resources." ([db type id] - (p/-instance-history db (codec/tid type) (codec/id-byte-string id) nil nil)) + (p/-instance-history db (codec/tid type) id nil nil)) ([db type id start-t] - (p/-instance-history db (codec/tid type) (codec/id-byte-string id) start-t - nil)) + (p/-instance-history db (codec/tid type) id start-t nil)) ([db type id start-t since] - (p/-instance-history db (codec/tid type) (codec/id-byte-string id) start-t - since))) + (p/-instance-history db (codec/tid type) id start-t since))) (defn total-num-of-instance-changes @@ -337,11 +330,9 @@ Optionally a `since` instant can be given to define a point in the past where the calculation should start." ([db type id] - (p/-total-num-of-instance-changes db (codec/tid type) - (codec/id-byte-string id) nil)) + (p/-total-num-of-instance-changes db (codec/tid type) id nil)) ([db type id since] - (p/-total-num-of-instance-changes db (codec/tid type) - (codec/id-byte-string id) since))) + (p/-total-num-of-instance-changes db (codec/tid type) id since))) @@ -362,11 +353,9 @@ ([db type start-t] (p/-type-history db (codec/tid type) start-t nil nil)) ([db type start-t start-id] - (p/-type-history db (codec/tid type) start-t - (some-> start-id codec/id-byte-string) nil)) + (p/-type-history db (codec/tid type) start-t start-id nil)) ([db type start-t start-id since] - (p/-type-history db (codec/tid type) start-t - (some-> start-id codec/id-byte-string) since))) + (p/-type-history db (codec/tid type) start-t start-id since))) (defn total-num-of-type-changes @@ -401,11 +390,9 @@ ([db start-t start-type] (p/-system-history db start-t (some-> start-type codec/tid) nil nil)) ([db start-t start-type start-id] - (p/-system-history db start-t (some-> start-type codec/tid) - (some-> start-id codec/id-byte-string) nil)) + (p/-system-history db start-t (some-> start-type codec/tid) start-id nil)) ([db start-t start-type start-id since] - (p/-system-history db start-t (some-> start-type codec/tid) - (some-> start-id codec/id-byte-string) since))) + (p/-system-history db start-t (some-> start-type codec/tid) start-id since))) (defn total-num-of-system-changes diff --git a/modules/db/src/blaze/db/impl/batch_db.clj b/modules/db/src/blaze/db/impl/batch_db.clj index b249aeb51..d2bec4508 100644 --- a/modules/db/src/blaze/db/impl/batch_db.clj +++ b/modules/db/src/blaze/db/impl/batch_db.clj @@ -10,6 +10,7 @@ [blaze.db.impl.index.compartment.resource :as cr] [blaze.db.impl.index.resource-as-of :as rao] [blaze.db.impl.index.resource-handle :as rh] + [blaze.db.impl.index.resource-id :as ri] [blaze.db.impl.index.search-param-value-resource :as sp-vr] [blaze.db.impl.index.system-as-of :as sao] [blaze.db.impl.index.system-stats :as system-stats] @@ -41,7 +42,8 @@ ;; ---- Instance-Level Functions -------------------------------------------- (-resource-handle [_ tid id] - ((:resource-handle context) tid id)) + (when-let [did ((:resource-id context) tid id)] + ((:resource-handle context) tid did))) @@ -51,7 +53,8 @@ (rao/type-list context tid)) (-type-list [_ tid start-id] - (rao/type-list context tid start-id)) + (when-let [start-did ((:resource-id context) tid start-id)] + (rao/type-list context tid start-did))) (-type-total [_ tid] (let [{:keys [snapshot t]} context] @@ -66,7 +69,8 @@ (rao/system-list context)) (-system-list [_ start-tid start-id] - (rao/system-list context start-tid start-id)) + (when-let [start-did ((:resource-id context) start-tid start-id)] + (rao/system-list context start-tid start-did))) (-system-total [_] (let [{:keys [snapshot t]} context] @@ -77,11 +81,14 @@ ;; ---- Compartment-Level Functions ----------------------------------------- - (-compartment-resource-handles [_ compartment tid] - (cr/resource-handles! context compartment tid)) + (-compartment-resource-handles [_ [code id] tid] + (when-let [did ((:resource-id context) (codec/tid code) id)] + (cr/resource-handles! context [(codec/c-hash code) did] tid))) - (-compartment-resource-handles [_ compartment tid start-id] - (cr/resource-handles! context compartment tid start-id)) + (-compartment-resource-handles [_ [code id] tid start-id] + (when-let [did ((:resource-id context) (codec/tid code) id)] + (when-let [start-did ((:resource-id context) tid start-id)] + (cr/resource-handles! context [(codec/c-hash code) did] tid start-did)))) @@ -98,28 +105,32 @@ ;; ---- Instance-Level History Functions ------------------------------------ (-instance-history [_ tid id start-t since] - (let [{:keys [snapshot raoi t]} context + (let [{:keys [snapshot resource-id raoi t]} context start-t (if (some-> start-t (<= t)) start-t t) end-t (or (some->> since (t-by-instant/t-by-instant snapshot)) 0)] - (rao/instance-history raoi tid id start-t end-t))) + (when-let [did (resource-id tid id)] + (rao/instance-history raoi tid did start-t end-t)))) (-total-num-of-instance-changes [_ tid id since] - (let [{:keys [snapshot resource-handle t]} context + (let [{:keys [snapshot resource-id resource-handle t]} context end-t (or (some->> since (t-by-instant/t-by-instant snapshot)) 0)] - (rao/num-of-instance-changes resource-handle tid id t end-t))) + (if-let [did (resource-id tid id)] + (rao/num-of-instance-changes resource-handle tid did t end-t) + 0))) ;; ---- Type-Level History Functions ---------------------------------------- (-type-history [_ tid start-t start-id since] - (let [{:keys [snapshot t]} context + (let [{:keys [snapshot t resource-id]} context start-t (if (some-> start-t (<= t)) start-t t) - end-t (or (some->> since (t-by-instant/t-by-instant snapshot)) 0)] + end-t (or (some->> since (t-by-instant/t-by-instant snapshot)) 0) + start-did (some->> start-id (resource-id tid))] (reify IReduceInit (reduce [_ rf init] (with-open [taoi (kv/new-iterator snapshot :type-as-of-index)] - (reduce rf init (tao/type-history taoi tid start-t start-id end-t))))))) + (reduce rf init (tao/type-history taoi tid start-t start-did end-t))))))) (-total-num-of-type-changes [_ type since] (let [{:keys [snapshot t]} context @@ -135,13 +146,14 @@ ;; ---- System-Level History Functions -------------------------------------- (-system-history [_ start-t start-tid start-id since] - (let [{:keys [snapshot t]} context + (let [{:keys [snapshot t resource-id]} context start-t (if (some-> start-t (<= t)) start-t t) - end-t (or (some->> since (t-by-instant/t-by-instant snapshot)) 0)] + end-t (or (some->> since (t-by-instant/t-by-instant snapshot)) 0) + start-did (some->> start-id (resource-id start-tid))] (reify IReduceInit (reduce [_ rf init] (with-open [saoi (kv/new-iterator snapshot :system-as-of-index)] - (reduce rf init (sao/system-history saoi start-t start-tid start-id end-t))))))) + (reduce rf init (sao/system-history saoi start-t start-tid start-did end-t))))))) (-total-num-of-system-changes [_ since] (let [{:keys [snapshot t]} context @@ -240,8 +252,9 @@ p/Query (-execute [_ context] (index/type-query context tid clauses)) - (-execute [_ context start-id] - (index/type-query context tid clauses (codec/id-byte-string start-id))) + (-execute [_ {:keys [resource-id] :as context} start-id] + (when-let [start-did (resource-id tid start-id)] + (index/type-query context tid clauses start-did))) (-clauses [_] (decode-clauses clauses))) @@ -250,8 +263,9 @@ p/Query (-execute [_ context] (rao/type-list context tid)) - (-execute [_ context start-id] - (rao/type-list context tid (codec/id-byte-string start-id))) + (-execute [_ {:keys [resource-id] :as context} start-id] + (when-let [start-did (resource-id tid start-id)] + (rao/type-list context tid start-did))) (-clauses [_])) @@ -261,19 +275,20 @@ (index/system-query context clauses))) -(defrecord CompartmentQuery [c-hash tid clauses] +(defrecord CompartmentQuery [c-hash c-tid tid clauses] p/Query - (-execute [_ context arg1] - (index/compartment-query context [c-hash (codec/id-byte-string arg1)] - tid clauses)) + (-execute [_ {:keys [resource-id] :as context} compartment-id] + (when-let [c-res-did (resource-id c-tid compartment-id)] + (index/compartment-query context [c-hash c-res-did] tid clauses))) (-clauses [_] (decode-clauses clauses))) -(defrecord EmptyCompartmentQuery [c-hash tid] +(defrecord EmptyCompartmentQuery [c-hash c-tid tid] p/Query - (-execute [_ context arg1] - (cr/resource-handles! context [c-hash (codec/id-byte-string arg1)] tid)) + (-execute [_ {:keys [resource-id] :as context} compartment-id] + (when-let [c-res-did (resource-id c-tid compartment-id)] + (cr/resource-handles! context [c-hash c-res-did] tid))) (-clauses [_])) @@ -291,6 +306,7 @@ basis-t (let [raoi (kv/new-iterator snapshot :resource-as-of-index)] {:snapshot snapshot + :resource-id (ri/resource-id kv-store) :raoi raoi :resource-handle (rao/resource-handle rh-cache raoi t) :svri (kv/new-iterator snapshot :search-param-value-index) diff --git a/modules/db/src/blaze/db/impl/codec.clj b/modules/db/src/blaze/db/impl/codec.clj index 76c2c5a10..e2436aac6 100644 --- a/modules/db/src/blaze/db/impl/codec.clj +++ b/modules/db/src/blaze/db/impl/codec.clj @@ -24,7 +24,8 @@ (def ^:const ^long c-hash-size Integer/BYTES) (def ^:const ^long v-hash-size Long/BYTES) (def ^:const ^long tid-size Integer/BYTES) -(def ^:const ^long t-size Long/BYTES) +(def ^:const ^long t-size 5) +(def ^:const ^long did-size Long/BYTES) (def ^:const ^long state-size Long/BYTES) (def ^:const ^long max-id-size 64) @@ -211,50 +212,29 @@ ;; ---- Identifier Functions -------------------------------------------------- -(defn id-byte-string - {:inline - (fn [id] - `(bs/from-string ~id StandardCharsets/ISO_8859_1))} - [id] - (bs/from-string id StandardCharsets/ISO_8859_1)) - - -(defn id-string - "Converts the byte-string representation of a resource id into it's string - representation." - {:inline - (fn [id-byte-string] - `(bs/to-string ~id-byte-string StandardCharsets/ISO_8859_1))} - [id-byte-string] - (bs/to-string id-byte-string StandardCharsets/ISO_8859_1)) +(defn id-from-byte-buffer [buf] + (let [id-bytes (byte-array (bb/remaining buf))] + (bb/copy-into-byte-array! buf id-bytes) + (String. id-bytes StandardCharsets/ISO_8859_1))) -(defn id - {:inline - (fn [id-bytes offset length] - `(String. ~id-bytes ~offset ~length StandardCharsets/ISO_8859_1))} - [^bytes id-bytes ^long offset ^long length] - (String. id-bytes offset length StandardCharsets/ISO_8859_1)) +(defn did [t idx] + (+ (bit-shift-left (unchecked-long t) 24) (unchecked-long idx))) ;; ---- Key Functions --------------------------------------------------------- (defn descending-long - "Converts positive longs so that they decrease from 0xFFFFFFFFFFFFFF. + "Converts positive longs so that they decrease from 0xFFFFFFFFFF. This function is used for the point in time `t` value, which is always ordered - descending in indices. The value 0xFFFFFFFFFFFFFF has 7 bytes, so the first - byte will be always the zero byte. This comes handy in indices, because the - zero byte terminates ordering of index segments preceding the `t` value. - - 7 bytes are also plenty for the `t` value because with 5 bytes one could carry - out a transaction every millisecond for 20 years." + descending in indices." {:inline (fn [l] - `(bit-and (bit-not (unchecked-long ~l)) 0xFFFFFFFFFFFFFF))} + `(bit-and (bit-not (unchecked-long ~l)) 0xFFFFFFFFFF))} [l] - (bit-and (bit-not (unchecked-long l)) 0xFFFFFFFFFFFFFF)) + (bit-and (bit-not (unchecked-long l)) 0xFFFFFFFFFF)) (defn c-hash [code] @@ -309,12 +289,12 @@ bs/from-byte-array)) -(defn tid-id - "Returns a byte string with `tid` followed by `id`." - [tid id] - (-> (bb/allocate (unchecked-add-int tid-size (bs/size id))) +(defn tid-did + "Returns a byte string with `tid` followed by `did`." + [tid did] + (-> (bb/allocate (+ tid-size did-size)) (bb/put-int! tid) - (bb/put-byte-string! id) + (bb/put-long! did) bb/flip! bs/from-byte-buffer!)) diff --git a/modules/db/src/blaze/db/impl/db.clj b/modules/db/src/blaze/db/impl/db.clj index 2b9f6ab0c..f8c22fa21 100644 --- a/modules/db/src/blaze/db/impl/db.clj +++ b/modules/db/src/blaze/db/impl/db.clj @@ -3,6 +3,7 @@ (:require [blaze.db.impl.batch-db :as batch-db] [blaze.db.impl.index.resource-as-of :as rao] + [blaze.db.impl.index.resource-id :as ri] [blaze.db.impl.macros :refer [with-open-coll]] [blaze.db.impl.protocols :as p] [blaze.db.kv :as kv]) @@ -35,9 +36,10 @@ (-resource-handle [_ tid id] (let [{:keys [kv-store rh-cache]} node] - (with-open [snapshot (kv/new-snapshot kv-store) - raoi (kv/new-iterator snapshot :resource-as-of-index)] - ((rao/resource-handle rh-cache raoi t) tid id)))) + (when-let [did ((ri/resource-id kv-store) tid id)] + (with-open [snapshot (kv/new-snapshot kv-store) + raoi (kv/new-iterator snapshot :resource-as-of-index)] + ((rao/resource-handle rh-cache raoi t) tid did))))) diff --git a/modules/db/src/blaze/db/impl/index.clj b/modules/db/src/blaze/db/impl/index.clj index 17860e784..1f666f6ce 100644 --- a/modules/db/src/blaze/db/impl/index.clj +++ b/modules/db/src/blaze/db/impl/index.clj @@ -28,15 +28,15 @@ search-param context tid modifier values)) (search-param/resource-handles search-param context tid modifier values)))) - ([context tid clauses start-id] + ([context tid clauses start-did] (let [[[search-param modifier _ values] & other-clauses] clauses] (if (seq other-clauses) (coll/eduction (other-clauses-filter context other-clauses) (search-param/resource-handles - search-param context tid modifier values start-id)) + search-param context tid modifier values start-did)) (search-param/resource-handles - search-param context tid modifier values start-id))))) + search-param context tid modifier values start-did))))) (defn system-query [_ _] @@ -67,12 +67,12 @@ {:arglists '([context resource-handle code] [context resource-handle code target-tid])} - ([{:keys [rsvi] :as context} {:keys [tid id hash]} code] + ([{:keys [rsvi] :as context} {:keys [tid did hash]} code] (coll/eduction (u/reference-resource-handle-mapper context) - (r-sp-v/prefix-keys! rsvi tid (codec/id-byte-string id) hash code))) - ([{:keys [rsvi] :as context} {:keys [tid id hash]} code target-tid] + (r-sp-v/prefix-keys! rsvi tid did hash code))) + ([{:keys [rsvi] :as context} {:keys [tid did hash]} code target-tid] (coll/eduction (u/reference-resource-handle-mapper context) - (r-sp-v/prefix-keys! rsvi tid (codec/id-byte-string id) hash code + (r-sp-v/prefix-keys! rsvi tid did hash code (codec/tid-byte-string target-tid))))) diff --git a/modules/db/src/blaze/db/impl/index/compartment/resource.clj b/modules/db/src/blaze/db/impl/index/compartment/resource.clj index 3c1d5a599..316830d10 100644 --- a/modules/db/src/blaze/db/impl/index/compartment/resource.clj +++ b/modules/db/src/blaze/db/impl/index/compartment/resource.clj @@ -13,29 +13,19 @@ (set! *unchecked-math* :warn-on-boxed) -(def ^:private ^:const ^long max-key-size - (+ codec/c-hash-size codec/max-id-size 1 codec/tid-size codec/max-id-size)) +(def ^:private ^:const ^long seek-key-size + (+ codec/c-hash-size codec/did-size codec/tid-size)) -(def ^:private ^:const ^long except-co-res-id-prefix-size - (+ codec/c-hash-size 1 codec/tid-size)) - - -(defn- key-prefix-size - {:inline - (fn [co-res-id] - `(unchecked-add-int ~except-co-res-id-prefix-size (bs/size ~co-res-id)))} - [co-res-id] - (unchecked-add-int except-co-res-id-prefix-size (bs/size co-res-id))) +(def ^:private ^:const ^long key-size + (+ seek-key-size codec/did-size)) (defn- decode-key - ([] (bb/allocate-direct max-key-size)) + ([] (bb/allocate-direct key-size)) ([buf] - (bb/set-position! buf (unchecked-add-int (bb/position buf) codec/c-hash-size)) - (let [id-size (long (bb/size-up-to-null buf))] - (bb/set-position! buf (+ (bb/position buf) id-size 1 codec/tid-size)) - (bs/from-byte-buffer! buf)))) + (bb/set-position! buf seek-key-size) + (bb/get-long! buf))) (def ^:private remove-deleted-xf @@ -50,29 +40,23 @@ (defn- encode-seek-key "Encodes the key without the id used for seeking to the start of scans." - [compartment tid] - (let [co-c-hash (coll/nth compartment 0) - co-res-id (coll/nth compartment 1)] - (-> (bb/allocate (key-prefix-size co-res-id)) - (bb/put-int! co-c-hash) - (bb/put-byte-string! co-res-id) - (bb/put-byte! 0) - (bb/put-int! tid) - bb/flip! - bs/from-byte-buffer!))) + [[co-c-hash co-res-did] tid] + (-> (bb/allocate seek-key-size) + (bb/put-int! co-c-hash) + (bb/put-long! co-res-did) + (bb/put-int! tid) + bb/flip! + bs/from-byte-buffer!)) (defn- encode-key-buf "Encodes the full key." - [compartment tid id] - (let [co-c-hash (coll/nth compartment 0) - co-res-id (coll/nth compartment 1)] - (-> (bb/allocate (unchecked-add-int (key-prefix-size co-res-id) (bs/size id))) - (bb/put-int! co-c-hash) - (bb/put-byte-string! co-res-id) - (bb/put-byte! 0) - (bb/put-int! tid) - (bb/put-byte-string! id)))) + [[co-c-hash co-res-did] tid did] + (-> (bb/allocate key-size) + (bb/put-int! co-c-hash) + (bb/put-long! co-res-did) + (bb/put-int! tid) + (bb/put-long! did))) (defn- encode-key @@ -85,32 +69,32 @@ "Returns a reducible collection of all resource handles of type with `tid` linked to `compartment`. - An optional `start-id` can be given. + An optional `start-did` can be given. Changes the state of `cri`. Consuming the collection requires exclusive access to `cri`. Doesn't close `cri`." {:arglists '([context compartment tid] - [context compartment tid start-id])} + [context compartment tid start-did])} ([{:keys [cri resource-handle]} compartment tid] (let [seek-key (encode-seek-key compartment tid)] (coll/eduction (resource-handles-xf resource-handle tid) (i/prefix-keys! cri seek-key decode-key seek-key)))) - ([{:keys [cri resource-handle]} compartment tid start-id] + ([{:keys [cri resource-handle]} compartment tid start-did] (coll/eduction (resource-handles-xf resource-handle tid) (i/prefix-keys! cri (encode-seek-key compartment tid) decode-key - (encode-key compartment tid start-id))))) + (encode-key compartment tid start-did))))) (defn index-entry "Returns an entry of the CompartmentResource index build from `compartment`, - `tid` and `id`." - [compartment tid id] + `tid` and `did`." + [compartment tid did] [:compartment-resource-type-index - (bb/array (encode-key-buf compartment tid id)) + (bb/array (encode-key-buf compartment tid did)) bytes/empty]) diff --git a/modules/db/src/blaze/db/impl/index/compartment/search_param_value_resource.clj b/modules/db/src/blaze/db/impl/index/compartment/search_param_value_resource.clj index 15073c634..5c8210919 100644 --- a/modules/db/src/blaze/db/impl/index/compartment/search_param_value_resource.clj +++ b/modules/db/src/blaze/db/impl/index/compartment/search_param_value_resource.clj @@ -3,7 +3,6 @@ (:require [blaze.byte-buffer :as bb] [blaze.byte-string :as bs] - [blaze.coll.core :as coll] [blaze.db.impl.bytes :as bytes] [blaze.db.impl.codec :as codec] [blaze.db.impl.index.search-param-value-resource :as sp-vr] @@ -15,7 +14,7 @@ (defn keys! - "Returns a reducible collection of `[prefix id hash-prefix]` triples starting + "Returns a reducible collection of `[prefix did hash-prefix]` triples starting at `start-key`. Changes the state of `iter`. Consuming the collection requires exclusive @@ -24,28 +23,25 @@ (i/keys! iter sp-vr/decode-key start-key)) -(defn- key-size ^long [co-res-id value] - (+ codec/c-hash-size (bs/size co-res-id) 1 +(defn- key-size ^long [value] + (+ codec/c-hash-size codec/did-size codec/c-hash-size codec/tid-size (bs/size value))) (defn encode-seek-key - [compartment sp-c-hash tid value] - (let [co-c-hash (coll/nth compartment 0) - co-res-id (coll/nth compartment 1)] - (-> (bb/allocate (key-size co-res-id value)) - (bb/put-int! co-c-hash) - (bb/put-byte-string! co-res-id) - (bb/put-byte! 0) - (bb/put-int! sp-c-hash) - (bb/put-int! tid) - (bb/put-byte-string! value) - bb/flip! - bs/from-byte-buffer!))) + [[co-c-hash co-res-did] sp-c-hash tid value] + (-> (bb/allocate (key-size value)) + (bb/put-int! co-c-hash) + (bb/put-long! co-res-did) + (bb/put-int! sp-c-hash) + (bb/put-int! tid) + (bb/put-byte-string! value) + bb/flip! + bs/from-byte-buffer!)) (defn prefix-keys! - "Returns a reducible collection of `[id hash-prefix]` tuples starting at + "Returns a reducible collection of `[did hash-prefix]` tuples starting at `value` and ending when `value` is no longer the prefix of the values processed. @@ -53,32 +49,27 @@ access to `iter`. Doesn't close `iter`." [iter compartment c-hash tid value] (let [seek-key (encode-seek-key compartment c-hash tid value)] - (i/prefix-keys! iter seek-key sp-vr/decode-id-hash-prefix seek-key))) + (i/prefix-keys! iter seek-key sp-vr/decode-did-hash-prefix seek-key))) (defn- encode-key - [compartment sp-c-hash tid value id hash] - (let [co-c-hash (coll/nth compartment 0) - co-res-id (coll/nth compartment 1)] - (-> (bb/allocate (+ (key-size co-res-id value) - (bs/size id) 2 hash/prefix-size)) - (bb/put-int! co-c-hash) - (bb/put-byte-string! co-res-id) - (bb/put-byte! 0) - (bb/put-int! sp-c-hash) - (bb/put-int! tid) - (bb/put-byte-string! value) - (bb/put-byte! 0) - (bb/put-byte-string! id) - (bb/put-byte! (bs/size id)) - (hash/prefix-into-byte-buffer! (hash/prefix hash)) - bb/array))) + [[co-c-hash co-res-did] sp-c-hash tid value did hash] + (-> (bb/allocate (+ (key-size value) 1 codec/did-size hash/prefix-size)) + (bb/put-int! co-c-hash) + (bb/put-long! co-res-did) + (bb/put-int! sp-c-hash) + (bb/put-int! tid) + (bb/put-byte-string! value) + (bb/put-byte! 0) + (bb/put-long! did) + (hash/prefix-into-byte-buffer! (hash/prefix hash)) + bb/array)) (defn index-entry "Returns an entry of the CompartmentSearchParamValueResource index build from - `compartment`, `c-hash`, `tid`, `value`, `id` and `hash`." - [compartment c-hash tid value id hash] + `compartment`, `c-hash`, `tid`, `value`, `did` and `hash`." + [compartment c-hash tid value did hash] [:compartment-search-param-value-index - (encode-key compartment c-hash tid value id hash) + (encode-key compartment c-hash tid value did hash) bytes/empty]) diff --git a/modules/db/src/blaze/db/impl/index/resource_as_of.clj b/modules/db/src/blaze/db/impl/index/resource_as_of.clj index ade5e8d1c..c7f726def 100644 --- a/modules/db/src/blaze/db/impl/index/resource_as_of.clj +++ b/modules/db/src/blaze/db/impl/index/resource_as_of.clj @@ -19,16 +19,16 @@ (set! *unchecked-math* :warn-on-boxed) -(def ^:private ^:const ^long max-key-size - (+ codec/tid-size codec/max-id-size codec/t-size)) +(def ^:private ^:const ^long tid-did-size + (+ codec/tid-size codec/did-size)) -(def ^:private ^:const ^long except-id-key-size - (+ codec/tid-size codec/t-size)) +(def ^:private ^:const ^long key-size + (+ codec/tid-size codec/did-size codec/t-size)) -(def ^:private ^:const ^long value-size - (+ hash/size codec/state-size)) +(def ^:private ^:const ^long max-value-size + (+ hash/size codec/state-size codec/max-id-size)) (defn- key-reader [iter kb] @@ -37,58 +37,28 @@ (kv/key! iter kb))) -(defn- focus-id! - "Reduces the limit of `kb` in order to hide the t and focus on id solely." - [kb] - (bb/set-limit! kb (- (bb/limit kb) codec/t-size))) +(defn- did-marker + "Compares the did part of the current key buffer with the did volatile that + may contain previously seen dids. - -(defn- copy-id! - "Copies the id bytes from the key buffer into the id buffer." - [kb ib] - (let [id-size (bb/remaining kb)] - (bb/set-limit! ib id-size) - (bb/put-byte-buffer! ib kb) - (bb/flip! ib) - (bb/rewind! ib)) - (bb/set-limit! kb (unchecked-add-int (bb/limit kb) codec/t-size))) - - -(defn- skip-id! - "Does the same to `position` and `limit` of `kb` as `copy-id!` but doesn't - copy anything." - [kb ib] - (bb/set-position! kb (unchecked-add-int (bb/position kb) (bb/remaining ib))) - (bb/set-limit! kb (unchecked-add-int (bb/limit kb) codec/t-size))) - - -(defn- id-marker - "Compares the id part of the current key buffer with the id buffer that may - contain previously seen ids. - - Returns true if the id changed over the previously seen one. Keeps the id - buffer up to date." - [kb ib] + Returns true if the did changed over the previously seen one. Keeps the did + volatile up to date." + [kb did-box] (fn [_] - (focus-id! kb) - (cond - (zero? (bb/limit ib)) + (if (nil? @did-box) (do - (copy-id! kb ib) + (vreset! did-box (bb/get-long! kb)) false) - (= kb ib) - (do - (skip-id! kb ib) - false) - - :else - (do - (copy-id! kb ib) - true)))) + (let [did (bb/get-long! kb)] + (if (= did @did-box) + false + (do + (vreset! did-box did) + true)))))) -(defn- tid-marker [kb tid-box ib id-marker] +(defn- tid-marker [kb tid-box did-box id-marker] (fn [x] (let [tid (bb/get-int! kb) last-tid @tid-box] @@ -104,15 +74,15 @@ :else (do (vreset! tid-box tid) - (bb/set-limit! ib 0) + (vreset! did-box nil) (id-marker x) true))))) (defn- new-entry! "Creates a new resource handle entry." - [tid ib vb t] - (rh/resource-handle tid (codec/id (bb/array ib) 0 (bb/remaining ib)) t vb)) + [tid did-box vb t] + (rh/resource-handle tid @did-box t vb)) (defn- type-entry-creator @@ -121,28 +91,29 @@ Uses `iter` to read the `hash` and `state` when needed. Supplies `tid` to the created entry." - [tid iter kb ib base-t] - (let [vb (bb/allocate-direct value-size)] - #(let [t (codec/descending-long (bb/get-long! kb))] - (when (<= t ^long base-t) + [tid iter kb did-box base-t] + (let [vb (bb/allocate-direct max-value-size)] + #(let [t (codec/descending-long (bb/get-5-byte-long! kb))] + (when (<= t (unchecked-long base-t)) (bb/clear! vb) (kv/value! iter vb) - (new-entry! tid ib vb t))))) + (new-entry! tid did-box vb t))))) (defn- system-entry-creator - [tid-box iter kb ib base-t] - (let [vb (bb/allocate-direct value-size)] - #(let [t (codec/descending-long (bb/get-long! kb))] - (when (<= t ^long base-t) + [tid-box iter kb did-box base-t] + (let [vb (bb/allocate-direct max-value-size)] + #(let [t (codec/descending-long (bb/get-5-byte-long! kb))] + (when (<= t (unchecked-long base-t)) (bb/clear! vb) (kv/value! iter vb) - (new-entry! @tid-box ib vb t))))) + (new-entry! @tid-box did-box vb t))))) (defn- group-by-id - "Returns a stateful transducer which takes flags from `id-marker` and supplies - resource handle entries to the reduce function after id changes happen." + "Returns a stateful transducer which takes flags from `did-marker` and + supplies resource handle entries to the reduce function after did changes + happen." [entry-creator] (let [state (volatile! nil) search-entry! #(when-let [e (entry-creator)] @@ -175,17 +146,17 @@ result)))))))) -(defn- encode-key-buf [tid id t] - (-> (bb/allocate (unchecked-add-int except-id-key-size (bs/size id))) +(defn- encode-key-buf [tid did t] + (-> (bb/allocate key-size) (bb/put-int! tid) - (bb/put-byte-string! id) - (bb/put-long! (codec/descending-long t)))) + (bb/put-long! did) + (bb/put-5-byte-long! (codec/descending-long t)))) (defn encode-key "Encodes the key of the ResourceAsOf index from `tid`, `id` and `t`." - [tid id t] - (bb/array (encode-key-buf tid id t))) + [tid did t] + (bb/array (encode-key-buf tid did t))) (defn- starts-with-tid? [^long tid kb] @@ -197,13 +168,13 @@ (defn- type-list-xf [{:keys [raoi t]} tid] - (let [kb (bb/allocate-direct max-key-size) - ib (bb/set-limit! (bb/allocate codec/max-id-size) 0) - entry-creator (type-entry-creator tid raoi kb ib t)] + (let [kb (bb/allocate-direct key-size) + did-box (volatile! nil) + entry-creator (type-entry-creator tid raoi kb did-box t)] (comp (map (key-reader raoi kb)) (take-while (starts-with-tid? tid kb)) - (map (id-marker kb ib)) + (map (did-marker kb did-box)) (group-by-id entry-creator) remove-deleted-xf))) @@ -211,49 +182,49 @@ (defn- start-key ([tid] (-> (Ints/toByteArray tid) bs/from-byte-array)) - ([tid start-id t] - (-> (encode-key-buf tid start-id t) + ([tid start-did t] + (-> (encode-key-buf tid start-did t) bb/flip! bs/from-byte-buffer!))) (defn type-list "Returns a reducible collection of all resource handles of type with `tid` - ordered by resource id. + ordered by resource did. - The list starts at the optional `start-id`. + The list starts at the optional `start-did`. - The ResourceAsOf index consists of keys with three parts: `tid`, `id` and - `t`. The `tid` is a 4-byte hash of the resource type, the `id` a variable - length byte array of the resource id and `t` is an 8-byte long of the - transaction number. The value of the ResourceAsOf index contains two parts: - `hash` and `state`. The `hash` is a 32-byte content hash of the resource and - the `state` is an 8-byte long encoding create, put, delete state and a local - version counter of the resource. + The ResourceAsOf index consists of keys with three parts: `tid`, `did` and + `t`. The `tid` is a 4-byte hash of the resource type, the `did` a 8-byte long + of the resource did and `t` is an 8-byte long of the transaction number. The + value of the ResourceAsOf index contains three parts: `hash`, `state` and + `id`. The `hash` is a 32-byte content hash of the resource, the `state` is an + 8-byte long encoding create, put, delete state and a local version counter of + the resource and the `id` is the resource id. The ResourceAsOf index contains one entry for each resource version. For the type list, only that versions not newer than `t` are returned. For example, an index containing two versions of the same resource looks like this: - < tid-0, id-0, t=2 > < hash-2, state-2 > - < tid-0, id-0, t=1 > < hash-1, state-1 > + < tid-0, did-0, t=2 > < hash-2, state-2, id-0 > + < tid-0, did-0, t=1 > < hash-1, state-1, id-0 > - Here the `tid` and `id` are the same, because the resource is the same and + Here the `tid` and `did` are the same, because the resource is the same and only the versions differ. The newer version has a `t` of `2` and the older version a `t` of `1`. `t` values are simply an incrementing transaction number. The content hashes and states also differ. A `type-list` call with a - `t` of two should return the newest version of the resource, were a call with + `t` of `2` should return the newest version of the resource, were a call with a `t` of `1` should return the older version. The implementation iterates over the ResourceAsOf index, starting with `tid` - and possible `start-id`. It goes from higher `t` values to lower `t` values + and possible `start-did`. It goes from higher `t` values to lower `t` values because they are encoded in descending order. For each entry in the index, the following things are done: * check if the end of the index is reached * check if the first 4 bytes are still the same `tid` as given - * for each `id` bytes seen over multiple entries, return that entry with + * for each `did` bytes seen over multiple entries, return that entry with `t` less than or equal to the `t` given A non-optimized implementation would use the following transducer, assuming @@ -263,72 +234,65 @@ (comp (map decode-entry) (take-while tid-valid?) - (partition-by id) + (partition-by did) (map pick-entry-according-to-t)) First, the whole entry consisting of its key and value are decoded completely into an immutable data structure. Than the `tid` is tested against the given `tid` in order to stop iterating before hitting the next type. Third, all - entries are partitioned by id which each partition containing a list of - entries with the same id. Last, the entry with the right `t` is picked. + entries are partitioned by did which each partition containing a list of + entries with the same did. Last, the entry with the right `t` is picked. This non-optimized implementation has several disadvantages. First, each entry is decoded fully but the `t`, `hash` and `state` part is only needed from - fewer entries. Even the `id` has only to be compared to a reference `id`, but - not fully decoded, for non-matching entries. Second `partition-by` creates an - vector of all entries with the same id. This leads to more object allocation - and time spend. + fewer entries. Second `partition-by` creates an vector of all entries with the + same sid. This leads to more object allocation and time spend. The implementation used here avoids excessive object allocation altogether. - Uses three byte buffers to avoid object allocation as much as + Uses two byte buffers and one volatile to avoid object allocation as much as possible. The first buffer is an off-heap key buffer. Each key is read into - the key buffer. The second buffer is an heap allocated id buffer. The id bytes - of the first key are copied into the id buffer and later copied only on id - change. To detect id changes, the id part of the key buffer and the id buffer - are compared. When a resource is created the heap allocated byte array of the - id buffer is fed into the string constructor. The string constructor will copy - the bytes from the id buffer for immutability. So for each returned resource - handle, a String and the internal byte array, it used, are allocated. The - second byte array which is allocated for each resource handle is the byte - array of the hash. No further byte arrays are allocated during the whole - iteration. The state and t which are both longs are read from the off-heap key - and value buffer. The hash and state which are read from the value buffer are - only read once for each resource handle." + the key buffer. To detect did changes, the did is parsed from the key buffer + and compared with the did from the volatile. The second byte array which is + allocated for each resource handle is the byte array of the hash. No further + byte arrays are allocated during the whole iteration. The state and t which + are both longs are read from the off-heap key and value buffer. The hash and + state which are read from the value buffer are only read once for each + resource handle." ([{:keys [raoi] :as context} tid] (coll/eduction (type-list-xf context tid) (i/iter! raoi (start-key tid)))) - ([{:keys [raoi t] :as context} tid start-id] + ([{:keys [raoi t] :as context} tid start-did] (coll/eduction (type-list-xf context tid) - (i/iter! raoi (start-key tid start-id t))))) + (i/iter! raoi (start-key tid start-did t))))) (defn- system-list-xf [{:keys [raoi t]} start-tid] - (let [kb (bb/allocate-direct max-key-size) + (let [kb (bb/allocate-direct key-size) tid-box (volatile! start-tid) - ib (bb/set-limit! (bb/allocate codec/max-id-size) 0)] + did-box (volatile! nil)] (comp (map (key-reader raoi kb)) - (map (tid-marker kb tid-box ib (id-marker kb ib))) - (group-by-id (system-entry-creator tid-box raoi kb ib t)) + (map (tid-marker kb tid-box did-box (did-marker kb did-box))) + (group-by-id (system-entry-creator tid-box raoi kb did-box t)) remove-deleted-xf))) (defn system-list - "Returns a reducible collection of all resource handles ordered by resource - tid and resource id. + "Returns a reducible collection of all resource handles ordered by tid and + did. - The list starts at the optional `start-tid` and `start-id`." + The list starts at the optional `start-tid` and `start-did`." ([{:keys [raoi] :as context}] (coll/eduction (system-list-xf context nil) (i/iter! raoi))) - ([{:keys [raoi t] :as context} start-tid start-id] + ([{:keys [raoi t] :as context} start-tid start-did] (coll/eduction (system-list-xf context start-tid) - (i/iter! raoi (start-key start-tid start-id t))))) + (i/iter! raoi (start-key start-tid start-did t))))) (defn decoder @@ -346,45 +310,42 @@ Both byte buffers are changed during decoding and have to be reset accordingly after decoding." [] - (let [ib (byte-array codec/max-id-size)] - (fn - ([] - [(bb/allocate-direct max-key-size) - (bb/allocate-direct value-size)]) - ([kb vb] - (rh/resource-handle - (bb/get-int! kb) - (let [id-size (- (bb/remaining kb) codec/t-size)] - (bb/copy-into-byte-array! kb ib 0 id-size) - (codec/id ib 0 id-size)) - (codec/descending-long (bb/get-long! kb)) - vb))))) - - -(defn- instance-history-key-valid? [^long tid id ^long end-t] + (fn + ([] + [(bb/allocate-direct key-size) + (bb/allocate-direct max-value-size)]) + ([kb vb] + (rh/resource-handle + (bb/get-int! kb) + (bb/get-long! kb) + (codec/descending-long (bb/get-5-byte-long! kb)) + vb)))) + + +(defn- instance-history-key-valid? [^long tid ^long did ^long end-t] (fn [resource-handle] (and (= (rh/tid resource-handle) tid) - (= (rh/id resource-handle) id) + (= (rh/did resource-handle) did) (< end-t (rh/t resource-handle))))) (defn instance-history "Returns a reducible collection of all versions between `start-t` (inclusive) - and `end-t` (exclusive) of the resource with `tid` and `id`. + and `end-t` (exclusive) of the resource with `tid` and `did`. Versions are resource handles." - [raoi tid id start-t end-t] + [raoi tid did start-t end-t] (coll/eduction - (take-while (instance-history-key-valid? tid (codec/id-string id) end-t)) - (i/kvs! raoi (decoder) (start-key tid id start-t)))) + (take-while (instance-history-key-valid? tid did end-t)) + (i/kvs! raoi (decoder) (start-key tid did start-t)))) -(defn- resource-handle** [raoi tb kb vb tid id t] +(defn- resource-handle* [raoi tb kb vb tid did t] ;; fill target buffer (bb/clear! tb) (bb/put-int! tb tid) - (bb/put-byte-string! tb id) - (bb/put-long! tb (codec/descending-long t)) + (bb/put-long! tb did) + (bb/put-5-byte-long! tb (codec/descending-long t)) ;; flip target buffer to be ready for seek (bb/flip! tb) (kv/seek-buffer! raoi tb) @@ -396,41 +357,39 @@ ;; would find the next resource ;; focus target buffer on tid and id (bb/rewind! tb) - (bb/set-limit! tb (unchecked-subtract-int (bb/limit tb) codec/t-size)) + (bb/set-limit! tb tid-did-size) ;; focus key buffer on tid and id - (bb/set-limit! kb (unchecked-subtract-int (bb/limit kb) codec/t-size)) + (bb/set-limit! kb tid-did-size) (when (= tb kb) ;; focus key buffer on t - (let [limit (bb/limit kb)] - (bb/set-position! kb limit) - (bb/set-limit! kb (unchecked-add-int limit codec/t-size))) + (bb/set-position! kb tid-did-size) + (bb/set-limit! kb key-size) ;; read value (bb/clear! vb) (kv/value! raoi vb) ;; create resource handle (rh/resource-handle tid - (codec/id-string id) - (codec/descending-long (bb/get-long! kb)) + did + (codec/descending-long (bb/get-5-byte-long! kb)) vb)))) ;; For performance reasons, we use that special Key class instead of a a simple ;; triple vector -(deftype Key [^long tid id ^long t] +(deftype Key [^long tid ^long did ^long t] Object - (equals [key x] - (or (identical? key x) - (and (instance? Key x) - (= tid (.-tid ^Key x)) - (.equals id (.-id ^Key x)) - (= t (.-t ^Key x))))) + (equals [_ x] + ;; skip the instanceof check, because keys are only compared to other keys + (and (= tid (.-tid ^Key x)) + (= did (.-did ^Key x)) + (= t (.-t ^Key x)))) (hashCode [_] - (-> tid + (-> (Long/hashCode tid) (unchecked-multiply-int 31) - (unchecked-add-int (.hashCode id)) + (unchecked-add-int (Long/hashCode did)) (unchecked-multiply-int 31) - (unchecked-add-int t)))) + (unchecked-add-int (Long/hashCode t))))) (defn resource-handle @@ -441,23 +400,23 @@ The returned function can't be called concurrently." [rh-cache raoi t] - (let [tb (bb/allocate-direct max-key-size) - kb (bb/allocate-direct max-key-size) - vb (bb/allocate-direct value-size) + (let [tb (bb/allocate-direct key-size) + kb (bb/allocate-direct key-size) + vb (bb/allocate-direct max-value-size) rh (reify Function (apply [_ key] - (resource-handle** raoi tb kb vb (.-tid ^Key key) - (.-id ^Key key) (.-t ^Key key))))] + (resource-handle* raoi tb kb vb (.-tid ^Key key) + (.-did ^Key key) (.-t ^Key key))))] (fn resource-handle - ([tid id] - (resource-handle tid id t)) - ([tid id t] - (.get ^Cache rh-cache (Key. tid id t) rh))))) + ([tid did] + (resource-handle tid did t)) + ([tid did t] + (.get ^Cache rh-cache (Key. tid did t) rh))))) (defn num-of-instance-changes "Returns the number of changes between `start-t` (inclusive) and `end-t` - (inclusive) of the resource with `tid` and `id`." - [resource-handle tid id start-t end-t] - (- (long (:num-changes (resource-handle tid id start-t) 0)) - (long (:num-changes (resource-handle tid id end-t) 0)))) + (inclusive) of the resource with `tid` and `did`." + [resource-handle tid did start-t end-t] + (- (long (:num-changes (resource-handle tid did start-t) 0)) + (long (:num-changes (resource-handle tid did end-t) 0)))) diff --git a/modules/db/src/blaze/db/impl/index/resource_handle.clj b/modules/db/src/blaze/db/impl/index/resource_handle.clj index 65b2b8bcb..13001007c 100644 --- a/modules/db/src/blaze/db/impl/index/resource_handle.clj +++ b/modules/db/src/blaze/db/impl/index/resource_handle.clj @@ -13,7 +13,7 @@ (set! *unchecked-math* :warn-on-boxed) -(deftype ResourceHandle [^int tid id ^long t hash ^long num-changes op] +(deftype ResourceHandle [^int tid ^long did ^long t hash ^long num-changes op id] p/FhirType (-type [_] ;; TODO: maybe cache this @@ -25,26 +25,29 @@ (valAt [_ key not-found] (case key :tid tid - :id id + :did did :t t :hash hash :num-changes num-changes :op op + :id id not-found)) Object + (toString [_] + (format "%s/%s" (codec/tid->type tid) id)) (equals [rh x] (or (identical? rh x) (and (instance? ResourceHandle x) (= tid (.-tid ^ResourceHandle x)) - (.equals id (.-id ^ResourceHandle x)) + (= did (.-did ^ResourceHandle x)) (= t (.-t ^ResourceHandle x))))) (hashCode [_] - (-> tid + (-> (Long/hashCode tid) (unchecked-multiply-int 31) - (unchecked-add-int (.hashCode id)) + (unchecked-add-int (Long/hashCode did)) (unchecked-multiply-int 31) - (unchecked-add-int t)))) + (unchecked-add-int (Long/hashCode t))))) (defn state->num-changes @@ -67,16 +70,11 @@ "Creates a new resource handle. The type of that handle will be the keyword `:fhir/`." - [tid id t value-buffer] + [tid did t value-buffer] (let [hash (hash/from-byte-buffer! value-buffer) state (bb/get-long! value-buffer)] - (ResourceHandle. - tid - id - t - hash - (state->num-changes state) - (state->op state)))) + (ResourceHandle. tid did t hash (state->num-changes state) (state->op state) + (codec/id-from-byte-buffer value-buffer)))) (defn resource-handle? [x] @@ -97,10 +95,10 @@ (.-tid ^ResourceHandle rh)) -(defn id - {:inline (fn [rh] `(.-id ~(with-meta rh {:tag `ResourceHandle})))} +(defn did + {:inline (fn [rh] `(.-did ~(with-meta rh {:tag `ResourceHandle})))} [rh] - (.-id ^ResourceHandle rh)) + (.-did ^ResourceHandle rh)) (defn t @@ -115,5 +113,11 @@ (.-hash ^ResourceHandle rh)) +(defn id + {:inline (fn [rh] `(.-id ~(with-meta rh {:tag `ResourceHandle})))} + [rh] + (.-id ^ResourceHandle rh)) + + (defn reference [rh] (str (codec/tid->type (tid rh)) "/" (id rh))) diff --git a/modules/db/src/blaze/db/impl/index/resource_id.clj b/modules/db/src/blaze/db/impl/index/resource_id.clj new file mode 100644 index 000000000..203f5470a --- /dev/null +++ b/modules/db/src/blaze/db/impl/index/resource_id.clj @@ -0,0 +1,34 @@ +(ns blaze.db.impl.index.resource-id + "Functions for accessing the ResourceId index." + (:require + [blaze.byte-buffer :as bb] + [blaze.db.impl.codec :as codec] + [blaze.db.kv :as kv]) + (:import + [com.google.common.primitives Longs] + [java.nio.charset StandardCharsets])) + + +(set! *warn-on-reflection* true) +(set! *unchecked-math* :warn-on-boxed) + + +(defn encode-key [tid ^String id] + (-> (bb/allocate (+ (int codec/tid-size) (.length id))) + (bb/put-int! tid) + (bb/put-byte-array! (.getBytes id StandardCharsets/ISO_8859_1)) + bb/array)) + + +(defn resource-id [kv-store] + (fn [tid id] + (some-> (kv/get kv-store :resource-id-index (encode-key tid id)) + Longs/fromByteArray))) + + +(defn encode-value [did] + (Longs/toByteArray did)) + + +(defn index-entry [tid id did] + [:resource-id-index (encode-key tid id) (encode-value did)]) diff --git a/modules/db/src/blaze/db/impl/index/resource_search_param_value.clj b/modules/db/src/blaze/db/impl/index/resource_search_param_value.clj index 66269819a..bf7008750 100644 --- a/modules/db/src/blaze/db/impl/index/resource_search_param_value.clj +++ b/modules/db/src/blaze/db/impl/index/resource_search_param_value.clj @@ -14,48 +14,48 @@ (set! *unchecked-math* :warn-on-boxed) -(def ^:private ^:const ^long key-buffer-capacity +(def ^:const ^long key-buffer-capacity "Most search param value keys should fit into this size." - 128) + 64) + + +(def ^:private ^:const ^long value-pos + (+ codec/tid-size codec/did-size hash/prefix-size codec/c-hash-size)) (defn- decode-value "Decodes the value from the key." ([] (bb/allocate-direct key-buffer-capacity)) ([buf] - (bb/set-position! buf (unchecked-add-int (bb/position buf) codec/tid-size)) - (let [id-size (long (bb/size-up-to-null buf))] - (bb/set-position! buf (+ (bb/position buf) id-size 1 hash/prefix-size - codec/c-hash-size)) - (bs/from-byte-buffer! buf)))) + (bb/set-position! buf value-pos) + (bs/from-byte-buffer! buf))) -(defn- key-size ^long [id] - (+ codec/tid-size 1 (bs/size id) hash/prefix-size codec/c-hash-size)) +(def ^:private ^:const ^long seek-key-size + (+ codec/tid-size codec/did-size hash/prefix-size codec/c-hash-size)) -(defn- encode-key-buf-1 [size tid id hash c-hash] +(defn- encode-key-buf-1 [size tid did hash c-hash] (-> (bb/allocate size) (bb/put-int! tid) - (bb/put-byte-string! id) - (bb/put-byte! 0) + (bb/put-long! did) (hash/prefix-into-byte-buffer! (hash/prefix hash)) (bb/put-int! c-hash))) (defn- encode-key-buf - ([tid id hash c-hash] - (encode-key-buf-1 (key-size id) tid id hash c-hash)) - ([tid id hash c-hash value] - (-> (encode-key-buf-1 (+ (key-size id) (bs/size value)) tid id hash c-hash) + ([tid did hash c-hash] + (encode-key-buf-1 seek-key-size tid did hash c-hash)) + ([tid did hash c-hash value] + (-> (encode-key-buf-1 (+ seek-key-size (bs/size value)) tid did hash c-hash) (bb/put-byte-string! value)))) (defn- encode-key - ([tid id hash c-hash] - (-> (encode-key-buf tid id hash c-hash) bb/flip! bs/from-byte-buffer!)) - ([tid id hash c-hash value] - (-> (encode-key-buf tid id hash c-hash value) bb/flip! bs/from-byte-buffer!))) + ([tid did hash c-hash] + (-> (encode-key-buf tid did hash c-hash) bb/flip! bs/from-byte-buffer!)) + ([tid did hash c-hash value] + (-> (encode-key-buf tid did hash c-hash value) bb/flip! bs/from-byte-buffer!))) (defn next-value! @@ -67,14 +67,12 @@ {:arglists '([iter resource-handle c-hash] [iter resource-handle c-hash prefix-value value])} - ([iter {:keys [tid id hash]} c-hash] - (let [id (codec/id-byte-string id) - key (encode-key tid id hash c-hash)] + ([iter {:keys [tid did hash]} c-hash] + (let [key (encode-key tid did hash c-hash)] (coll/first (i/prefix-keys! iter key decode-value key)))) - ([iter {:keys [tid id hash]} c-hash prefix-value value] - (let [id (codec/id-byte-string id) - prefix-key (encode-key tid id hash c-hash prefix-value) - start-key (encode-key tid id hash c-hash value)] + ([iter {:keys [tid did hash]} c-hash prefix-value value] + (let [prefix-key (encode-key tid did hash c-hash prefix-value) + start-key (encode-key tid did hash c-hash value)] (coll/first (i/prefix-keys! iter prefix-key decode-value start-key))))) @@ -84,34 +82,33 @@ `prefix-value`." {:arglists '([iter resource-handle c-hash prefix-value value])} - [iter {:keys [tid id hash]} c-hash prefix-value value] - (let [id (codec/id-byte-string id) - prefix-key (encode-key tid id hash c-hash prefix-value) - start-key (encode-key tid id hash c-hash value)] + [iter {:keys [tid did hash]} c-hash prefix-value value] + (let [prefix-key (encode-key tid did hash c-hash prefix-value) + start-key (encode-key tid did hash c-hash value)] (coll/first (i/prefix-keys-prev! iter prefix-key decode-value start-key)))) (defn prefix-keys! "Returns a reducible collection of decoded values from keys starting at - `start-value` (optional) and ending when the prefix of `tid`, `id`, `hash`, + `start-value` (optional) and ending when the prefix of `tid`, `did`, `hash`, `c-hash` and `prefix-value` (optional) is no longer a prefix of the keys processed. Changes the state of `iter`. Consuming the collection requires exclusive access to `iter`. Doesn't close `iter`." - ([iter tid id hash c-hash] - (let [key (encode-key tid id hash c-hash)] + ([iter tid did hash c-hash] + (let [key (encode-key tid did hash c-hash)] (i/prefix-keys! iter key decode-value key))) - ([iter tid id hash c-hash prefix-value] - (let [prefix-key (encode-key tid id hash c-hash prefix-value)] + ([iter tid did hash c-hash prefix-value] + (let [prefix-key (encode-key tid did hash c-hash prefix-value)] (i/prefix-keys! iter prefix-key decode-value prefix-key))) - ([iter tid id hash c-hash prefix-value start-value] - (let [prefix-key (encode-key tid id hash c-hash prefix-value) - start-key (encode-key tid id hash c-hash start-value)] + ([iter tid did hash c-hash prefix-value start-value] + (let [prefix-key (encode-key tid did hash c-hash prefix-value) + start-key (encode-key tid did hash c-hash start-value)] (i/prefix-keys! iter prefix-key decode-value start-key)))) -(defn index-entry [tid id hash c-hash value] +(defn index-entry [tid did hash c-hash value] [:resource-value-index - (bb/array (encode-key-buf tid id hash c-hash value)) + (bb/array (encode-key-buf tid did hash c-hash value)) bytes/empty]) diff --git a/modules/db/src/blaze/db/impl/index/rts_as_of.clj b/modules/db/src/blaze/db/impl/index/rts_as_of.clj index 590b909af..a8b34eaa6 100644 --- a/modules/db/src/blaze/db/impl/index/rts_as_of.clj +++ b/modules/db/src/blaze/db/impl/index/rts_as_of.clj @@ -12,32 +12,30 @@ [blaze.db.impl.index.type-as-of :as tao] [blaze.fhir.hash :as hash]) (:import - [clojure.lang Numbers])) + [clojure.lang Numbers] + [java.nio.charset StandardCharsets])) (set! *warn-on-reflection* true) (set! *unchecked-math* :warn-on-boxed) -(def ^:private ^:const ^long value-size - (+ hash/size codec/state-size)) - - (defn- state ^long [^long num-changes op] (cond-> (bit-shift-left num-changes 8) (identical? :create op) (Numbers/setBit 1) (identical? :delete op) (Numbers/setBit 0))) -(defn encode-value [hash num-changes op] - (-> (bb/allocate value-size) +(defn encode-value [hash num-changes op ^String id] + (-> (bb/allocate (+ hash/size codec/state-size (.length id))) (hash/into-byte-buffer! hash) (bb/put-long! (state num-changes op)) + (bb/put-byte-array! (.getBytes id StandardCharsets/ISO_8859_1)) bb/array)) -(defn index-entries [tid id t hash num-changes op] - (let [value (encode-value hash num-changes op)] - [[:resource-as-of-index (rao/encode-key tid id t) value] - [:type-as-of-index (tao/encode-key tid t id) value] - [:system-as-of-index (sao/encode-key t tid id) value]])) +(defn index-entries [tid did t hash num-changes op id] + (let [value (encode-value hash num-changes op id)] + [[:resource-as-of-index (rao/encode-key tid did t) value] + [:type-as-of-index (tao/encode-key tid t did) value] + [:system-as-of-index (sao/encode-key t tid did) value]])) diff --git a/modules/db/src/blaze/db/impl/index/search_param_value_resource.clj b/modules/db/src/blaze/db/impl/index/search_param_value_resource.clj index 35ba22a73..8a97fdb60 100644 --- a/modules/db/src/blaze/db/impl/index/search_param_value_resource.clj +++ b/modules/db/src/blaze/db/impl/index/search_param_value_resource.clj @@ -5,7 +5,6 @@ [blaze.byte-string :as bs] [blaze.db.impl.bytes :as bytes] [blaze.db.impl.codec :as codec] - [blaze.db.impl.index.search-param-value-resource.impl :as impl] [blaze.db.impl.iterators :as i] [blaze.fhir.hash :as hash])) @@ -13,29 +12,27 @@ (set! *unchecked-math* :warn-on-boxed) -(def ^:private ^:const ^long key-buffer-capacity +(def ^:const ^long key-buffer-capacity "Most search param value keys should fit into this size." - 128) + 64) (defn decode-key - "Returns a triple of `[prefix id hash-prefix]`. + "Returns a triple of `[prefix did hash-prefix]`. The prefix contains the c-hash, tid and value parts as encoded byte string." ([] (bb/allocate-direct key-buffer-capacity)) ([buf] - (let [id-size (impl/id-size buf) - all-size (bb/remaining buf) - prefix-size (- all-size 2 id-size hash/prefix-size) + (let [all-size (bb/remaining buf) + prefix-size (- all-size 1 codec/did-size hash/prefix-size) prefix (bs/from-byte-buffer! buf prefix-size) _ (bb/get-byte! buf) - id (bs/from-byte-buffer! buf id-size)] - (bb/get-byte! buf) - [prefix id (hash/prefix-from-byte-buffer! buf)]))) + did (bb/get-long! buf)] + [prefix did (hash/prefix-from-byte-buffer! buf)]))) (defn keys! - "Returns a reducible collection of `[prefix id hash-prefix]` triples starting + "Returns a reducible collection of `[prefix did hash-prefix]` triples starting at `start-key`. The prefix contains the c-hash, tid and value parts as encoded byte string. @@ -50,13 +47,6 @@ (+ codec/c-hash-size codec/tid-size)) -(defn- key-size - (^long [value] - (+ base-key-size (bs/size value))) - (^long [value id] - (+ (key-size value) (bs/size id) 2))) - - (defn encode-seek-key ([c-hash tid] (-> (bb/allocate base-key-size) @@ -65,20 +55,19 @@ bb/flip! bs/from-byte-buffer!)) ([c-hash tid value] - (-> (bb/allocate (key-size value)) + (-> (bb/allocate (+ base-key-size (bs/size value))) (bb/put-int! c-hash) (bb/put-int! tid) (bb/put-byte-string! value) bb/flip! bs/from-byte-buffer!)) - ([c-hash tid value id] - (-> (bb/allocate (key-size value id)) + ([c-hash tid value did] + (-> (bb/allocate (+ base-key-size (bs/size value) 1 codec/did-size)) (bb/put-int! c-hash) (bb/put-int! tid) (bb/put-byte-string! value) (bb/put-byte! 0) - (bb/put-byte-string! id) - (bb/put-byte! (bs/size id)) + (bb/put-long! did) bb/flip! bs/from-byte-buffer!))) @@ -90,86 +79,80 @@ (defn encode-seek-key-for-prev ([c-hash tid value] (bs/concat (encode-seek-key c-hash tid value) bs-ff)) - ([c-hash tid value id] - (bs/concat (encode-seek-key c-hash tid value id) bs-ff))) + ([c-hash tid value did] + (bs/concat (encode-seek-key c-hash tid value did) bs-ff))) -(defn decode-value-id-hash-prefix - "Returns a triple of `[value id hash-prefix]`." +(defn decode-value-did-hash-prefix + "Returns a triple of `[value did hash-prefix]`." ([] (bb/allocate-direct key-buffer-capacity)) ([buf] - (let [id-size (impl/id-size buf) - _ (bb/set-position! buf base-key-size) - all-size (bb/remaining buf) - value-size (- all-size 2 id-size hash/prefix-size) + (let [_ (bb/set-position! buf base-key-size) + remaining-size (bb/remaining buf) + value-size (- remaining-size 1 codec/did-size hash/prefix-size) value (bs/from-byte-buffer! buf value-size) _ (bb/get-byte! buf) - id (bs/from-byte-buffer! buf id-size)] - (bb/get-byte! buf) - [value id (hash/prefix-from-byte-buffer! buf)]))) + did (bb/get-long! buf)] + [value did (hash/prefix-from-byte-buffer! buf)]))) (defn all-keys! - "Returns a reducible collection of `[value id hash-prefix]` triples of the + "Returns a reducible collection of `[value did hash-prefix]` triples of the whole range prefixed with `c-hash` and `tid` starting with `start-value` and - `start-id` (optional). + `start-did` (optional). Changes the state of `iter`. Consuming the collection requires exclusive access to `iter`. Doesn't close `iter`." ([iter c-hash tid] (let [prefix (encode-seek-key c-hash tid)] - (i/prefix-keys! iter prefix decode-value-id-hash-prefix prefix))) - ([iter c-hash tid start-value start-id] + (i/prefix-keys! iter prefix decode-value-did-hash-prefix prefix))) + ([iter c-hash tid start-value start-did] (let [prefix (encode-seek-key c-hash tid) - start-key (encode-seek-key c-hash tid start-value start-id)] - (i/prefix-keys! iter prefix decode-value-id-hash-prefix start-key)))) + start-key (encode-seek-key c-hash tid start-value start-did)] + (i/prefix-keys! iter prefix decode-value-did-hash-prefix start-key)))) -(defn decode-id-hash-prefix - "Returns a tuple of `[id hash-prefix]`." +(defn decode-did-hash-prefix + "Returns a tuple of `[did hash-prefix]`." ([] (bb/allocate-direct key-buffer-capacity)) ([buf] - (let [id-size (impl/id-size buf) - all-size (unchecked-inc-int (unchecked-add-int id-size hash/prefix-size)) - _ (bb/set-position! buf (unchecked-subtract-int (bb/limit buf) all-size)) - id (bs/from-byte-buffer! buf id-size)] - (bb/get-byte! buf) - [id (hash/prefix-from-byte-buffer! buf)]))) + (bb/set-position! buf (- (bb/limit buf) codec/did-size hash/prefix-size)) + [(bb/get-long! buf) (hash/prefix-from-byte-buffer! buf)])) (defn prefix-keys! - "Returns a reducible collection of decoded `[id hash-prefix]` tuples from keys - starting at `start-value` and optional `start-id` and ending when + "Returns a reducible collection of decoded `[did hash-prefix]` tuples from keys + starting at `start-value` and optional `start-did` and ending when `prefix-value` is no longer a prefix of the values processed. Changes the state of `iter`. Consuming the collection requires exclusive access to `iter`. Doesn't close `iter`." ([iter c-hash tid prefix-value start-value] (i/prefix-keys! - iter (encode-seek-key c-hash tid prefix-value) decode-id-hash-prefix + iter (encode-seek-key c-hash tid prefix-value) decode-did-hash-prefix (encode-seek-key c-hash tid start-value))) - ([iter c-hash tid prefix-value start-value start-id] + ([iter c-hash tid prefix-value start-value start-did] (i/prefix-keys! - iter (encode-seek-key c-hash tid prefix-value) decode-id-hash-prefix - (encode-seek-key c-hash tid start-value start-id)))) + iter (encode-seek-key c-hash tid prefix-value) decode-did-hash-prefix + (encode-seek-key c-hash tid start-value start-did)))) (defn prefix-keys'! - "Returns a reducible collection of decoded `[id hash-prefix]` tuples from keys - starting at `start-value` and optional `start-id` and ending when + "Returns a reducible collection of decoded `[did hash-prefix]` tuples from keys + starting at `start-value` and optional `start-did` and ending when `prefix-value` is no longer a prefix of the values processed. Changes the state of `iter`. Consuming the collection requires exclusive access to `iter`. Doesn't close `iter`." [iter c-hash tid prefix-value start-value] (i/prefix-keys! - iter (encode-seek-key c-hash tid prefix-value) decode-id-hash-prefix + iter (encode-seek-key c-hash tid prefix-value) decode-did-hash-prefix (encode-seek-key-for-prev c-hash tid start-value))) (defn prefix-keys-prev! - "Returns a reducible collection of decoded `[id hash-prefix]` tuples from keys - starting at `start-value` and optional `start-id` and ending when + "Returns a reducible collection of decoded `[did hash-prefix]` tuples from keys + starting at `start-value` and optional `start-did` and ending when `prefix-value` is no longer a prefix of the values processed, iterating in reverse. @@ -177,17 +160,17 @@ access to `iter`. Doesn't close `iter`." ([iter c-hash tid prefix-value start-value] (i/prefix-keys-prev! - iter (encode-seek-key c-hash tid prefix-value) decode-id-hash-prefix + iter (encode-seek-key c-hash tid prefix-value) decode-did-hash-prefix (encode-seek-key-for-prev c-hash tid start-value))) - ([iter c-hash tid prefix-value start-value start-id] + ([iter c-hash tid prefix-value start-value start-did] (i/prefix-keys-prev! - iter (encode-seek-key c-hash tid prefix-value) decode-id-hash-prefix - (encode-seek-key-for-prev c-hash tid start-value start-id)))) + iter (encode-seek-key c-hash tid prefix-value) decode-did-hash-prefix + (encode-seek-key-for-prev c-hash tid start-value start-did)))) (defn prefix-keys-prev'! - "Returns a reducible collection of decoded `[id hash-prefix]` tuples from keys - starting at `start-value` and optional `start-id` and ending when + "Returns a reducible collection of decoded `[did hash-prefix]` tuples from keys + starting at `start-value` and optional `start-did` and ending when `prefix-value` is no longer a prefix of the values processed, iterating in reverse. @@ -195,24 +178,23 @@ access to `iter`. Doesn't close `iter`." [iter c-hash tid prefix-value start-value] (i/prefix-keys-prev! - iter (encode-seek-key c-hash tid prefix-value) decode-id-hash-prefix + iter (encode-seek-key c-hash tid prefix-value) decode-did-hash-prefix (encode-seek-key c-hash tid start-value))) -(defn encode-key [c-hash tid value id hash] - (-> (bb/allocate (unchecked-add-int (key-size value id) hash/prefix-size)) +(defn encode-key [c-hash tid value did hash] + (-> (bb/allocate (+ base-key-size (bs/size value) 1 codec/did-size hash/prefix-size)) (bb/put-int! c-hash) (bb/put-int! tid) (bb/put-byte-string! value) (bb/put-byte! 0) - (bb/put-byte-string! id) - (bb/put-byte! (bs/size id)) + (bb/put-long! did) (hash/prefix-into-byte-buffer! (hash/prefix hash)) bb/array)) (defn index-entry "Returns an entry of the SearchParamValueResource index build from `c-hash`, - `tid`, `value`, `id` and `hash`." - [c-hash tid value id hash] - [:search-param-value-index (encode-key c-hash tid value id hash) bytes/empty]) + `tid`, `value`, `did` and `hash`." + [c-hash tid value did hash] + [:search-param-value-index (encode-key c-hash tid value did hash) bytes/empty]) diff --git a/modules/db/src/blaze/db/impl/index/search_param_value_resource/impl.clj b/modules/db/src/blaze/db/impl/index/search_param_value_resource/impl.clj deleted file mode 100644 index 1da9690ad..000000000 --- a/modules/db/src/blaze/db/impl/index/search_param_value_resource/impl.clj +++ /dev/null @@ -1,15 +0,0 @@ -(ns blaze.db.impl.index.search-param-value-resource.impl - (:require - [blaze.byte-buffer :as bb] - [blaze.fhir.hash :as hash])) - - -(set! *unchecked-math* :warn-on-boxed) - - -(defn id-size - {:inline - (fn [buf] - `(int (bb/get-byte! ~buf (unchecked-dec-int (unchecked-subtract-int (bb/limit ~buf) hash/prefix-size)))))} - [buf] - (int (bb/get-byte! buf (unchecked-dec-int (unchecked-subtract-int (bb/limit buf) hash/prefix-size))))) diff --git a/modules/db/src/blaze/db/impl/index/system_as_of.clj b/modules/db/src/blaze/db/impl/index/system_as_of.clj index 275e22839..0eadd8ca8 100644 --- a/modules/db/src/blaze/db/impl/index/system_as_of.clj +++ b/modules/db/src/blaze/db/impl/index/system_as_of.clj @@ -7,9 +7,7 @@ [blaze.db.impl.codec :as codec] [blaze.db.impl.index.resource-handle :as rh] [blaze.db.impl.iterators :as i] - [blaze.fhir.hash :as hash]) - (:import - [com.google.common.primitives Longs])) + [blaze.fhir.hash :as hash])) (set! *warn-on-reflection* true) @@ -20,12 +18,12 @@ (+ codec/t-size codec/tid-size)) -(def ^:private ^:const ^long max-key-size - (+ t-tid-size codec/max-id-size)) +(def ^:private ^:const ^long key-size + (+ t-tid-size codec/did-size)) -(def ^:private ^:const ^long value-size - (+ hash/size codec/state-size)) +(def ^:private ^:const ^long max-value-size + (+ hash/size codec/state-size codec/max-id-size)) (defn- key-valid? [^long end-t] @@ -48,57 +46,53 @@ Both byte buffers are changed during decoding and have to be reset accordingly after decoding." [] - (let [ib (byte-array codec/max-id-size)] - (fn - ([] - [(bb/allocate-direct max-key-size) - (bb/allocate-direct value-size)]) - ([kb vb] - (let [t (codec/descending-long (bb/get-long! kb))] - (rh/resource-handle - (bb/get-int! kb) - (let [id-size (bb/remaining kb)] - (bb/copy-into-byte-array! kb ib 0 id-size) - (codec/id ib 0 id-size)) - t vb)))))) + (fn + ([] + [(bb/allocate-direct key-size) + (bb/allocate-direct max-value-size)]) + ([kb vb] + (let [t (codec/descending-long (bb/get-5-byte-long! kb))] + (rh/resource-handle (bb/get-int! kb) (bb/get-long! kb) t vb))))) (defn encode-key - "Encodes the key of the SystemAsOf index from `t`, `tid` and `id`." - [t tid id] - (-> (bb/allocate (unchecked-add-int t-tid-size (bs/size id))) - (bb/put-long! (codec/descending-long ^long t)) + "Encodes the key of the SystemAsOf index from `t`, `tid` and `did`." + [t tid did] + (-> (bb/allocate key-size) + (bb/put-5-byte-long! (codec/descending-long (unchecked-long t))) (bb/put-int! tid) - (bb/put-byte-string! id) + (bb/put-long! did) bb/array)) (defn- encode-t-tid [start-t start-tid] (-> (bb/allocate t-tid-size) - (bb/put-long! (codec/descending-long ^long start-t)) + (bb/put-5-byte-long! (codec/descending-long (unchecked-long start-t))) (bb/put-int! start-tid) bb/array)) -(defn- start-key [start-t start-tid start-id] +(defn- start-key [start-t start-tid start-did] (cond - start-id - (encode-key start-t start-tid start-id) + start-did + (encode-key start-t start-tid start-did) start-tid (encode-t-tid start-t start-tid) :else - (Longs/toByteArray (codec/descending-long ^long start-t)))) + (-> (bb/allocate codec/t-size) + (bb/put-5-byte-long! (codec/descending-long (unchecked-long start-t))) + bb/array))) (defn system-history "Returns a reducible collection of all versions between `start-t` (inclusive), - `start-tid` (optional, inclusive), `start-id` (optional, inclusive) and + `start-tid` (optional, inclusive), `start-did` (optional, inclusive) and `end-t` (inclusive) of all resources. Versions are resource handles." - [saoi start-t start-tid start-id end-t] + [saoi start-t start-tid start-did end-t] (coll/eduction (take-while (key-valid? end-t)) - (i/kvs! saoi (decoder) (bs/from-byte-array (start-key start-t start-tid start-id))))) + (i/kvs! saoi (decoder) (bs/from-byte-array (start-key start-t start-tid start-did))))) diff --git a/modules/db/src/blaze/db/impl/index/system_stats.clj b/modules/db/src/blaze/db/impl/index/system_stats.clj index 2360466fd..007729f74 100644 --- a/modules/db/src/blaze/db/impl/index/system_stats.clj +++ b/modules/db/src/blaze/db/impl/index/system_stats.clj @@ -50,7 +50,7 @@ is used to get near `t`." [iter t] (let [buf (bb/allocate-direct kv-capacity)] - (bb/put-long! buf (codec/descending-long ^long t)) + (bb/put-5-byte-long! buf (codec/descending-long (unchecked-long t))) (bb/flip! buf) (kv/seek-buffer! iter buf) (when (kv/valid? iter) @@ -59,13 +59,13 @@ (decode-value! buf)))) -(defn- encode-key [t] +(defn encode-key [t] (-> (bb/allocate key-size) - (bb/put-long! (codec/descending-long ^long t)) + (bb/put-5-byte-long! (codec/descending-long (unchecked-long t))) bb/array)) -(defn- encode-value [{:keys [total num-changes]}] +(defn encode-value [{:keys [total num-changes]}] (-> (bb/allocate value-size) (bb/put-long! total) (bb/put-long! num-changes) diff --git a/modules/db/src/blaze/db/impl/index/type_as_of.clj b/modules/db/src/blaze/db/impl/index/type_as_of.clj index ef4f5d9d8..9e15b680a 100644 --- a/modules/db/src/blaze/db/impl/index/type_as_of.clj +++ b/modules/db/src/blaze/db/impl/index/type_as_of.clj @@ -18,12 +18,12 @@ (+ codec/tid-size codec/t-size)) -(def ^:private ^:const ^long max-key-size - (+ tid-t-size codec/max-id-size)) +(def ^:private ^:const ^long key-size + (+ tid-t-size codec/did-size)) -(def ^:private ^:const ^long value-size - (+ hash/size codec/state-size)) +(def ^:private ^:const ^long max-value-size + (+ hash/size codec/state-size codec/max-id-size)) (defn- key-valid? [^long tid ^long end-t] @@ -46,49 +46,47 @@ Both byte buffers are changed during decoding and have to be reset accordingly after decoding." [] - (let [ib (byte-array codec/max-id-size)] - (fn - ([] - [(bb/allocate-direct max-key-size) - (bb/allocate-direct value-size)]) - ([kb vb] - (let [tid (bb/get-int! kb) - t (codec/descending-long (bb/get-long! kb))] - (rh/resource-handle - tid - (let [id-size (bb/remaining kb)] - (bb/copy-into-byte-array! kb ib 0 id-size) - (codec/id ib 0 id-size)) - t vb)))))) + (fn + ([] + [(bb/allocate-direct key-size) + (bb/allocate-direct max-value-size)]) + ([kb vb] + (let [tid (bb/get-int! kb) + t (codec/descending-long (bb/get-5-byte-long! kb))] + (rh/resource-handle + tid + (bb/get-long! kb) + t + vb))))) (defn encode-key - "Encodes the key of the TypeAsOf index from `tid`, `t` and `id`." - [tid t id] - (-> (bb/allocate (unchecked-add-int tid-t-size (bs/size id))) + "Encodes the key of the TypeAsOf index from `tid`, `t` and `did`." + [tid t did] + (-> (bb/allocate key-size) (bb/put-int! tid) - (bb/put-long! (codec/descending-long ^long t)) - (bb/put-byte-string! id) + (bb/put-5-byte-long! (codec/descending-long (unchecked-long t))) + (bb/put-long! (unchecked-long did)) bb/array)) -(defn- start-key [tid start-t start-id] - (if start-id - (bs/from-byte-array (encode-key tid start-t start-id)) +(defn- start-key [tid start-t start-did] + (if start-did + (bs/from-byte-array (encode-key tid start-t start-did)) (-> (bb/allocate tid-t-size) (bb/put-int! tid) - (bb/put-long! (codec/descending-long ^long start-t)) + (bb/put-5-byte-long! (codec/descending-long (unchecked-long start-t))) bb/flip! bs/from-byte-buffer!))) (defn type-history "Returns a reducible collection of all versions between `start-t` (inclusive), - `start-id` (optional, inclusive) and `end-t` (inclusive) of resources with + `start-did` (optional, inclusive) and `end-t` (inclusive) of resources with `tid`. Versions are resource handles." - [taoi tid start-t start-id end-t] + [taoi tid start-t start-did end-t] (coll/eduction (take-while (key-valid? tid end-t)) - (i/kvs! taoi (decoder) (start-key tid start-t start-id)))) + (i/kvs! taoi (decoder) (start-key tid start-t start-did)))) diff --git a/modules/db/src/blaze/db/impl/index/type_stats.clj b/modules/db/src/blaze/db/impl/index/type_stats.clj index 4d585b419..8032765d5 100644 --- a/modules/db/src/blaze/db/impl/index/type_stats.clj +++ b/modules/db/src/blaze/db/impl/index/type_stats.clj @@ -53,26 +53,26 @@ [iter tid t] (let [buf (bb/allocate-direct kv-capacity)] (bb/put-int! buf tid) - (bb/put-long! buf (codec/descending-long ^long t)) + (bb/put-5-byte-long! buf (codec/descending-long t)) (bb/flip! buf) (kv/seek-buffer! iter buf) (when (kv/valid? iter) (bb/clear! buf) (kv/key! iter buf) - (when (= ^long tid (bb/get-int! buf)) + (when (= (long tid) (bb/get-int! buf)) (bb/clear! buf) (kv/value! iter buf) (decode-value! buf))))) -(defn- encode-key [tid t] +(defn encode-key [tid t] (-> (bb/allocate key-size) (bb/put-int! tid) - (bb/put-long! (codec/descending-long ^long t)) + (bb/put-5-byte-long! (codec/descending-long t)) bb/array)) -(defn- encode-value [{:keys [total num-changes]}] +(defn encode-value [{:keys [total num-changes]}] (-> (bb/allocate value-size) (bb/put-long! total) (bb/put-long! num-changes) diff --git a/modules/db/src/blaze/db/impl/search_param.clj b/modules/db/src/blaze/db/impl/search_param.clj index f49bb4d5e..8a630dfee 100644 --- a/modules/db/src/blaze/db/impl/search_param.clj +++ b/modules/db/src/blaze/db/impl/search_param.clj @@ -49,18 +49,17 @@ (mapcat (partial p/-resource-handles search-param context tid modifier)) (distinct)) values))) - ([search-param context tid modifier values start-id] + ([search-param context tid modifier values start-did] (if (= 1 (count values)) (p/-resource-handles search-param context tid modifier (first values) - start-id) - (let [start-id (codec/id-string start-id)] - (coll/eduction - (drop-while #(not= start-id (rh/id %))) - (resource-handles search-param context tid modifier values)))))) + start-did) + (coll/eduction + (drop-while #(not= start-did (rh/did %))) + (resource-handles search-param context tid modifier values))))) (defn- compartment-keys - "Returns a reducible collection of `[prefix id hash-prefix]` triples." + "Returns a reducible collection of `[prefix did hash-prefix]` triples." [search-param context compartment tid compiled-values] (coll/eduction (mapcat #(p/-compartment-keys search-param context compartment tid %)) @@ -105,19 +104,11 @@ (defn index-entries "Returns reducible collection of index entries of `resource` with `hash` for `search-param` or an anomaly in case of errors." - {:arglists '([search-param linked-compartments hash resource])} - [{:keys [code c-hash] :as search-param} linked-compartments hash resource] - (when-ok [values (p/-index-values search-param stub-resolver resource)] - (let [{:keys [id]} resource - type (name (fhir-spec/fhir-type resource)) - tid (codec/tid type) - id (codec/id-byte-string id) - linked-compartments - (mapv - (fn [[code comp-id]] - [(codec/c-hash code) - (codec/id-byte-string comp-id)]) - linked-compartments)] + {:arglists '([search-param resource-id linked-compartments did hash resource])} + [{:keys [code c-hash] :as search-param} resource-id linked-compartments did hash resource] + (when-ok [values (p/-index-values search-param resource-id stub-resolver resource)] + (let [type (name (fhir-spec/fhir-type resource)) + tid (codec/tid type)] (coll/eduction (mapcat (fn index-entry [[modifier value]] @@ -130,10 +121,10 @@ c-hash tid value - id + did hash))) conj - [(sp-vr/index-entry c-hash tid value id hash) - (r-sp-v/index-entry tid id hash c-hash value)] + [(sp-vr/index-entry c-hash tid value did hash) + (r-sp-v/index-entry tid did hash c-hash value)] linked-compartments)))) values)))) diff --git a/modules/db/src/blaze/db/impl/search_param/chained.clj b/modules/db/src/blaze/db/impl/search_param/chained.clj index 64cc89150..536ea076e 100644 --- a/modules/db/src/blaze/db/impl/search_param/chained.clj +++ b/modules/db/src/blaze/db/impl/search_param/chained.clj @@ -41,11 +41,10 @@ (p/-resource-handles search-param (assoc context :svri svri) ref-tid modifier compiled-value)))) - (-resource-handles [this context tid modifier compiled-value start-id] - (let [start-id (codec/id-string start-id)] - (coll/eduction - (drop-while #(not= start-id (rh/id %))) - (p/-resource-handles this context tid modifier compiled-value)))) + (-resource-handles [this context tid modifier compiled-value start-did] + (coll/eduction + (drop-while #(not= start-did (rh/did %))) + (p/-resource-handles this context tid modifier compiled-value))) (-matches? [_ context resource-handle modifier compiled-values] (some diff --git a/modules/db/src/blaze/db/impl/search_param/composite/common.clj b/modules/db/src/blaze/db/impl/search_param/composite/common.clj index 1f2f4cefc..b45ae7bf4 100644 --- a/modules/db/src/blaze/db/impl/search_param/composite/common.clj +++ b/modules/db/src/blaze/db/impl/search_param/composite/common.clj @@ -22,25 +22,25 @@ (defn- component-index-values - [resolver main-value {:keys [expression search-param]}] + [resource-id resolver main-value {:keys [expression search-param]}] (when-ok [values (fhir-path/eval resolver expression main-value)] (coll/eduction (comp - (p/-index-value-compiler search-param) + (p/-index-value-compiler search-param resource-id) (filter (fn [[modifier]] (nil? modifier))) (map (fn [[_ value]] value))) values))) -(defn index-values [resolver c1 c2] +(defn index-values [resource-id resolver c1 c2] (comp (map (fn [main-value] - (when-ok [c1-values (component-index-values resolver main-value c1)] + (when-ok [c1-values (component-index-values resource-id resolver main-value c1)] (.reduce ^IReduceInit c1-values (fn [res v1] - (if-ok [c2-values (component-index-values resolver main-value c2)] + (if-ok [c2-values (component-index-values resource-id resolver main-value c2)] (.reduce ^IReduceInit c2-values (fn [res v2] diff --git a/modules/db/src/blaze/db/impl/search_param/composite/token_quantity.clj b/modules/db/src/blaze/db/impl/search_param/composite/token_quantity.clj index 8a1ba69a1..7e781ba77 100644 --- a/modules/db/src/blaze/db/impl/search_param/composite/token_quantity.clj +++ b/modules/db/src/blaze/db/impl/search_param/composite/token_quantity.clj @@ -35,7 +35,7 @@ (defrecord SearchParamCompositeTokenQuantity [name url type base code c-hash main-expression c1 c2] p/SearchParam - (-compile-value [_ _ value] + (-compile-value [_ _modifier value] (let [[v1 v2] (cc/split-value value) token-value (cc/compile-component-value c1 v1)] (if-ok [quantity-value (cc/compile-component-value c2 v2)] @@ -56,15 +56,15 @@ (u/resource-handle-mapper context tid) (spq/resource-keys! context c-hash tid prefix-length value))) - (-resource-handles [_ context tid _ value start-id] + (-resource-handles [_ context tid _ value start-did] (coll/eduction (u/resource-handle-mapper context tid) - (spq/resource-keys! context c-hash tid prefix-length value start-id))) + (spq/resource-keys! context c-hash tid prefix-length value start-did))) (-matches? [_ context resource-handle _ values] (some? (some #(spq/matches? context c-hash resource-handle prefix-length %) values))) - (-index-values [_ resolver resource] + (-index-values [_ resource-id resolver resource] (when-ok [values (fhir-path/eval resolver main-expression resource)] - (coll/eduction (cc/index-values resolver c1 c2) values)))) + (coll/eduction (cc/index-values resource-id resolver c1 c2) values)))) diff --git a/modules/db/src/blaze/db/impl/search_param/composite/token_token.clj b/modules/db/src/blaze/db/impl/search_param/composite/token_token.clj index 8c3900c2e..a38161f0f 100644 --- a/modules/db/src/blaze/db/impl/search_param/composite/token_token.clj +++ b/modules/db/src/blaze/db/impl/search_param/composite/token_token.clj @@ -13,7 +13,7 @@ (defrecord SearchParamCompositeTokenToken [name url type base code c-hash main-expression c1 c2] p/SearchParam - (-compile-value [_ _ value] + (-compile-value [_ _modifier value] (let [[v1 v2] (cc/split-value value) v1 (cc/compile-component-value c1 v1) v2 (cc/compile-component-value c1 v2)] @@ -24,14 +24,14 @@ (u/resource-handle-mapper context tid) (spt/resource-keys! context c-hash tid value))) - (-resource-handles [_ context tid _ value start-id] + (-resource-handles [_ context tid _ value start-did] (coll/eduction (u/resource-handle-mapper context tid) - (spt/resource-keys! context c-hash tid value start-id))) + (spt/resource-keys! context c-hash tid value start-did))) (-matches? [_ context resource-handle _ values] (some? (some #(spt/matches? context c-hash resource-handle %) values))) - (-index-values [_ resolver resource] + (-index-values [_ resource-id resolver resource] (when-ok [values (fhir-path/eval resolver main-expression resource)] - (coll/eduction (cc/index-values resolver c1 c2) values)))) + (coll/eduction (cc/index-values resource-id resolver c1 c2) values)))) diff --git a/modules/db/src/blaze/db/impl/search_param/date.clj b/modules/db/src/blaze/db/impl/search_param/date.clj index 4f0ef4f17..9a5dd051d 100644 --- a/modules/db/src/blaze/db/impl/search_param/date.clj +++ b/modules/db/src/blaze/db/impl/search_param/date.clj @@ -101,23 +101,23 @@ (eq-overlaps? v-lb v-ub q-lb a-lb))))) -(defn- all-keys! [{:keys [svri] :as context} c-hash tid start-id] - (sp-vr/all-keys! svri c-hash tid (resource-value! context c-hash tid start-id) - start-id)) +(defn- all-keys! [{:keys [svri] :as context} c-hash tid start-did] + (sp-vr/all-keys! svri c-hash tid (resource-value! context c-hash tid start-did) + start-did)) (defn- eq-keys! - "Returns a reducible collection of `[value id hash-prefix]` triples of all + "Returns a reducible collection of `[value did hash-prefix]` triples of all keys with overlapping date/time intervals with the interval specified by - `lower-bound` and `upper-bound` starting at `start-id` (optional)." + `lower-bound` and `upper-bound` starting at `start-did` (optional)." ([{:keys [svri]} c-hash tid lower-bound upper-bound] (coll/eduction (eq-filter lower-bound upper-bound) (sp-vr/all-keys! svri c-hash tid))) - ([context c-hash tid lower-bound upper-bound start-id] + ([context c-hash tid lower-bound upper-bound start-did] (coll/eduction (eq-filter lower-bound upper-bound) - (all-keys! context c-hash tid start-id)))) + (all-keys! context c-hash tid start-did)))) (defn- ge-overlaps? [v-lb v-ub q-lb] @@ -133,17 +133,17 @@ (defn- ge-keys! - "Returns a reducible collection of `[value id hash-prefix]` triples of all + "Returns a reducible collection of `[value did hash-prefix]` triples of all keys with overlapping date/time intervals with the interval specified by - `lower-bound` and an infinite upper bound starting at `start-id` (optional)." + `lower-bound` and an infinite upper bound starting at `start-did` (optional)." ([{:keys [svri]} c-hash tid lower-bound] (coll/eduction (ge-filter lower-bound) (sp-vr/all-keys! svri c-hash tid))) - ([context c-hash tid lower-bound start-id] + ([context c-hash tid lower-bound start-did] (coll/eduction (ge-filter lower-bound) - (all-keys! context c-hash tid start-id)))) + (all-keys! context c-hash tid start-did)))) (defn- le-overlaps? [v-lb v-ub q-ub] @@ -159,17 +159,17 @@ (defn- le-keys! - "Returns a reducible collection of `[value id hash-prefix]` triples of all + "Returns a reducible collection of `[value did hash-prefix]` triples of all keys with overlapping date/time intervals with the interval specified by - an infinite lower bound and `upper-bound` starting at `start-id` (optional)." + an infinite lower bound and `upper-bound` starting at `start-did` (optional)." ([{:keys [svri]} c-hash tid upper-bound] (coll/eduction (le-filter upper-bound) (sp-vr/all-keys! svri c-hash tid))) - ([context c-hash tid upper-bound start-id] + ([context c-hash tid upper-bound start-did] (coll/eduction (le-filter upper-bound) - (all-keys! context c-hash tid start-id)))) + (all-keys! context c-hash tid start-did)))) (defn- invalid-date-time-value-msg [code value] @@ -182,11 +182,11 @@ :eq (eq-keys! context c-hash tid lower-bound upper-bound) (:ge :gt) (ge-keys! context c-hash tid lower-bound) (:le :lt) (le-keys! context c-hash tid upper-bound))) - ([context c-hash tid {:keys [op lower-bound upper-bound]} start-id] + ([context c-hash tid {:keys [op lower-bound upper-bound]} start-did] (case op - :eq (eq-keys! context c-hash tid lower-bound upper-bound start-id) - (:ge :gt) (ge-keys! context c-hash tid lower-bound start-id) - (:le :lt) (le-keys! context c-hash tid upper-bound start-id)))) + :eq (eq-keys! context c-hash tid lower-bound upper-bound start-did) + (:ge :gt) (ge-keys! context c-hash tid lower-bound start-did) + (:le :lt) (le-keys! context c-hash tid upper-bound start-did)))) (defn- matches? @@ -203,7 +203,7 @@ (defrecord SearchParamDate [name url type base code c-hash expression] p/SearchParam - (-compile-value [_ _ value] + (-compile-value [_ _modifier value] (let [[op value] (u/separate-op value)] (if-ok [date-time-value (system/parse-date-time value)] (case op @@ -223,25 +223,25 @@ (-resource-handles [_ context tid _ value] (coll/eduction (comp - (map (fn [[_value id hash-prefix]] [id hash-prefix])) + (map (fn [[_value did hash-prefix]] [did hash-prefix])) (u/resource-handle-mapper context tid)) (resource-keys! context c-hash tid value))) - (-resource-handles [_ context tid _ value start-id] + (-resource-handles [_ context tid _ value start-did] (coll/eduction (comp - (map (fn [[_value id hash-prefix]] [id hash-prefix])) + (map (fn [[_value did hash-prefix]] [did hash-prefix])) (u/resource-handle-mapper context tid)) - (resource-keys! context c-hash tid value start-id))) + (resource-keys! context c-hash tid value start-did))) (-matches? [_ context resource-handle _ values] (some? (some #(matches? context c-hash resource-handle %) values))) - (-index-values [search-param resolver resource] + (-index-values [search-param resource-id resolver resource] (when-ok [values (fhir-path/eval resolver expression resource)] - (coll/eduction (p/-index-value-compiler search-param) values))) + (coll/eduction (p/-index-value-compiler search-param resource-id) values))) - (-index-value-compiler [_] + (-index-value-compiler [_ _resource-id] (mapcat (partial index-entries url)))) diff --git a/modules/db/src/blaze/db/impl/search_param/has.clj b/modules/db/src/blaze/db/impl/search_param/has.clj index da3ebf52f..e525ca121 100644 --- a/modules/db/src/blaze/db/impl/search_param/has.clj +++ b/modules/db/src/blaze/db/impl/search_param/has.clj @@ -41,26 +41,24 @@ [] (u/reference-resource-handle-mapper context) (let [tid-byte-string (codec/tid-byte-string tid) - {:keys [tid id hash]} resource-handle] - (r-sp-v/prefix-keys! rsvi tid (codec/id-byte-string id) hash c-hash - tid-byte-string)))) + {:keys [tid did hash]} resource-handle] + (r-sp-v/prefix-keys! rsvi tid did hash c-hash tid-byte-string)))) -(def ^:private id-cmp +(def ^:private did-cmp (reify Comparator (compare [_ a b] - (let [^String id-a (rh/id a)] - (.compareTo id-a (rh/id b)))))) + (Long/compare (rh/did a) (rh/did b))))) -(defn- drop-lesser-ids [start-id] - (drop-while #(neg? (let [^String id (rh/id %)] (.compareTo id start-id))))) +(defn- drop-lesser-ids [^long start-did] + (drop-while #(< (rh/did %) start-did))) (defn- resource-handles* [context tid [search-param chain-search-param join-tid value]] (into - (sorted-set-by id-cmp) + (sorted-set-by did-cmp) (mapcat #(resolve-resource-handles context chain-search-param tid %)) (p/-resource-handles search-param context join-tid nil value))) @@ -102,15 +100,13 @@ (-resource-handles [_ context tid _ value] (resource-handles context tid value)) - (-resource-handles [_ context tid _ value start-id] - (coll/eduction - (drop-lesser-ids (codec/id-string start-id)) - (resource-handles context tid value))) + (-resource-handles [_ context tid _ value start-did] + (coll/eduction (drop-lesser-ids start-did) (resource-handles context tid value))) (-matches? [_ context resource-handle _ values] (some? (some #(matches? context resource-handle %) values))) - (-index-values [_ _ _] + (-index-values [_ _ _ _] [])) diff --git a/modules/db/src/blaze/db/impl/search_param/list.clj b/modules/db/src/blaze/db/impl/search_param/list.clj index 4be150c64..159476823 100644 --- a/modules/db/src/blaze/db/impl/search_param/list.clj +++ b/modules/db/src/blaze/db/impl/search_param/list.clj @@ -13,51 +13,52 @@ (set! *warn-on-reflection* true) -(def ^:private list-tid (codec/tid "List")) -(def ^:private item-c-hash (codec/c-hash "item")) +(def ^:private ^:const list-tid (codec/tid "List")) +(def ^:private ^:const item-c-hash (codec/c-hash "item")) + + +(defn- list-resource-handle [{:keys [resource-id resource-handle]} list-id] + (when-let [list-did (resource-id list-tid list-id)] + (u/non-deleted-resource-handle resource-handle list-tid list-did))) (defn- referenced-resource-handles! "Returns a reducible collection of resource handles of type `tid` that are - referenced by the list with `list-id` and `list-hash`, starting with - `start-id` (optional). + referenced by the list with `list-did` and `list-hash`, starting with + `start-did` (optional). Changes the state of `rsvi`. Consuming the collection requires exclusive access to `rsvi`. Doesn't close `rsvi`." {:arglists - '([context list-id list-hash tid] - [context list-id list-hash tid start-id])} - ([{:keys [rsvi] :as context} list-id list-hash tid] + '([context list-did list-hash tid] + [context list-did list-hash tid start-did])} + ([{:keys [rsvi] :as context} list-did list-hash tid] (coll/eduction (u/reference-resource-handle-mapper context) - (r-sp-v/prefix-keys! rsvi list-tid list-id list-hash item-c-hash + (r-sp-v/prefix-keys! rsvi list-tid list-did list-hash item-c-hash (codec/tid-byte-string tid)))) - ([{:keys [rsvi] :as context} list-id list-hash tid start-id] + ([{:keys [rsvi] :as context} list-did list-hash tid start-did] (coll/eduction (u/reference-resource-handle-mapper context) - (r-sp-v/prefix-keys! rsvi list-tid list-id list-hash item-c-hash + (r-sp-v/prefix-keys! rsvi list-tid list-did list-hash item-c-hash (codec/tid-byte-string tid) - (codec/tid-id tid start-id))))) + (codec/tid-did tid start-did))))) (defrecord SearchParamList [name type code] p/SearchParam - (-compile-value [_ _ value] - (codec/id-byte-string value)) + (-compile-value [_ _modifier list-id] + list-id) (-resource-handles [_ context tid _ list-id] - (let [{:keys [resource-handle]} context] - (when-let [{:keys [hash]} (u/non-deleted-resource-handle - resource-handle list-tid list-id)] - (referenced-resource-handles! context list-id hash tid)))) - - (-resource-handles [_ context tid _ list-id start-id] - (let [{:keys [resource-handle]} context] - (when-let [{:keys [hash]} (u/non-deleted-resource-handle - resource-handle list-tid list-id)] - (referenced-resource-handles! context list-id hash tid start-id)))) - - (-index-values [_ _ _] + (when-let [{:keys [did hash]} (list-resource-handle context list-id)] + (referenced-resource-handles! context did hash tid))) + + (-resource-handles [_ context tid _ list-id start-did] + (when-let [{:keys [did hash]} (list-resource-handle context list-id)] + (referenced-resource-handles! context did hash tid start-did))) + + (-index-values [_ _ _ _] [])) diff --git a/modules/db/src/blaze/db/impl/search_param/number.clj b/modules/db/src/blaze/db/impl/search_param/number.clj index fc9072dd9..23e0905b9 100644 --- a/modules/db/src/blaze/db/impl/search_param/number.clj +++ b/modules/db/src/blaze/db/impl/search_param/number.clj @@ -42,7 +42,7 @@ (defrecord SearchParamQuantity [name url type base code c-hash expression] p/SearchParam - (-compile-value [_ _ value] + (-compile-value [_ _modifier value] (let [[op value] (u/separate-op value)] (if-ok [decimal-value (system/parse-decimal value)] (case op @@ -63,19 +63,19 @@ (u/resource-handle-mapper context tid) (spq/resource-keys! context c-hash tid 0 value))) - (-resource-handles [_ context tid _ value start-id] + (-resource-handles [_ context tid _ value start-did] (coll/eduction (u/resource-handle-mapper context tid) - (spq/resource-keys! context c-hash tid 0 value start-id))) + (spq/resource-keys! context c-hash tid 0 value start-did))) (-matches? [_ context resource-handle _ values] (some? (some #(spq/matches? context c-hash resource-handle 0 %) values))) - (-index-values [search-param resolver resource] + (-index-values [search-param resource-id resolver resource] (when-ok [values (fhir-path/eval resolver expression resource)] - (coll/eduction (p/-index-value-compiler search-param) values))) + (coll/eduction (p/-index-value-compiler search-param resource-id) values))) - (-index-value-compiler [_] + (-index-value-compiler [_ _resource-id] (mapcat (partial index-entries url)))) diff --git a/modules/db/src/blaze/db/impl/search_param/quantity.clj b/modules/db/src/blaze/db/impl/search_param/quantity.clj index ba4fa9198..5087f6749 100644 --- a/modules/db/src/blaze/db/impl/search_param/quantity.clj +++ b/modules/db/src/blaze/db/impl/search_param/quantity.clj @@ -61,7 +61,7 @@ (defn- resource-value! - "Returns the value of the resource with `tid` and `id` according to the + "Returns the value of the resource with `tid` and `did` according to the search parameter with `c-hash` starting with `prefix`. The `prefix` is important, because resources have more than one index entry @@ -71,16 +71,16 @@ Changes the state of `context`. Calling this function requires exclusive access to `context`." - {:arglists '([context c-hash tid id prefix])} - [{:keys [rsvi resource-handle]} c-hash tid id prefix] - (let [handle (resource-handle tid id)] + {:arglists '([context c-hash tid did prefix])} + [{:keys [rsvi resource-handle]} c-hash tid did prefix] + (when-let [handle (resource-handle tid did)] (r-sp-v/next-value! rsvi handle c-hash prefix prefix))) -(defn- id-start-key! [context c-hash tid prefix start-id] - (let [start-value (resource-value! context c-hash tid start-id prefix)] +(defn- did-start-key! [context c-hash tid prefix start-did] + (let [start-value (resource-value! context c-hash tid start-did prefix)] (assert start-value) - (sp-vr/encode-seek-key c-hash tid start-value start-id))) + (sp-vr/encode-seek-key c-hash tid start-value start-did))) (defn- take-while-less-equal [c-hash tid value] @@ -89,8 +89,8 @@ (defn- eq-keys! - "Returns a reducible collection of `[id hash-prefix]` tuples of values between - `lower-bound` and `upper-bound` starting at `start-id` (optional). + "Returns a reducible collection of `[did hash-prefix]` tuples of values between + `lower-bound` and `upper-bound` starting at `start-did` (optional). The `prefix` is a fix prefix of `value` which all found values have to have. @@ -100,21 +100,21 @@ (coll/eduction (comp (take-while-less-equal c-hash tid upper-bound) - (map (fn [[_prefix id hash-prefix]] [id hash-prefix]))) + (map (fn [[_prefix did hash-prefix]] [did hash-prefix]))) (sp-vr/keys! svri (sp-vr/encode-seek-key c-hash tid lower-bound)))) ([{:keys [svri] :as context} c-hash tid lower-bound-prefix upper-bound - start-id] + start-did] (coll/eduction (comp (take-while-less-equal c-hash tid upper-bound) - (map (fn [[_prefix id hash-prefix]] [id hash-prefix]))) - (sp-vr/keys! svri (id-start-key! context c-hash tid lower-bound-prefix - start-id))))) + (map (fn [[_prefix did hash-prefix]] [did hash-prefix]))) + (sp-vr/keys! svri (did-start-key! context c-hash tid lower-bound-prefix + start-did))))) (defn- gt-keys! - "Returns a reducible collection of `[id hash-prefix]` tuples of values greater - than `value` starting at `start-id` (optional). + "Returns a reducible collection of `[did hash-prefix]` tuples of values + greater than `value` starting at `start-did` (optional). The `prefix` is a fix prefix of `value` which all found values have to have. @@ -122,15 +122,15 @@ access to `iter`. Doesn't close `iter`." ([{:keys [svri]} c-hash tid prefix value] (sp-vr/prefix-keys'! svri c-hash tid prefix value)) - ([{:keys [svri] :as context} c-hash tid prefix _value start-id] - (let [start-value (resource-value! context c-hash tid start-id prefix)] + ([{:keys [svri] :as context} c-hash tid prefix _value start-did] + (let [start-value (resource-value! context c-hash tid start-did prefix)] (assert start-value) - (sp-vr/prefix-keys! svri c-hash tid prefix start-value start-id)))) + (sp-vr/prefix-keys! svri c-hash tid prefix start-value start-did)))) (defn- lt-keys! - "Returns a reducible collection of `[id hash-prefix]` tuples of values less - than `value` starting at `start-id` (optional). + "Returns a reducible collection of `[did hash-prefix]` tuples of values less + than `value` starting at `start-did` (optional). The `prefix` is a fix prefix of `value` which all found values have to have. @@ -138,15 +138,15 @@ access to `iter`. Doesn't close `iter`." ([{:keys [svri]} c-hash tid prefix value] (sp-vr/prefix-keys-prev'! svri c-hash tid prefix value)) - ([{:keys [svri] :as context} c-hash tid prefix _value start-id] - (let [start-value (resource-value! context c-hash tid start-id prefix)] + ([{:keys [svri] :as context} c-hash tid prefix _value start-did] + (let [start-value (resource-value! context c-hash tid start-did prefix)] (assert start-value) - (sp-vr/prefix-keys-prev! svri c-hash tid prefix start-value start-id)))) + (sp-vr/prefix-keys-prev! svri c-hash tid prefix start-value start-did)))) (defn- ge-keys! - "Returns a reducible collection of `[id hash-prefix]` tuples of values greater - or equal `value` starting at `start-id` (optional). + "Returns a reducible collection of `[did hash-prefix]` tuples of values greater + or equal `value` starting at `start-did` (optional). The `prefix` is a fix prefix of `value` which all found values have to have. @@ -154,15 +154,15 @@ access to `iter`. Doesn't close `iter`." ([{:keys [svri]} c-hash tid prefix value] (sp-vr/prefix-keys! svri c-hash tid prefix value)) - ([{:keys [svri] :as context} c-hash tid prefix _value start-id] - (let [start-value (resource-value! context c-hash tid start-id prefix)] + ([{:keys [svri] :as context} c-hash tid prefix _value start-did] + (let [start-value (resource-value! context c-hash tid start-did prefix)] (assert start-value) - (sp-vr/prefix-keys! svri c-hash tid prefix start-value start-id)))) + (sp-vr/prefix-keys! svri c-hash tid prefix start-value start-did)))) (defn- le-keys! - "Returns a reducible collection of `[id hash-prefix]` tuples of values less - or equal `value` starting at `start-id` (optional). + "Returns a reducible collection of `[did hash-prefix]` tuples of values less + or equal `value` starting at `start-did` (optional). The `prefix` is a fix prefix of `value` which all found values have to have. @@ -170,15 +170,15 @@ access to `iter`. Doesn't close `iter`." ([{:keys [svri]} c-hash tid prefix value] (sp-vr/prefix-keys-prev! svri c-hash tid prefix value)) - ([{:keys [svri] :as context} c-hash tid prefix _value start-id] - (let [start-value (resource-value! context c-hash tid start-id prefix)] + ([{:keys [svri] :as context} c-hash tid prefix _value start-did] + (let [start-value (resource-value! context c-hash tid start-did prefix)] (assert start-value) - (sp-vr/prefix-keys-prev! svri c-hash tid prefix start-value start-id)))) + (sp-vr/prefix-keys-prev! svri c-hash tid prefix start-value start-did)))) (defn resource-keys! - "Returns a reducible collection of `[id hash-prefix]` tuples of values - according to `op` and values starting at `start-id` (optional). + "Returns a reducible collection of `[did hash-prefix]` tuples of values + according to `op` and values starting at `start-did` (optional). The `prefix-length` is the length of the fix prefix that all found values have to have. @@ -187,7 +187,7 @@ access to `context`." {:arglists '([context c-hash tid prefix-length value] - [context c-hash tid prefix-length value start-id])} + [context c-hash tid prefix-length value start-did])} ([context c-hash tid prefix-length {:keys [op lower-bound exact-value upper-bound]}] (case op @@ -202,18 +202,18 @@ exact-value))) ([context c-hash tid prefix-length {:keys [op lower-bound exact-value upper-bound]} - start-id] + start-did] (case op :eq (eq-keys! context c-hash tid (bs/subs lower-bound 0 prefix-length) - upper-bound start-id) + upper-bound start-did) :gt (gt-keys! context c-hash tid (bs/subs exact-value 0 prefix-length) - exact-value start-id) + exact-value start-did) :lt (lt-keys! context c-hash tid (bs/subs exact-value 0 prefix-length) - exact-value start-id) + exact-value start-did) :ge (ge-keys! context c-hash tid (bs/subs exact-value 0 prefix-length) - exact-value start-id) + exact-value start-did) :le (le-keys! context c-hash tid (bs/subs exact-value 0 prefix-length) - exact-value start-id)))) + exact-value start-did)))) (defn- take-while-compartment-less-equal [compartment c-hash tid value] @@ -226,7 +226,7 @@ (coll/eduction (comp (take-while-compartment-less-equal compartment c-hash tid upper-bound) - (map (fn [[_prefix id hash-prefix]] [id hash-prefix]))) + (map (fn [[_prefix did hash-prefix]] [did hash-prefix]))) (c-sp-vr/keys! csvri (c-sp-vr/encode-seek-key compartment c-hash tid lower-bound)))) @@ -281,7 +281,7 @@ (defrecord SearchParamQuantity [name url type base code c-hash expression] p/SearchParam - (-compile-value [_ _ value] + (-compile-value [_ _modifier value] (let [[op value-and-unit] (u/separate-op value) [value unit] (str/split value-and-unit #"\s*\|\s*" 2)] (if-ok [decimal-value (system/parse-decimal value)] @@ -303,10 +303,10 @@ (u/resource-handle-mapper context tid) (resource-keys! context c-hash tid codec/v-hash-size value))) - (-resource-handles [_ context tid _ value start-id] + (-resource-handles [_ context tid _ value start-did] (coll/eduction (u/resource-handle-mapper context tid) - (resource-keys! context c-hash tid codec/v-hash-size value start-id))) + (resource-keys! context c-hash tid codec/v-hash-size value start-did))) (-compartment-keys [_ context compartment tid value] (compartment-keys context compartment c-hash tid value)) @@ -315,11 +315,11 @@ (some? (some #(matches? context c-hash resource-handle codec/v-hash-size %) values))) - (-index-values [search-param resolver resource] + (-index-values [search-param resource-id resolver resource] (when-ok [values (fhir-path/eval resolver expression resource)] - (coll/eduction (p/-index-value-compiler search-param) values))) + (coll/eduction (p/-index-value-compiler search-param resource-id) values))) - (-index-value-compiler [_] + (-index-value-compiler [_ _resource-id] (mapcat (partial index-entries url)))) diff --git a/modules/db/src/blaze/db/impl/search_param/string.clj b/modules/db/src/blaze/db/impl/search_param/string.clj index 64f8e2b88..e340eb44b 100644 --- a/modules/db/src/blaze/db/impl/search_param/string.clj +++ b/modules/db/src/blaze/db/impl/search_param/string.clj @@ -84,18 +84,19 @@ (defn- resource-keys! - "Returns a reducible collection of `[id hash-prefix]` tuples starting at - `start-id` (optional). + "Returns a reducible collection of `[did hash-prefix]` tuples starting at + `start-did` (optional). Changes the state of `context`. Calling this function requires exclusive access to `context`." - {:arglists '([context c-hash tid value] [context c-hash tid value start-id])} + {:arglists '([context c-hash tid value] + [context c-hash tid value start-did])} ([{:keys [svri]} c-hash tid value] (sp-vr/prefix-keys! svri c-hash tid value value)) - ([{:keys [svri] :as context} c-hash tid _value start-id] - (let [start-value (resource-value! context c-hash tid start-id)] + ([{:keys [svri] :as context} c-hash tid _value start-did] + (let [start-value (resource-value! context c-hash tid start-did)] (assert start-value) - (sp-vr/prefix-keys! svri c-hash tid start-value start-value start-id)))) + (sp-vr/prefix-keys! svri c-hash tid start-value start-value start-did)))) (defn- matches? [{:keys [rsvi]} c-hash resource-handle value] @@ -104,7 +105,7 @@ (defrecord SearchParamString [name url type base code c-hash expression] p/SearchParam - (-compile-value [_ _ value] + (-compile-value [_ _modifier value] (codec/string (normalize-string value))) (-resource-handles [_ context tid _ value] @@ -112,10 +113,10 @@ (u/resource-handle-mapper context tid) (resource-keys! context c-hash tid value))) - (-resource-handles [_ context tid _ value start-id] + (-resource-handles [_ context tid _ value start-did] (coll/eduction (u/resource-handle-mapper context tid) - (resource-keys! context c-hash tid value start-id))) + (resource-keys! context c-hash tid value start-did))) (-compartment-keys [_ context compartment tid value] (c-sp-vr/prefix-keys! (:csvri context) compartment c-hash tid value)) @@ -123,11 +124,11 @@ (-matches? [_ context resource-handle _ values] (some? (some #(matches? context c-hash resource-handle %) values))) - (-index-values [search-param resolver resource] + (-index-values [search-param resource-id resolver resource] (when-ok [values (fhir-path/eval resolver expression resource)] - (coll/eduction (p/-index-value-compiler search-param) values))) + (coll/eduction (p/-index-value-compiler search-param resource-id) values))) - (-index-value-compiler [_] + (-index-value-compiler [_ _resource-id] (mapcat (partial index-entries url)))) diff --git a/modules/db/src/blaze/db/impl/search_param/token.clj b/modules/db/src/blaze/db/impl/search_param/token.clj index d7def7235..6ca7fd586 100644 --- a/modules/db/src/blaze/db/impl/search_param/token.clj +++ b/modules/db/src/blaze/db/impl/search_param/token.clj @@ -20,42 +20,42 @@ (defmulti index-entries "Returns index entries for `value` from a resource." - {:arglists '([url value])} - (fn [_ value] (fhir-spec/fhir-type value))) + {:arglists '([resolve-id url value])} + (fn [_ _ value] (fhir-spec/fhir-type value))) (defmethod index-entries :fhir/id - [_ id] + [_ _ id] (when-let [value (type/value id)] [[nil (codec/v-hash value)]])) (defmethod index-entries :fhir/string - [_ s] + [_ _ s] (when-let [value (type/value s)] [[nil (codec/v-hash value)]])) (defmethod index-entries :fhir/uri - [_ uri] + [_ _ uri] (when-let [value (type/value uri)] [[nil (codec/v-hash value)]])) (defmethod index-entries :fhir/boolean - [_ boolean] + [_ _ boolean] (when-some [value (type/value boolean)] [[nil (codec/v-hash (str value))]])) (defmethod index-entries :fhir/canonical - [_ uri] + [_ _ uri] (when-let [value (type/value uri)] [[nil (codec/v-hash value)]])) (defmethod index-entries :fhir/code - [_ code] + [_ _ code] ;; TODO: system (when-let [value (type/value code)] [[nil (codec/v-hash value)]])) @@ -76,12 +76,12 @@ (defmethod index-entries :fhir/Coding - [_ coding] + [_ _ coding] (token-coding-entries coding)) (defmethod index-entries :fhir/CodeableConcept - [_ {:keys [coding]}] + [_ _ {:keys [coding]}] (coll/eduction (mapcat token-coding-entries) coding)) @@ -100,39 +100,40 @@ (defmethod index-entries :fhir/Identifier - [_ identifier] + [_ _ identifier] (identifier-entries nil identifier)) -(defn- literal-reference-entries [reference] +(defn- literal-reference-entries [resolve-id reference] (when-let [value (type/value reference)] (if-let [[type id] (u/split-literal-ref value)] - [[nil (codec/v-hash id)] - [nil (codec/v-hash (str type "/" id))] - [nil (codec/tid-id (codec/tid type) - (codec/id-byte-string id))]] + (let [tid (codec/tid type) + did (resolve-id tid id)] + (cond-> [[nil (codec/v-hash id)] + [nil (codec/v-hash (str type "/" id))]] + did (conj [nil (codec/tid-did tid did)]))) [[nil (codec/v-hash value)]]))) (defmethod index-entries :fhir/Reference - [_ {:keys [reference identifier]}] + [resolve-id _ {:keys [reference identifier]}] (coll/eduction cat (cond-> [] reference - (conj (literal-reference-entries reference)) + (conj (literal-reference-entries resolve-id reference)) identifier (conj (identifier-entries "identifier" identifier))))) (defmethod index-entries :fhir/ContactPoint - [_ {:keys [value]}] + [_ _ {:keys [value]}] (when-let [value (type/value value)] [[nil (codec/v-hash value)]])) (defmethod index-entries :default - [url value] + [_ url value] (log/warn (u/format-skip-indexing-msg value url "token"))) @@ -143,15 +144,15 @@ (defn resource-keys! - "Returns a reducible collection of [id hash-prefix] tuples starting at - `start-id` (optional). + "Returns a reducible collection of [did hash-prefix] tuples starting at + `start-did` (optional). Changes the state of `iter`. Consuming the collection requires exclusive access to `iter`. Doesn't close `iter`." ([{:keys [svri]} c-hash tid value] (sp-vr/prefix-keys! svri c-hash tid value value)) - ([{:keys [svri]} c-hash tid value start-id] - (sp-vr/prefix-keys! svri c-hash tid value value start-id))) + ([{:keys [svri]} c-hash tid value start-did] + (sp-vr/prefix-keys! svri c-hash tid value value start-did))) (defn matches? [{:keys [rsvi]} c-hash resource-handle value] @@ -160,7 +161,7 @@ (defrecord SearchParamToken [name url type base code target c-hash expression] p/SearchParam - (-compile-value [_ _ value] + (-compile-value [_ _modifier value] (codec/v-hash value)) (-resource-handles [_ context tid modifier value] @@ -169,11 +170,11 @@ (resource-keys! context (c-hash-w-modifier c-hash code modifier) tid value))) - (-resource-handles [_ context tid modifier value start-id] + (-resource-handles [_ context tid modifier value start-did] (coll/eduction (u/resource-handle-mapper context tid) (resource-keys! context (c-hash-w-modifier c-hash code modifier) tid value - start-id))) + start-did))) (-compartment-keys [_ context compartment tid value] (c-sp-vr/prefix-keys! (:csvri context) compartment c-hash tid value)) @@ -192,12 +193,12 @@ (some-> (u/split-literal-ref reference) (coll/nth 1)))))) values))) - (-index-values [search-param resolver resource] + (-index-values [search-param resource-id resolver resource] (when-ok [values (fhir-path/eval resolver expression resource)] - (coll/eduction (p/-index-value-compiler search-param) values))) + (coll/eduction (p/-index-value-compiler search-param resource-id) values))) - (-index-value-compiler [_] - (mapcat (partial index-entries url)))) + (-index-value-compiler [_ resource-id] + (mapcat (partial index-entries resource-id url)))) (defn- fix-expr diff --git a/modules/db/src/blaze/db/impl/search_param/util.clj b/modules/db/src/blaze/db/impl/search_param/util.clj index 119142ce4..aab7f47f0 100644 --- a/modules/db/src/blaze/db/impl/search_param/util.clj +++ b/modules/db/src/blaze/db/impl/search_param/util.clj @@ -31,13 +31,13 @@ (str value) (fhir-spec/fhir-type value) url type)) -(def ^:private by-id-grouper - "Transducer which groups `[id hash-prefix]` tuples by `id`." - (partition-by (fn [[id]] id))) +(def ^:private by-did-grouper + "Transducer which groups `[did hash-prefix]` tuples by `did`." + (partition-by (fn [[did]] did))) -(defn non-deleted-resource-handle [resource-handle tid id] - (when-let [handle (resource-handle tid id)] +(defn non-deleted-resource-handle [resource-handle tid did] + (when-let [handle (resource-handle tid did)] (when-not (rh/deleted? handle) handle))) @@ -58,7 +58,7 @@ (defn resource-handle-mapper [context tid] (comp - by-id-grouper + by-did-grouper (resource-handle-mapper* context tid))) @@ -75,12 +75,10 @@ [{:keys [resource-handle]}] (comp ;; there has to be at least some bytes for the id - (filter #(< codec/tid-size (bs/size %))) + (filter #(= (+ codec/tid-size codec/did-size) (bs/size %))) (map bs/as-read-only-byte-buffer) - (keep - #(let [tid (bb/get-int! %) - id (bs/from-byte-buffer! %)] - (non-deleted-resource-handle resource-handle tid id))))) + (keep #(non-deleted-resource-handle resource-handle (bb/get-int! %) + (bb/get-long! %))))) (defn split-literal-ref [^String s] diff --git a/modules/db/src/blaze/db/node.clj b/modules/db/src/blaze/db/node.clj index c340d7818..dfe0a05e1 100644 --- a/modules/db/src/blaze/db/node.clj +++ b/modules/db/src/blaze/db/node.clj @@ -168,13 +168,13 @@ (log/trace "index transaction with t =" t "and" (count tx-cmds) "command(s)") (prom/observe! transaction-sizes (count tx-cmds)) (let [timer (prom/timer duration-seconds "index-resources") - future (resource-indexer/index-resources resource-indexer tx-data) result (index-tx (np/-db node) tx-data)] (if (ba/anomaly? result) (commit-error! node t result) - (do - (store-tx-entries! kv-store result) - (wait-for-resources future timer) + (let [[entries tx-cmds] result] + (store-tx-entries! kv-store entries) + (let [future (resource-indexer/index-resources resource-indexer (assoc tx-data :tx-cmds tx-cmds))] + (wait-for-resources future timer)) (commit-success! node t instant))))) @@ -298,15 +298,15 @@ (-compile-compartment-query [_ code type clauses] (when-ok [clauses (resolve-search-params search-param-registry type clauses false)] - (batch-db/->CompartmentQuery (codec/c-hash code) (codec/tid type) - (seq clauses)))) + (batch-db/->CompartmentQuery (codec/c-hash code) (codec/tid code) + (codec/tid type) (seq clauses)))) (-compile-compartment-query-lenient [_ code type clauses] (if-let [clauses (seq (resolve-search-params search-param-registry type clauses true))] - (batch-db/->CompartmentQuery (codec/c-hash code) (codec/tid type) + (batch-db/->CompartmentQuery (codec/c-hash code) (codec/tid code) (codec/tid type) clauses) - (batch-db/->EmptyCompartmentQuery (codec/c-hash code) (codec/tid type)))) + (batch-db/->EmptyCompartmentQuery (codec/c-hash code) (codec/tid code) (codec/tid type)))) p/Pull (-pull [_ resource-handle] @@ -388,7 +388,7 @@ [:blaze.db/enforce-referential-integrity])) -(def ^:private expected-kv-store-version 1) +(def ^:private expected-kv-store-version 2) (def ^:private incompatible-kv-store-version-msg diff --git a/modules/db/src/blaze/db/node/resource_indexer.clj b/modules/db/src/blaze/db/node/resource_indexer.clj index 3e6af90be..cc36ceabb 100644 --- a/modules/db/src/blaze/db/node/resource_indexer.clj +++ b/modules/db/src/blaze/db/node/resource_indexer.clj @@ -4,6 +4,7 @@ [blaze.async.comp :as ac] [blaze.db.impl.codec :as codec] [blaze.db.impl.index.compartment.resource :as cr] + [blaze.db.impl.index.resource-id :as ri] [blaze.db.impl.search-param :as search-param] [blaze.db.kv :as kv] [blaze.db.kv.spec] @@ -43,33 +44,25 @@ "type") -(defn- compartment-resource-type-entry - "Returns an entry into the :compartment-resource-type-index where `resource` - is linked to `compartment`." - {:arglists '([compartment resource])} - [[comp-code comp-id] {:keys [id] :as resource}] - (cr/index-entry - [(codec/c-hash comp-code) (codec/id-byte-string comp-id)] - (codec/tid (name (fhir-spec/fhir-type resource))) - (codec/id-byte-string id))) +(defn- compartment-resource-type-entry [compartment {:keys [type did]}] + (cr/index-entry compartment (codec/tid type) did)) -(defn- compartment-resource-type-entries [resource compartments] - (mapv #(compartment-resource-type-entry % resource) compartments)) +(defn- compartment-resource-type-entries [entry compartments] + (mapv #(compartment-resource-type-entry % entry) compartments)) -(defn- skip-indexing-msg [search-param resource cause-msg] +(defn- skip-indexing-msg [search-param {:keys [type resource]} cause-msg] (format "Skip indexing for search parameter `%s` on resource `%s/%s`. Cause: %s" - (:url search-param) (name (fhir-spec/fhir-type resource)) - (:id resource) (or cause-msg ""))) + (:url search-param) type (:id resource) (or cause-msg ""))) (defn- search-param-index-entries - [search-param linked-compartments hash resource] - (-> (search-param/index-entries search-param linked-compartments hash resource) + [search-param resource-id compartments {:keys [did hash resource] :as entry}] + (-> (search-param/index-entries search-param resource-id compartments did hash resource) (ba/exceptionally (fn [{::anom/keys [message]}] - (log/warn (skip-indexing-msg search-param resource message)) + (log/warn (skip-indexing-msg search-param entry message)) nil)))) @@ -94,21 +87,34 @@ (update resource :meta (fnil assoc #fhir/Meta{}) :lastUpdated last-updated)) +(defn- code-compartment [resource-id [comp-code comp-id]] + (when-let [co-res-did (resource-id (codec/tid comp-code) comp-id)] + [(codec/c-hash comp-code) co-res-did])) + + +(defn- code-compartments [resource-id compartments] + (into + [] + (keep (partial code-compartment resource-id)) + compartments)) + + (defn- index-resource* - [{:keys [search-param-registry last-updated]} hash resource] - (let [resource (enhance-resource last-updated resource) - compartments (linked-compartments search-param-registry hash resource)] + [{:keys [search-param-registry kv-store]} {:keys [hash resource] :as entry}] + (let [compartments (linked-compartments search-param-registry hash resource) + resource-id (ri/resource-id kv-store) + compartments (code-compartments resource-id compartments)] (into - (compartment-resource-type-entries resource compartments) - (mapcat #(search-param-index-entries % compartments hash resource)) + (compartment-resource-type-entries entry compartments) + (mapcat #(search-param-index-entries % resource-id compartments entry)) (search-params search-param-registry resource)))) -(defn- index-resource [context [hash resource]] +(defn- index-resource [context {:keys [hash type] :as entry}] (log/trace "Index resource with hash" hash) (with-open [_ (prom/timer duration-seconds "index-resource")] - (let [entries (index-resource* context hash resource)] - (prom/observe! index-entries (name (:fhir/type resource)) (count entries)) + (let [entries (index-resource* context entry)] + (prom/observe! index-entries type (count entries)) entries))) @@ -132,6 +138,20 @@ (into [] (keep :hash) tx-cmds)) +(defn- entries [last-updated tx-cmds resources] + (reduce + (fn [res {:keys [did hash]}] + (let [resource (some->> hash (get resources))] + (cond-> res + resource + (conj {:did did + :hash hash + :type (name (fhir-spec/fhir-type resource)) + :resource (enhance-resource last-updated resource)})))) + [] + tx-cmds)) + + (defn index-resources "Returns a CompletableFuture that will complete after all resources of `tx-data` are indexed. @@ -143,10 +163,10 @@ {:keys [tx-cmds] resources :local-payload last-updated :instant}] (let [context (assoc resource-indexer :last-updated last-updated)] (if resources - (index-resources* context resources) + (index-resources* context (entries last-updated tx-cmds resources)) (-> (rs/multi-get resource-store (hashes tx-cmds)) (ac/then-compose - (partial index-resources* context)))))) + #(index-resources* context (entries last-updated tx-cmds %))))))) (defmethod ig/pre-init-spec :blaze.db.node/resource-indexer [_] diff --git a/modules/db/src/blaze/db/node/tx_indexer/verify.clj b/modules/db/src/blaze/db/node/tx_indexer/verify.clj index a6a04d9a8..87bd8b3fd 100644 --- a/modules/db/src/blaze/db/node/tx_indexer/verify.clj +++ b/modules/db/src/blaze/db/node/tx_indexer/verify.clj @@ -4,6 +4,7 @@ [blaze.db.api :as d] [blaze.db.impl.codec :as codec] [blaze.db.impl.index.resource-handle :as rh] + [blaze.db.impl.index.resource-id :as ri] [blaze.db.impl.index.rts-as-of :as rts] [blaze.db.impl.index.system-stats :as system-stats] [blaze.db.impl.index.type-stats :as type-stats] @@ -115,7 +116,7 @@ Throws an anomaly on conflicts." {:arglists '([db-before t res cmd])} - (fn [_ _ _ {:keys [op]}] op)) + (fn [_db-before _t _idx _res {:keys [op]}] op)) (defn- verify-tx-cmd-create-msg [type id] @@ -131,20 +132,22 @@ (throw-anom (ba/conflict (id-collision-msg type id))))) -(defn- index-entries [tid id t hash num-changes op] - (rts/index-entries tid (codec/id-byte-string id) t hash num-changes op)) +(def ^:private inc-0 (fnil inc 0)) (defmethod verify-tx-cmd "create" - [db-before t res {:keys [type id hash]}] + [db-before t idx res {:keys [type id hash] :as cmd}] (log/trace (verify-tx-cmd-create-msg type id)) (with-open [_ (prom/timer duration-seconds "verify-create")] (check-id-collision! db-before type id) - (let [tid (codec/tid type)] - (-> (update res :entries into (index-entries tid id t hash 1 :create)) + (let [tid (codec/tid type) + did (codec/did t idx)] + (-> (update res :entries into (rts/index-entries tid did t hash 1 :create id)) + (update :entries conj (ri/index-entry tid id did)) (update :new-resources conj [type id]) - (update-in [:stats tid :num-changes] (fnil inc 0)) - (update-in [:stats tid :total] (fnil inc 0)))))) + (update :cmds conj (assoc cmd :did did)) + (update-in [:stats tid :num-changes] inc-0) + (update-in [:stats tid :total] inc-0))))) (defn- verify-tx-cmd-put-msg [type id matches] @@ -162,49 +165,57 @@ (defmethod verify-tx-cmd "put" - [db-before t res {:keys [type id hash if-match]}] + [db-before t idx res {:keys [type id hash if-match] :as cmd}] (log/trace (verify-tx-cmd-put-msg type id if-match)) (with-open [_ (prom/timer duration-seconds "verify-put")] (let [tid (codec/tid type) - {:keys [num-changes op] :or {num-changes 0} old-t :t} - (d/resource-handle db-before type id)] + {:keys [did num-changes op] :or {did (codec/did t idx) num-changes 0} + old-t :t} (d/resource-handle db-before type id)] (if (or (nil? if-match) (= if-match old-t)) (cond-> - (-> (update res :entries into (index-entries tid id t hash (inc num-changes) :put)) + (-> (update res :entries into (rts/index-entries tid did t hash (inc num-changes) :put id)) (update :new-resources conj [type id]) - (update-in [:stats tid :num-changes] (fnil inc 0))) + (update :cmds conj (assoc cmd :did did)) + (update-in [:stats tid :num-changes] inc-0)) + (nil? old-t) + (update :entries conj (ri/index-entry tid id did)) (or (nil? old-t) (identical? :delete op)) - (update-in [:stats tid :total] (fnil inc 0))) + (update-in [:stats tid :total] inc-0)) (throw-anom (precondition-failed-anomaly if-match type id)))))) (defmethod verify-tx-cmd "delete" - [db-before t res {:keys [type id]}] + [db-before t idx res {:keys [type id]}] (log/trace "verify-tx-cmd :delete" (str type "/" id)) (with-open [_ (prom/timer duration-seconds "verify-delete")] (let [tid (codec/tid type) - {:keys [num-changes op] :or {num-changes 0}} + {:keys [did num-changes op] :or {did (codec/did t idx) num-changes 0}} (d/resource-handle db-before type id)] (cond-> - (-> (update res :entries into (index-entries tid id t hash/deleted-hash (inc num-changes) :delete)) + (-> (update res :entries into (rts/index-entries tid did t hash/deleted-hash (inc num-changes) :delete id)) (update :del-resources conj [type id]) - (update-in [:stats tid :num-changes] (fnil inc 0))) + (update-in [:stats tid :num-changes] inc-0)) + (nil? op) + (update :entries conj (ri/index-entry tid id did)) (and op (not (identical? :delete op))) (update-in [:stats tid :total] (fnil dec 0)))))) (defmethod verify-tx-cmd :default - [_ _ res _] + [_db-before _t _idx res _tx-cmd] res) (defn- verify-tx-cmds** [db-before t tx-cmds] - (reduce - (partial verify-tx-cmd db-before t) - {:entries [] - :new-resources #{} - :del-resources #{}} - tx-cmds)) + (let [idx (volatile! -1)] + (reduce + (fn [res tx-cmd] + (verify-tx-cmd db-before t (vswap! idx inc) res tx-cmd)) + {:entries [] + :cmds [] + :new-resources #{} + :del-resources #{}} + tx-cmds))) (def ^:private empty-stats @@ -230,10 +241,11 @@ (system-stats/index-entry new-t (apply merge-with + current-stats (vals stats)))))) -(defn- post-process-res [db-before t {:keys [entries stats]}] - (cond-> (conj-type-stats entries db-before t stats) - stats - (conj (system-stats db-before t stats)))) +(defn- post-process-res [db-before t {:keys [entries stats cmds]}] + [(cond-> (conj-type-stats entries db-before t stats) + stats + (conj (system-stats db-before t stats))) + cmds]) (defn- resource-exists? [db type id] diff --git a/modules/db/test-perf/blaze/db/api_test_perf.clj b/modules/db/test-perf/blaze/db/api_test_perf.clj index c31b39296..81dcc47c8 100644 --- a/modules/db/test-perf/blaze/db/api_test_perf.clj +++ b/modules/db/test-perf/blaze/db/api_test_perf.clj @@ -61,6 +61,7 @@ :tx-success-index {:reverse-comparator? true} :tx-error-index nil :t-by-instant-index {:reverse-comparator? true} + :resource-id-index nil :resource-as-of-index nil :type-as-of-index nil :system-as-of-index nil @@ -96,7 +97,7 @@ (deftest transact-test (with-system [{:blaze.db/keys [node]} system] - ;; 58.8 µs / 1.76 µs - Macbook Pro M1 Pro, Oracle OpenJDK 17.0.2 + ;; 66.7 µs / 1.92 µs - Macbook Pro M1 Pro, Oracle OpenJDK 17.0.2 (criterium/bench @(d/transact node [[:put {:fhir/type :fhir/Patient :id "0"}]])))) diff --git a/modules/db/test-perf/blaze/db/impl/index/resource_handle_test_perf.clj b/modules/db/test-perf/blaze/db/impl/index/resource_handle_test_perf.clj deleted file mode 100644 index bc6279628..000000000 --- a/modules/db/test-perf/blaze/db/impl/index/resource_handle_test_perf.clj +++ /dev/null @@ -1,51 +0,0 @@ -(ns blaze.db.impl.index.resource-handle-test-perf - (:require - [blaze.byte-buffer :as bb] - [blaze.db.impl.index.resource-handle :as rh] - [clojure.spec.test.alpha :as st] - [clojure.test :as test :refer [are deftest testing]] - [cuerdas.core :as str]) - (:import - [org.openjdk.jol.info GraphLayout])) - - -(set! *warn-on-reflection* true) -(st/instrument) - - -(defn- fixture [f] - (st/instrument) - (f) - (st/unstrument)) - - -(test/use-fixtures :each fixture) - - -(defn- total-size [& xs] - (.totalSize (GraphLayout/parseInstance (object-array xs)))) - - -(defn- resource-handle [id-size] - (rh/resource-handle 0 (str/repeat "0" id-size) 0 (bb/allocate 40))) - - -(deftest resource-handle-test - (testing "instance size" - (are [id-size size] (= size (total-size (resource-handle id-size))) - 1 272 - 8 272 - 9 280 - 16 280 - 17 288 - 24 288 - 25 296 - 32 296 - 33 304 - 40 304 - 41 312 - 48 312 - 49 320 - 56 320 - 57 328 - 64 328))) diff --git a/modules/db/test/blaze/db/api_test.clj b/modules/db/test/blaze/db/api_test.clj index c32472d3f..d9355a867 100644 --- a/modules/db/test/blaze/db/api_test.clj +++ b/modules/db/test/blaze/db/api_test.clj @@ -8,6 +8,7 @@ [blaze.coll.core :as coll] [blaze.db.api :as d] [blaze.db.api-spec] + [blaze.db.impl.codec :as codec] [blaze.db.impl.db-spec] [blaze.db.impl.index.resource-search-param-value-test-util :as r-sp-v-tu] [blaze.db.kv.mem-spec] @@ -2183,32 +2184,32 @@ (testing "ResourceSearchParamValue index looks like it should" (is (= (r-sp-v-tu/decode-index-entries (:kv-store node) - :type :id :hash-prefix :code :v-hash) - [["Observation" "id-0" #blaze/hash-prefix"36A9F36D" + :type :did :hash-prefix :code :v-hash) + [["Observation" (codec/did 1 0) #blaze/hash-prefix"36A9F36D" "value-quantity" #blaze/byte-string"4F40902F3B6AE19A80"] - ["Observation" "id-0" #blaze/hash-prefix"36A9F36D" + ["Observation" (codec/did 1 0) #blaze/hash-prefix"36A9F36D" "value-quantity" #blaze/byte-string"9CEABF1B055DDDCF80"] - ["Observation" "id-0" #blaze/hash-prefix"36A9F36D" + ["Observation" (codec/did 1 0) #blaze/hash-prefix"36A9F36D" "value-quantity" #blaze/byte-string"B658D8AF4F417A2B80"] - ["Observation" "id-0" #blaze/hash-prefix"36A9F36D" + ["Observation" (codec/did 1 0) #blaze/hash-prefix"36A9F36D" "combo-value-quantity" #blaze/byte-string"4F40902F3B6AE19A80"] - ["Observation" "id-0" #blaze/hash-prefix"36A9F36D" + ["Observation" (codec/did 1 0) #blaze/hash-prefix"36A9F36D" "combo-value-quantity" #blaze/byte-string"9CEABF1B055DDDCF80"] - ["Observation" "id-0" #blaze/hash-prefix"36A9F36D" + ["Observation" (codec/did 1 0) #blaze/hash-prefix"36A9F36D" "combo-value-quantity" #blaze/byte-string"B658D8AF4F417A2B80"] - ["Observation" "id-0" #blaze/hash-prefix"36A9F36D" + ["Observation" (codec/did 1 0) #blaze/hash-prefix"36A9F36D" "_id" #blaze/byte-string"490E5C1C8B04CCEC"] - ["Observation" "id-0" #blaze/hash-prefix"36A9F36D" + ["Observation" (codec/did 1 0) #blaze/hash-prefix"36A9F36D" "_lastUpdated" #blaze/byte-string"80008001"] - ["TestScript" "id-0" #blaze/hash-prefix"51E67D28" + ["TestScript" (codec/did 1 1) #blaze/hash-prefix"51E67D28" "context-quantity" #blaze/byte-string"4F40902F3B6AE19A80"] - ["TestScript" "id-0" #blaze/hash-prefix"51E67D28" + ["TestScript" (codec/did 1 1) #blaze/hash-prefix"51E67D28" "context-quantity" #blaze/byte-string"9CEABF1B055DDDCF80"] - ["TestScript" "id-0" #blaze/hash-prefix"51E67D28" + ["TestScript" (codec/did 1 1) #blaze/hash-prefix"51E67D28" "context-quantity" #blaze/byte-string"B658D8AF4F417A2B80"] - ["TestScript" "id-0" #blaze/hash-prefix"51E67D28" + ["TestScript" (codec/did 1 1) #blaze/hash-prefix"51E67D28" "_id" #blaze/byte-string"490E5C1C8B04CCEC"] - ["TestScript" "id-0" #blaze/hash-prefix"51E67D28" + ["TestScript" (codec/did 1 1) #blaze/hash-prefix"51E67D28" "_lastUpdated" #blaze/byte-string"80008001"]]))) @@ -3145,14 +3146,6 @@ [0 :id] := "0" [0 :meta :versionId] := #fhir/id"1")) - (testing "starting with Measure also returns the patient, - because in type hash order, Measure comes before - Patient but after Observation" - (given @(d/pull-many node (d/system-list (d/db node) "Measure" "0")) - [0 :fhir/type] := :fhir/Patient - [0 :id] := "0" - [0 :meta :versionId] := #fhir/id"1")) - (testing "overshooting the start-id returns an empty collection" (is (coll/empty? (d/system-list (d/db node) "Patient" "1"))))))) @@ -3250,7 +3243,7 @@ (testing "Unknown compartment is not a problem" (with-system [{:blaze.db/keys [node]} system] (is (coll/empty? (d/list-compartment-resource-handles - (d/db node) "foo" "bar" "Condition")))))) + (d/db node) "Foo" "bar" "Condition")))))) (defn- pull-compartment-query [node code id type clauses] @@ -3455,7 +3448,7 @@ (testing "Unknown compartment is not a problem" (with-system [{:blaze.db/keys [node]} system] (is (coll/empty? (d/compartment-query - (d/db node) "foo" "bar" "Condition" + (d/db node) "Foo" "bar" "Condition" [["code" "baz"]]))))) (testing "Unknown type is not a problem" diff --git a/modules/db/test/blaze/db/impl/codec/spec.clj b/modules/db/test/blaze/db/impl/codec/spec.clj index 751b9f994..2fc741c74 100644 --- a/modules/db/test/blaze/db/impl/codec/spec.clj +++ b/modules/db/test/blaze/db/impl/codec/spec.clj @@ -1,10 +1,8 @@ (ns blaze.db.impl.codec.spec (:require [blaze.byte-string :as bs :refer [byte-string?]] - [blaze.db.impl.codec :as codec] [clojure.spec.alpha :as s] [clojure.spec.gen.alpha :as gen] - [clojure.string :as str] [clojure.test.check.generators :as gen2])) @@ -32,13 +30,15 @@ (s/with-gen int? gen/int)) -(def ^:private id-gen - #(gen/fmap (comp codec/id-byte-string str/join) - (gen/vector gen2/char-alphanumeric 1 64))) - - -(s/def :blaze.db/id-byte-string - (s/with-gen (s/and byte-string? #(<= 1 (bs/size %) 64)) id-gen)) +;; A database resource id is a long value were the first 5 bytes is the `t` at +;; which the resource was created and the last 3 bytes are the index of the +;; resource in the transaction. +(s/def :blaze.db/did + (s/with-gen + (s/and int? #(< 0xFFF %)) + #(gen/fmap + (fn [[t n]] (+ (bit-shift-left t 24) n)) + (gen/tuple (s/gen :blaze.db/t) (gen/choose 0 0xFFFFFE))))) (def ^:private byte-string-gen diff --git a/modules/db/test/blaze/db/impl/codec_spec.clj b/modules/db/test/blaze/db/impl/codec_spec.clj index 6c4471ecd..d344833a9 100644 --- a/modules/db/test/blaze/db/impl/codec_spec.clj +++ b/modules/db/test/blaze/db/impl/codec_spec.clj @@ -1,10 +1,12 @@ (ns blaze.db.impl.codec-spec (:require + [blaze.byte-buffer :as bb] [blaze.byte-string :refer [byte-string?]] [blaze.byte-string-spec] [blaze.db.api-spec] [blaze.db.impl.codec :as codec] [blaze.db.impl.codec.spec] + [blaze.db.tx-log.spec] [blaze.fhir.spec] [blaze.fhir.spec.type.system :as system] [blaze.fhir.spec.type.system-spec] @@ -17,14 +19,14 @@ ;; ---- Identifier Functions -------------------------------------------------- -(s/fdef codec/id-byte-string - :args (s/cat :id :blaze.resource/id) - :ret :blaze.db/id-byte-string) +(s/fdef codec/id-from-byte-buffer + :args (s/cat :buf bb/byte-buffer?) + :ret string?) -(s/fdef codec/id-string - :args (s/cat :id-byte-string :blaze.db/id-byte-string) - :ret :blaze.resource/id) +(s/fdef codec/did + :args (s/cat :t :blaze.db/t :idx nat-int?) + :ret :blaze.db/did) ;; ---- Other Functions ------------------------------------------------------- @@ -49,8 +51,8 @@ :ret byte-string?) -(s/fdef codec/tid-id - :args (s/cat :type :blaze.db/tid :id :blaze.db/id-byte-string) +(s/fdef codec/tid-did + :args (s/cat :type :blaze.db/tid :did :blaze.db/did) :ret byte-string?) diff --git a/modules/db/test/blaze/db/impl/codec_test.clj b/modules/db/test/blaze/db/impl/codec_test.clj index 7e7461c1d..ffc601342 100644 --- a/modules/db/test/blaze/db/impl/codec_test.clj +++ b/modules/db/test/blaze/db/impl/codec_test.clj @@ -8,7 +8,6 @@ [clojure.spec.alpha :as s] [clojure.spec.test.alpha :as st] [clojure.test :as test :refer [are deftest is testing]] - [clojure.test.check.generators :as gen] [clojure.test.check.properties :as prop]) (:import [java.nio.charset StandardCharsets] @@ -35,21 +34,13 @@ `(is (not-every? :failure (st/check ~sym ~opts))))) -(deftest id-string-id-byte-string-test - (satisfies-prop 1000 - (prop/for-all [s (s/gen :blaze.resource/id)] - (= s - (codec/id-string (codec/id-byte-string s)) - (apply codec/id-string [(apply codec/id-byte-string [s])]))))) - - (deftest descending-long-test (are [t dt] (= dt (codec/descending-long t)) - 1 0xFFFFFFFFFFFFFE - 0 0xFFFFFFFFFFFFFF) + 1 0xFFFFFFFFFE + 0 0xFFFFFFFFFF) (satisfies-prop 100000 - (prop/for-all [t gen/nat] + (prop/for-all [t (s/gen :blaze.db/t)] (= t (codec/descending-long (codec/descending-long t)) (apply codec/descending-long [(apply codec/descending-long [t])]))))) diff --git a/modules/db/test/blaze/db/impl/db_spec.clj b/modules/db/test/blaze/db/impl/db_spec.clj index 1efc82cd5..baf7e52ac 100644 --- a/modules/db/test/blaze/db/impl/db_spec.clj +++ b/modules/db/test/blaze/db/impl/db_spec.clj @@ -5,6 +5,7 @@ [blaze.db.impl.codec-spec] [blaze.db.impl.db :as db] [blaze.db.impl.index-spec] + [blaze.db.impl.index.resource-as-of-spec] [blaze.db.impl.index.system-stats-spec] [blaze.db.impl.index.type-stats-spec] [blaze.db.impl.search-param-spec] diff --git a/modules/db/test/blaze/db/impl/index/compartment/resource_spec.clj b/modules/db/test/blaze/db/impl/index/compartment/resource_spec.clj index bf5ab5fc9..abf9737df 100644 --- a/modules/db/test/blaze/db/impl/index/compartment/resource_spec.clj +++ b/modules/db/test/blaze/db/impl/index/compartment/resource_spec.clj @@ -14,12 +14,12 @@ :args (s/cat :context :blaze.db.impl.batch-db/context :compartment :blaze.db/compartment :tid :blaze.db/tid - :start-id (s/? :blaze.db/id-byte-string)) + :start-did (s/? :blaze.db/did)) :ret (s/coll-of :blaze.db/resource-handle :kind sequential?)) (s/fdef cr/index-entry :args (s/cat :compartment :blaze.db/compartment :tid :blaze.db/tid - :id :blaze.db/id-byte-string) + :did :blaze.db/did) :ret :blaze.db.kv/put-entry) diff --git a/modules/db/test/blaze/db/impl/index/compartment/resource_test_util.clj b/modules/db/test/blaze/db/impl/index/compartment/resource_test_util.clj index 1d8c28d56..db1fae55c 100644 --- a/modules/db/test/blaze/db/impl/index/compartment/resource_test_util.clj +++ b/modules/db/test/blaze/db/impl/index/compartment/resource_test_util.clj @@ -18,10 +18,9 @@ {:compartment [(let [c-hash (bb/get-int! buf)] (tu/co-c-hash->code c-hash (Integer/toHexString c-hash))) - (let [id-size (bb/size-up-to-null buf)] - (codec/id-string (bs/from-byte-buffer! buf id-size)))] - :type (do (bb/get-byte! buf) (codec/tid->type (bb/get-int! buf))) - :id (codec/id-string (bs/from-byte-buffer! buf))})) + (bb/get-long! buf)] + :type (codec/tid->type (bb/get-int! buf)) + :did (bb/get-long! buf)})) (defn decode-index-entries [kv-store & keys] diff --git a/modules/db/test/blaze/db/impl/index/compartment/search_param_value_resource_spec.clj b/modules/db/test/blaze/db/impl/index/compartment/search_param_value_resource_spec.clj index 9bcf8c07f..cdff3babe 100644 --- a/modules/db/test/blaze/db/impl/index/compartment/search_param_value_resource_spec.clj +++ b/modules/db/test/blaze/db/impl/index/compartment/search_param_value_resource_spec.clj @@ -37,6 +37,6 @@ :c-hash :blaze.db/c-hash :tid :blaze.db/tid :value byte-string? - :id :blaze.db/id-byte-string + :did :blaze.db/did :hash :blaze.resource/hash) :ret :blaze.db.kv/put-entry) diff --git a/modules/db/test/blaze/db/impl/index/compartment/search_param_value_resource_test_util.clj b/modules/db/test/blaze/db/impl/index/compartment/search_param_value_resource_test_util.clj index 6f39254ea..b23b1af60 100644 --- a/modules/db/test/blaze/db/impl/index/compartment/search_param_value_resource_test_util.clj +++ b/modules/db/test/blaze/db/impl/index/compartment/search_param_value_resource_test_util.clj @@ -13,24 +13,21 @@ (set! *unchecked-math* :warn-on-boxed) -(defn decode-key-human +(defn- decode-key-human ([] (bb/allocate-direct 128)) ([buf] - (let [id-size (bb/get-byte! buf (- (bb/limit buf) hash/prefix-size 1))] - {:compartment - [(let [c-hash (bb/get-int! buf)] - (tu/co-c-hash->code c-hash (Integer/toHexString c-hash))) - (-> (bs/from-byte-buffer! buf (bb/size-up-to-null buf)) codec/id-string)] - :code (let [_ (bb/get-byte! buf) - c-hash (bb/get-int! buf)] - (codec/c-hash->code c-hash (Integer/toHexString c-hash))) - :type (codec/tid->type (bb/get-int! buf)) - :v-hash (let [size (- (bb/remaining buf) hash/prefix-size id-size 2)] - (bs/from-byte-buffer! buf size)) - :id (do (bb/get-byte! buf) - (codec/id-string (bs/from-byte-buffer! buf id-size))) - :hash-prefix (do (bb/get-byte! buf) - (hash/prefix-from-byte-buffer! buf))}))) + {:compartment + [(let [c-hash (bb/get-int! buf)] + (tu/co-c-hash->code c-hash (Integer/toHexString c-hash))) + (bb/get-long! buf)] + :code (let [c-hash (bb/get-int! buf)] + (codec/c-hash->code c-hash (Integer/toHexString c-hash))) + :type (codec/tid->type (bb/get-int! buf)) + :v-hash (let [size (- (bb/remaining buf) hash/prefix-size codec/did-size 1)] + (bs/from-byte-buffer! buf size)) + :did (do (bb/get-byte! buf) + (bb/get-long! buf)) + :hash-prefix (hash/prefix-from-byte-buffer! buf)})) (defn decode-index-entries [kv-store & keys] diff --git a/modules/db/test/blaze/db/impl/index/resource_as_of_spec.clj b/modules/db/test/blaze/db/impl/index/resource_as_of_spec.clj index 2bc0426f5..d1d481c42 100644 --- a/modules/db/test/blaze/db/impl/index/resource_as_of_spec.clj +++ b/modules/db/test/blaze/db/impl/index/resource_as_of_spec.clj @@ -13,28 +13,28 @@ (s/fdef rao/encode-key - :args (s/cat :tid :blaze.db/tid :id :blaze.db/id-byte-string :t :blaze.db/t) + :args (s/cat :tid :blaze.db/tid :did :blaze.db/did :t :blaze.db/t) :ret bytes?) (s/fdef rao/type-list :args (s/cat :context :blaze.db.impl.batch-db/context :tid :blaze.db/tid - :start-id (s/? :blaze.db/id-byte-string)) + :start-did (s/? :blaze.db/did)) :ret (s/coll-of :blaze.db/resource-handle :kind sequential?)) (s/fdef rao/system-list :args (s/cat :context :blaze.db.impl.batch-db/context :start (s/? (s/cat :start-tid :blaze.db/tid - :start-id :blaze.db/id-byte-string))) + :start-did :blaze.db/did))) :ret (s/coll-of :blaze.db/resource-handle :kind sequential?)) (s/fdef rao/instance-history :args (s/cat :raoi :blaze.db/kv-iterator :tid :blaze.db/tid - :id :blaze.db/id-byte-string + :did :blaze.db/did :start-t :blaze.db/t :end-t :blaze.db/t) :ret (s/coll-of :blaze.db/resource-handle :kind sequential?)) @@ -45,7 +45,7 @@ :args (s/cat :tid :blaze.db/tid - :id :blaze.db/id-byte-string + :did :blaze.db/did :t (s/? :blaze.db/t)) :ret (s/nilable :blaze.db/resource-handle))) @@ -61,7 +61,7 @@ (s/fdef rao/num-of-instance-changes :args (s/cat :resource-handle ::resource-handle-fn :tid :blaze.db/tid - :id :blaze.db/id-byte-string + :did :blaze.db/did :start-t :blaze.db/t :end-t :blaze.db/t) :ret nat-int?) diff --git a/modules/db/test/blaze/db/impl/index/resource_as_of_test.clj b/modules/db/test/blaze/db/impl/index/resource_as_of_test.clj new file mode 100644 index 000000000..bc7906fbb --- /dev/null +++ b/modules/db/test/blaze/db/impl/index/resource_as_of_test.clj @@ -0,0 +1,25 @@ +(ns blaze.db.impl.index.resource-as-of-test + (:require + [blaze.byte-buffer :as bb] + [blaze.db.impl.codec :as codec] + [blaze.db.impl.codec.spec] + [blaze.db.impl.index.resource-as-of :as rao] + [blaze.test-util :refer [satisfies-prop]] + [clojure.spec.alpha :as s] + [clojure.test :refer [deftest]] + [clojure.test.check.properties :as prop])) + + +(defn decode-key [buf] + {:tid (bb/get-int! buf) + :did (bb/get-long! buf) + :t (codec/descending-long (bb/get-5-byte-long! buf))}) + + +(deftest encode-key-test + (satisfies-prop 10000 + (prop/for-all [tid (s/gen :blaze.db/tid) + did (s/gen :blaze.db/did) + t (s/gen :blaze.db/t)] + (= {:tid tid :did did :t t} + (decode-key (bb/wrap (rao/encode-key tid did t))))))) diff --git a/modules/db/test/blaze/db/impl/index/resource_as_of_test_util.clj b/modules/db/test/blaze/db/impl/index/resource_as_of_test_util.clj index d39284785..9ac0621b0 100644 --- a/modules/db/test/blaze/db/impl/index/resource_as_of_test_util.clj +++ b/modules/db/test/blaze/db/impl/index/resource_as_of_test_util.clj @@ -1,7 +1,6 @@ (ns blaze.db.impl.index.resource-as-of-test-util (:require [blaze.byte-buffer :as bb] - [blaze.byte-string :as bs] [blaze.db.impl.codec :as codec] [blaze.db.impl.index.resource-handle :as rh] [blaze.fhir.hash :as hash])) @@ -10,26 +9,18 @@ (set! *unchecked-math* :warn-on-boxed) -(defn decode-key-human - ([] (bb/allocate-direct 128)) - ([buf] - (let [tid (bb/get-int! buf) - id-size (- (bb/remaining buf) codec/t-size)] - {:type (codec/tid->type tid) - :id (codec/id-string (bs/from-byte-buffer! buf id-size)) - :t (codec/descending-long (bb/get-long! buf))}))) +(defn decode-key [byte-array] + (let [buf (bb/wrap byte-array)] + {:type (codec/tid->type (bb/get-int! buf)) + :did (bb/get-long! buf) + :t (codec/descending-long (bb/get-5-byte-long! buf))})) -(defn decode-value-human - ([] (bb/allocate-direct (+ hash/size Long/BYTES))) - ([buf] - (let [hash (bs/from-byte-buffer! buf hash/size) - state (bb/get-long! buf)] - {:hash hash - :num-changes (rh/state->num-changes state) - :op (rh/state->op state)}))) - - -(defn decode-index-entry [[k v]] - [(decode-key-human (bb/wrap k)) - (decode-value-human (bb/wrap v))]) +(defn decode-val [byte-array] + (let [buf (bb/wrap byte-array) + hash (hash/from-byte-buffer! buf) + state (bb/get-long! buf)] + {:hash hash + :num-changes (rh/state->num-changes state) + :op (rh/state->op state) + :id (codec/id-from-byte-buffer buf)})) diff --git a/modules/db/test/blaze/db/impl/index/resource_handle_spec.clj b/modules/db/test/blaze/db/impl/index/resource_handle_spec.clj index 2a55b79ed..790f6103b 100644 --- a/modules/db/test/blaze/db/impl/index/resource_handle_spec.clj +++ b/modules/db/test/blaze/db/impl/index/resource_handle_spec.clj @@ -11,7 +11,7 @@ (s/fdef rh/resource-handle - :args (s/cat :tid :blaze.db/tid :id :blaze.resource/id + :args (s/cat :tid :blaze.db/tid :did :blaze.db/did :t :blaze.db/t :value-buffer byte-buffer?) :ret :blaze.db/resource-handle) @@ -31,9 +31,9 @@ :ret :blaze.db/tid) -(s/fdef rh/id +(s/fdef rh/did :args (s/cat :rh rh/resource-handle?) - :ret :blaze.resource/id) + :ret :blaze.db/did) (s/fdef rh/t @@ -46,6 +46,11 @@ :ret :blaze.resource/hash) +(s/fdef rh/id + :args (s/cat :rh rh/resource-handle?) + :ret :blaze.resource/id) + + (s/fdef rh/reference :args (s/cat :rh rh/resource-handle?) :ret :blaze.fhir/local-ref) diff --git a/modules/db/test/blaze/db/impl/index/resource_handle_test.clj b/modules/db/test/blaze/db/impl/index/resource_handle_test.clj index 60be894b8..0284c7eee 100644 --- a/modules/db/test/blaze/db/impl/index/resource_handle_test.clj +++ b/modules/db/test/blaze/db/impl/index/resource_handle_test.clj @@ -1,6 +1,7 @@ (ns blaze.db.impl.index.resource-handle-test (:refer-clojure :exclude [hash]) (:require + [blaze.db.impl.codec :as codec] [blaze.db.impl.index.resource-handle :as rh] [blaze.db.impl.index.resource-handle-spec] [blaze.fhir.hash :as hash] @@ -29,14 +30,14 @@ (defn- resource-handle - ([tid id] - (resource-handle tid id 0)) - ([tid id t] - (resource-handle tid id t hash)) - ([tid id t hash] - (resource-handle tid id t hash :create)) - ([tid id t hash op] - (rh/->ResourceHandle tid id t hash 0 op))) + ([tid did] + (resource-handle tid did 0)) + ([tid did t] + (resource-handle tid did t hash)) + ([tid did t hash] + (resource-handle tid did t hash :create "0")) + ([tid did t hash op id] + (rh/->ResourceHandle tid did t hash 0 op id))) (deftest state->num-changes-test @@ -51,62 +52,73 @@ (deftest deleted-test (are [rh] (and (rh/deleted? rh) (apply rh/deleted? [rh])) - (resource-handle 0 "0" 0 hash :delete))) + (resource-handle 0 0 0 hash :delete "0"))) (deftest tid-test (satisfies-prop 100 (prop/for-all [tid (s/gen :blaze.db/tid)] - (let [rh (resource-handle tid "foo")] + (let [rh (resource-handle tid 0)] (= tid (:tid rh) (rh/tid rh) (apply rh/tid [rh])))))) -(deftest id-test +(deftest did-test (satisfies-prop 100 - (prop/for-all [id (s/gen :blaze.resource/id)] - (let [rh (resource-handle 0 id)] - (= id (:id rh) (rh/id rh) (apply rh/id [rh])))))) + (prop/for-all [did (s/gen :blaze.db/did)] + (let [rh (resource-handle 0 did)] + (= did (:did rh) (rh/did rh) (apply rh/did [rh])))))) (deftest t-test (satisfies-prop 100 (prop/for-all [t (s/gen :blaze.db/t)] - (let [rh (resource-handle 0 "foo" t)] + (let [rh (resource-handle 0 0 t)] (= t (:t rh) (rh/t rh) (apply rh/t [rh])))))) (deftest hash-test (satisfies-prop 100 (prop/for-all [hash (s/gen :blaze.resource/hash)] - (let [rh (resource-handle 0 "foo" 0 hash)] + (let [rh (resource-handle 0 0 0 hash)] (= hash (:hash rh) (rh/hash rh) (apply rh/hash [rh])))))) +(deftest id-test + (satisfies-prop 100 + (prop/for-all [id (s/gen :blaze.resource/id)] + (let [rh (resource-handle 0 0 0 hash :create id)] + (= id (:id rh) (rh/id rh) (apply rh/id [rh])))))) + + (deftest reference-test (satisfies-prop 100 (prop/for-all [id (s/gen :blaze.resource/id)] - (let [rh (resource-handle 1495153489 id)] + (let [rh (resource-handle 1495153489 0 0 hash :create id)] (= (str "Condition/" id) (rh/reference rh) (apply rh/reference [rh])))))) (deftest not-found-key-test - (is (nil? (:foo (resource-handle 0 "foo")))) - (is (= ::not-found (:foo (resource-handle 0 "foo") ::not-found)))) + (is (nil? (:foo (resource-handle 0 0)))) + (is (= ::not-found (:foo (resource-handle 0 0) ::not-found)))) (deftest equals-test (satisfies-prop 100 (prop/for-all [tid (s/gen :blaze.db/tid) - id (s/gen :blaze.resource/id) + did (s/gen :blaze.db/did) t (s/gen :blaze.db/t)] (testing "same instance" - (let [rh (resource-handle tid id t)] + (let [rh (resource-handle tid did t)] (= rh rh))) (testing "separate instances" - (let [rh-1 (resource-handle tid id t) - rh-2 (resource-handle tid id t)] + (let [rh-1 (resource-handle tid did t) + rh-2 (resource-handle tid did t)] (= rh-1 rh-2)))))) + + +(deftest toString-test + (is (= "Patient/182457" (str (resource-handle (codec/tid "Patient") 0 0 hash :put "182457"))))) diff --git a/modules/db/test/blaze/db/impl/index/resource_id_spec.clj b/modules/db/test/blaze/db/impl/index/resource_id_spec.clj new file mode 100644 index 000000000..5b6235bc5 --- /dev/null +++ b/modules/db/test/blaze/db/impl/index/resource_id_spec.clj @@ -0,0 +1,17 @@ +(ns blaze.db.impl.index.resource-id-spec + (:require + [blaze.db.impl.codec.spec] + [blaze.db.impl.index.resource-id :as ri] + [blaze.db.kv.spec] + [blaze.fhir.spec.spec] + [clojure.spec.alpha :as s])) + + +(s/fdef ri/resource-id + :args (s/cat :kv-store :blaze.db/kv-store) + :ret :blaze.db/did) + + +(s/fdef ri/index-entry + :args (s/cat :tid :blaze.db/tid :id :blaze.resource/id :did :blaze.db/did) + :ret bytes?) diff --git a/modules/db/test/blaze/db/impl/index/resource_id_test_util.clj b/modules/db/test/blaze/db/impl/index/resource_id_test_util.clj new file mode 100644 index 000000000..04857a99e --- /dev/null +++ b/modules/db/test/blaze/db/impl/index/resource_id_test_util.clj @@ -0,0 +1,20 @@ +(ns blaze.db.impl.index.resource-id-test-util + (:require + [blaze.byte-buffer :as bb] + [blaze.db.impl.codec :as codec]) + (:import + [com.google.common.primitives Longs])) + + +(set! *warn-on-reflection* true) +(set! *unchecked-math* :warn-on-boxed) + + +(defn decode-key [byte-array] + (let [buf (bb/wrap byte-array)] + {:type (codec/tid->type (bb/get-int! buf)) + :id (codec/id-from-byte-buffer buf)})) + + +(defn decode-val [byte-array] + {:did (Longs/fromByteArray byte-array)}) diff --git a/modules/db/test/blaze/db/impl/index/resource_search_param_value_spec.clj b/modules/db/test/blaze/db/impl/index/resource_search_param_value_spec.clj index 83780712e..b505ceca5 100644 --- a/modules/db/test/blaze/db/impl/index/resource_search_param_value_spec.clj +++ b/modules/db/test/blaze/db/impl/index/resource_search_param_value_spec.clj @@ -26,20 +26,20 @@ :ret (s/nilable byte-string?)) -(s/fdef r-sp-v/index-entry - :args (s/cat :tid :blaze.db/tid - :id :blaze.db/id-byte-string - :hash :blaze.resource/hash - :c-hash :blaze.db/c-hash - :value byte-string?) - :ret :blaze.db.kv/put-entry) - - (s/fdef r-sp-v/prefix-keys! :args (s/cat :iter :blaze.db/kv-iterator :tid :blaze.db/tid - :id :blaze.db/id-byte-string + :did :blaze.db/did :hash :blaze.resource/hash :c-hash :blaze.db/c-hash :prefix-value (s/? byte-string?) :start-value (s/? byte-string?))) + + +(s/fdef r-sp-v/index-entry + :args (s/cat :tid :blaze.db/tid + :did :blaze.db/did + :hash :blaze.resource/hash + :c-hash :blaze.db/c-hash + :value byte-string?) + :ret :blaze.db.kv/put-entry) diff --git a/modules/db/test/blaze/db/impl/index/resource_search_param_value_test_util.clj b/modules/db/test/blaze/db/impl/index/resource_search_param_value_test_util.clj index 27a3a33ee..3028e981c 100644 --- a/modules/db/test/blaze/db/impl/index/resource_search_param_value_test_util.clj +++ b/modules/db/test/blaze/db/impl/index/resource_search_param_value_test_util.clj @@ -3,6 +3,7 @@ [blaze.byte-buffer :as bb] [blaze.byte-string :as bs] [blaze.db.impl.codec :as codec] + [blaze.db.impl.index.resource-search-param-value :as r-sp-v] [blaze.db.impl.iterators :as i] [blaze.db.kv :as kv] [blaze.fhir.hash :as hash])) @@ -13,16 +14,14 @@ (defn decode-key-human - ([] (bb/allocate-direct 128)) + ([] (bb/allocate-direct r-sp-v/key-buffer-capacity)) ([buf] (let [tid (bb/get-int! buf) - id-size (bb/size-up-to-null buf) - id (bs/from-byte-buffer! buf id-size) - _ (bb/get-byte! buf) + did (bb/get-long! buf) hash-prefix (hash/prefix-from-byte-buffer! buf) c-hash (bb/get-int! buf)] {:type (codec/tid->type tid) - :id (codec/id-string id) + :did did :hash-prefix hash-prefix :code (codec/c-hash->code c-hash (Integer/toHexString c-hash)) :v-hash (bs/from-byte-buffer! buf)}))) diff --git a/modules/db/test/blaze/db/impl/index/rts_as_of_spec.clj b/modules/db/test/blaze/db/impl/index/rts_as_of_spec.clj index 3ec8174a5..bd6cc3d5a 100644 --- a/modules/db/test/blaze/db/impl/index/rts_as_of_spec.clj +++ b/modules/db/test/blaze/db/impl/index/rts_as_of_spec.clj @@ -1,6 +1,7 @@ (ns blaze.db.impl.index.rts-as-of-spec (:require [blaze.db.impl.codec.spec] + [blaze.db.impl.index.resource-as-of-spec] [blaze.db.impl.index.rts-as-of :as rts] [blaze.db.tx-log.spec] [blaze.fhir.hash-spec] @@ -9,11 +10,20 @@ [clojure.spec.alpha :as s])) +(s/fdef rts/encode-value + :args (s/cat :hash :blaze.resource/hash + :num-changes nat-int? + :op keyword? + :id :blaze.resource/id) + :ret bytes?) + + (s/fdef rts/index-entries :args (s/cat :tid :blaze.db/tid - :id :blaze.db/id-byte-string + :did :blaze.db/did :t :blaze.db/t :hash :blaze.resource/hash :num-changes nat-int? - :op keyword?) + :op keyword? + :id :blaze.resource/id) :ret bytes?) diff --git a/modules/db/test/blaze/db/impl/index/search_param_value_resource/impl_test.clj b/modules/db/test/blaze/db/impl/index/search_param_value_resource/impl_test.clj deleted file mode 100644 index 73d7e1457..000000000 --- a/modules/db/test/blaze/db/impl/index/search_param_value_resource/impl_test.clj +++ /dev/null @@ -1,35 +0,0 @@ -(ns blaze.db.impl.index.search-param-value-resource.impl-test - (:require - [blaze.byte-buffer :as bb] - [blaze.byte-string :as bs] - [blaze.db.impl.index.search-param-value-resource :as sp-vr] - [blaze.db.impl.index.search-param-value-resource-spec] - [blaze.db.impl.index.search-param-value-resource.impl :as impl] - [blaze.test-util :refer [satisfies-prop]] - [clojure.spec.alpha :as s] - [clojure.spec.test.alpha :as st] - [clojure.test :as test :refer [deftest]] - [clojure.test.check.properties :as prop])) - - -(st/instrument) - - -(defn- fixture [f] - (st/instrument) - (f) - (st/unstrument)) - - -(test/use-fixtures :each fixture) - - -(deftest id-size-test - (satisfies-prop 1000 - (prop/for-all [c-hash (s/gen :blaze.db/c-hash) - tid (s/gen :blaze.db/tid) - value (s/gen :blaze.db/byte-string) - id (s/gen :blaze.db/id-byte-string) - hash (s/gen :blaze.resource/hash)] - (let [buf (bb/wrap (sp-vr/encode-key c-hash tid value id hash))] - (= (bs/size id) (impl/id-size buf) (apply impl/id-size [buf])))))) diff --git a/modules/db/test/blaze/db/impl/index/search_param_value_resource_spec.clj b/modules/db/test/blaze/db/impl/index/search_param_value_resource_spec.clj index e155c5bc8..0d7a04fd7 100644 --- a/modules/db/test/blaze/db/impl/index/search_param_value_resource_spec.clj +++ b/modules/db/test/blaze/db/impl/index/search_param_value_resource_spec.clj @@ -21,14 +21,14 @@ :tid :blaze.db/tid :prefix-value byte-string? :start-value byte-string? - :start-id (s/? :blaze.db/id-byte-string))) + :start-did (s/? :blaze.db/did))) (s/fdef sp-vr/encode-seek-key :args (s/cat :c-hash :blaze.db/c-hash :tid :blaze.db/tid - :more (s/? (s/cat :value byte-string? - :id (s/? :blaze.db/id-byte-string)))) + :value (s/? byte-string?) + :did (s/? :blaze.db/did)) :ret byte-string?) @@ -36,14 +36,23 @@ :args (s/cat :c-hash :blaze.db/c-hash :tid :blaze.db/tid :value byte-string? - :id (s/? :blaze.db/id-byte-string)) + :did (s/? :blaze.db/did)) :ret byte-string?) +(s/fdef sp-vr/encode-key + :args (s/cat :c-hash :blaze.db/c-hash + :tid :blaze.db/tid + :value byte-string? + :did :blaze.db/did + :hash :blaze.resource/hash) + :ret bytes?) + + (s/fdef sp-vr/index-entry :args (s/cat :c-hash :blaze.db/c-hash :tid :blaze.db/tid :value byte-string? - :id :blaze.db/id-byte-string + :did :blaze.db/did :hash :blaze.resource/hash) :ret :blaze.db.kv/put-entry) diff --git a/modules/db/test/blaze/db/impl/index/search_param_value_resource_test.clj b/modules/db/test/blaze/db/impl/index/search_param_value_resource_test.clj index b1e8851d7..2593a338e 100644 --- a/modules/db/test/blaze/db/impl/index/search_param_value_resource_test.clj +++ b/modules/db/test/blaze/db/impl/index/search_param_value_resource_test.clj @@ -36,41 +36,41 @@ (deftest decode-key-test - (satisfies-prop 100 + (satisfies-prop 1000 (prop/for-all [c-hash (s/gen :blaze.db/c-hash) tid (s/gen :blaze.db/tid) value (s/gen :blaze.db/byte-string) - id (s/gen :blaze.db/id-byte-string) + did (s/gen :blaze.db/did) hash (s/gen :blaze.resource/hash)] - (let [buf (bb/wrap (sp-vr/encode-key c-hash tid value id hash)) - [prefix act_id hash-prefix] (sp-vr/decode-key buf)] + (let [buf (bb/wrap (sp-vr/encode-key c-hash tid value did hash)) + [prefix act-did hash-prefix] (sp-vr/decode-key buf)] (and (= (create-prefix c-hash tid value) prefix) - (= id act_id) + (= did act-did) (= (hash/prefix hash) hash-prefix)))))) (deftest decode-value-id-hash-prefix-test - (satisfies-prop 100 + (satisfies-prop 1000 (prop/for-all [c-hash (s/gen :blaze.db/c-hash) tid (s/gen :blaze.db/tid) value (s/gen :blaze.db/byte-string) - id (s/gen :blaze.db/id-byte-string) + did (s/gen :blaze.db/did) hash (s/gen :blaze.resource/hash)] - (let [buf (bb/wrap (sp-vr/encode-key c-hash tid value id hash)) - [act_value act_id hash-prefix] (sp-vr/decode-value-id-hash-prefix buf)] - (and (= value act_value) - (= id act_id) + (let [buf (bb/wrap (sp-vr/encode-key c-hash tid value did hash)) + [act-value act-did hash-prefix] (sp-vr/decode-value-did-hash-prefix buf)] + (and (= value act-value) + (= did act-did) (= (hash/prefix hash) hash-prefix)))))) (deftest decode-id-hash-prefix-test - (satisfies-prop 100 + (satisfies-prop 1000 (prop/for-all [c-hash (s/gen :blaze.db/c-hash) tid (s/gen :blaze.db/tid) value (s/gen :blaze.db/byte-string) - id (s/gen :blaze.db/id-byte-string) + did (s/gen :blaze.db/did) hash (s/gen :blaze.resource/hash)] - (let [buf (bb/wrap (sp-vr/encode-key c-hash tid value id hash)) - [act_id hash-prefix] (sp-vr/decode-id-hash-prefix buf)] - (and (= id act_id) + (let [buf (bb/wrap (sp-vr/encode-key c-hash tid value did hash)) + [act-did hash-prefix] (sp-vr/decode-did-hash-prefix buf)] + (and (= did act-did) (= (hash/prefix hash) hash-prefix)))))) diff --git a/modules/db/test/blaze/db/impl/index/search_param_value_resource_test_util.clj b/modules/db/test/blaze/db/impl/index/search_param_value_resource_test_util.clj index 4a4fd32c0..2d534d190 100644 --- a/modules/db/test/blaze/db/impl/index/search_param_value_resource_test_util.clj +++ b/modules/db/test/blaze/db/impl/index/search_param_value_resource_test_util.clj @@ -3,6 +3,7 @@ [blaze.byte-buffer :as bb] [blaze.byte-string :as bs] [blaze.db.impl.codec :as codec] + [blaze.db.impl.index.search-param-value-resource :as sp-vr] [blaze.db.impl.iterators :as i] [blaze.db.kv :as kv] [blaze.fhir.hash :as hash])) @@ -13,21 +14,19 @@ (defn decode-key-human - ([] (bb/allocate-direct 128)) + ([] (bb/allocate-direct sp-vr/key-buffer-capacity)) ([buf] - (let [id-size (bb/get-byte! buf (- (bb/limit buf) hash/prefix-size 1)) - value-size (- (bb/remaining buf) id-size 2 hash/prefix-size - codec/c-hash-size codec/tid-size) + (let [value-size (- (bb/remaining buf) 1 codec/did-size hash/prefix-size + sp-vr/base-key-size) c-hash (bb/get-int! buf) tid (bb/get-int! buf) value (bs/from-byte-buffer! buf value-size) _ (bb/get-byte! buf) - id (bs/from-byte-buffer! buf id-size) - _ (bb/get-byte! buf)] + did (bb/get-long! buf)] {:code (codec/c-hash->code c-hash (Integer/toHexString c-hash)) :type (codec/tid->type tid) :v-hash value - :id (codec/id-string id) + :did did :hash-prefix (hash/prefix-from-byte-buffer! buf)}))) diff --git a/modules/db/test/blaze/db/impl/index/system_as_of_spec.clj b/modules/db/test/blaze/db/impl/index/system_as_of_spec.clj index 7a9e15b9e..a8194d6ea 100644 --- a/modules/db/test/blaze/db/impl/index/system_as_of_spec.clj +++ b/modules/db/test/blaze/db/impl/index/system_as_of_spec.clj @@ -15,6 +15,6 @@ :args (s/cat :saoi :blaze.db/kv-iterator :start-t :blaze.db/t :start-tid (s/nilable :blaze.db/tid) - :start-id (s/nilable :blaze.db/id-byte-string) + :start-did (s/nilable :blaze.db/did) :end-t :blaze.db/t) :ret (s/coll-of :blaze.db/resource-handle :kind sequential?)) diff --git a/modules/db/test/blaze/db/impl/index/system_as_of_test_util.clj b/modules/db/test/blaze/db/impl/index/system_as_of_test_util.clj index 7f8c3356e..762e42381 100644 --- a/modules/db/test/blaze/db/impl/index/system_as_of_test_util.clj +++ b/modules/db/test/blaze/db/impl/index/system_as_of_test_util.clj @@ -1,7 +1,6 @@ (ns blaze.db.impl.index.system-as-of-test-util (:require [blaze.byte-buffer :as bb] - [blaze.byte-string :as bs] [blaze.db.impl.codec :as codec] [blaze.db.impl.index.resource-handle :as rh] [blaze.fhir.hash :as hash])) @@ -10,24 +9,18 @@ (set! *unchecked-math* :warn-on-boxed) -(defn decode-key-human - ([] (bb/allocate-direct 128)) - ([buf] - {:t (codec/descending-long (bb/get-long! buf)) - :type (codec/tid->type (bb/get-int! buf)) - :id (codec/id-string (bs/from-byte-buffer! buf (bb/remaining buf)))})) +(defn decode-key [byte-array] + (let [buf (bb/wrap byte-array)] + {:t (codec/descending-long (bb/get-5-byte-long! buf)) + :type (codec/tid->type (bb/get-int! buf)) + :did (bb/get-long! buf)})) -(defn decode-value-human - ([] (bb/allocate-direct (+ hash/size Long/BYTES))) - ([buf] - (let [hash (bs/from-byte-buffer! buf hash/size) - state (bb/get-long! buf)] - {:hash hash - :num-changes (rh/state->num-changes state) - :op (rh/state->op state)}))) - - -(defn decode-index-entry [[k v]] - [(decode-key-human (bb/wrap k)) - (decode-value-human (bb/wrap v))]) +(defn decode-val [byte-array] + (let [buf (bb/wrap byte-array) + hash (hash/from-byte-buffer! buf) + state (bb/get-long! buf)] + {:hash hash + :num-changes (rh/state->num-changes state) + :op (rh/state->op state) + :id (codec/id-from-byte-buffer buf)})) diff --git a/modules/db/test/blaze/db/impl/index/system_stats_test_util.clj b/modules/db/test/blaze/db/impl/index/system_stats_test_util.clj index 2cb07ff77..c310f5b52 100644 --- a/modules/db/test/blaze/db/impl/index/system_stats_test_util.clj +++ b/modules/db/test/blaze/db/impl/index/system_stats_test_util.clj @@ -7,19 +7,12 @@ (set! *unchecked-math* :warn-on-boxed) -(defn decode-key-human - ([] (bb/allocate-direct codec/t-size)) - ([buf] - {:t (codec/descending-long (bb/get-long! buf))})) +(defn decode-key [byte-array] + (let [buf (bb/wrap byte-array)] + {:t (codec/descending-long (bb/get-5-byte-long! buf))})) -(defn decode-value-human - ([] (bb/allocate-direct (+ Long/BYTES Long/BYTES))) - ([buf] - {:total (bb/get-long! buf) - :num-changes (bb/get-long! buf)})) - - -(defn decode-index-entry [[k v]] - [(decode-key-human (bb/wrap k)) - (decode-value-human (bb/wrap v))]) +(defn decode-val [byte-array] + (let [buf (bb/wrap byte-array)] + {:total (bb/get-long! buf) + :num-changes (bb/get-long! buf)})) diff --git a/modules/db/test/blaze/db/impl/index/type_as_of_spec.clj b/modules/db/test/blaze/db/impl/index/type_as_of_spec.clj index 792dfba2d..ef2950b2c 100644 --- a/modules/db/test/blaze/db/impl/index/type_as_of_spec.clj +++ b/modules/db/test/blaze/db/impl/index/type_as_of_spec.clj @@ -14,6 +14,6 @@ :args (s/cat :taoi :blaze.db/kv-iterator :tid :blaze.db/tid :start-t :blaze.db/t - :start-id (s/nilable :blaze.db/id-byte-string) + :start-did (s/nilable :blaze.db/did) :end-t :blaze.db/t) :ret (s/coll-of :blaze.db/resource-handle :kind sequential?)) diff --git a/modules/db/test/blaze/db/impl/index/type_as_of_test_util.clj b/modules/db/test/blaze/db/impl/index/type_as_of_test_util.clj index 525bdf2d1..36ad8b26e 100644 --- a/modules/db/test/blaze/db/impl/index/type_as_of_test_util.clj +++ b/modules/db/test/blaze/db/impl/index/type_as_of_test_util.clj @@ -1,7 +1,6 @@ (ns blaze.db.impl.index.type-as-of-test-util (:require [blaze.byte-buffer :as bb] - [blaze.byte-string :as bs] [blaze.db.impl.codec :as codec] [blaze.db.impl.index.resource-handle :as rh] [blaze.fhir.hash :as hash])) @@ -10,24 +9,18 @@ (set! *unchecked-math* :warn-on-boxed) -(defn decode-key-human - ([] (bb/allocate-direct 128)) - ([buf] - {:type (codec/tid->type (bb/get-int! buf)) - :t (codec/descending-long (bb/get-long! buf)) - :id (codec/id-string (bs/from-byte-buffer! buf (bb/remaining buf)))})) +(defn decode-key [byte-array] + (let [buf (bb/wrap byte-array)] + {:type (codec/tid->type (bb/get-int! buf)) + :t (codec/descending-long (bb/get-5-byte-long! buf)) + :did (bb/get-long! buf)})) -(defn decode-value-human - ([] (bb/allocate-direct (+ hash/size Long/BYTES))) - ([buf] - (let [hash (bs/from-byte-buffer! buf hash/size) - state (bb/get-long! buf)] - {:hash hash - :num-changes (rh/state->num-changes state) - :op (rh/state->op state)}))) - - -(defn decode-index-entry [[k v]] - [(decode-key-human (bb/wrap k)) - (decode-value-human (bb/wrap v))]) +(defn decode-val [byte-array] + (let [buf (bb/wrap byte-array) + hash (hash/from-byte-buffer! buf) + state (bb/get-long! buf)] + {:hash hash + :num-changes (rh/state->num-changes state) + :op (rh/state->op state) + :id (codec/id-from-byte-buffer buf)})) diff --git a/modules/db/test/blaze/db/impl/index/type_stats_test_util.clj b/modules/db/test/blaze/db/impl/index/type_stats_test_util.clj index e3bf442e0..8e9b0782c 100644 --- a/modules/db/test/blaze/db/impl/index/type_stats_test_util.clj +++ b/modules/db/test/blaze/db/impl/index/type_stats_test_util.clj @@ -7,20 +7,13 @@ (set! *unchecked-math* :warn-on-boxed) -(defn decode-key-human - ([] (bb/allocate-direct (+ codec/tid-size codec/t-size))) - ([buf] - {:type (codec/tid->type (bb/get-int! buf)) - :t (codec/descending-long (bb/get-long! buf))})) +(defn decode-key [byte-array] + (let [buf (bb/wrap byte-array)] + {:type (codec/tid->type (bb/get-int! buf)) + :t (codec/descending-long (bb/get-5-byte-long! buf))})) -(defn decode-value-human - ([] (bb/allocate-direct (+ Long/BYTES Long/BYTES))) - ([buf] - {:total (bb/get-long! buf) - :num-changes (bb/get-long! buf)})) - - -(defn decode-index-entry [[k v]] - [(decode-key-human (bb/wrap k)) - (decode-value-human (bb/wrap v))]) +(defn decode-val [byte-array] + (let [buf (bb/wrap byte-array)] + {:total (bb/get-long! buf) + :num-changes (bb/get-long! buf)})) diff --git a/modules/db/test/blaze/db/impl/index_spec.clj b/modules/db/test/blaze/db/impl/index_spec.clj index c703a5d40..919953ce5 100644 --- a/modules/db/test/blaze/db/impl/index_spec.clj +++ b/modules/db/test/blaze/db/impl/index_spec.clj @@ -25,7 +25,7 @@ :args (s/cat :context :blaze.db.impl.batch-db/context :tid :blaze.db/tid :clauses :blaze.db.index.query/clauses - :start-id (s/? :blaze.db/id-byte-string)) + :start-did (s/? :blaze.db/did)) :ret (s/coll-of :blaze.db/resource-handle :kind sequential?)) diff --git a/modules/db/test/blaze/db/impl/search_param/composite_test.clj b/modules/db/test/blaze/db/impl/search_param/composite_test.clj index 7a7e33fe6..2cd1298c6 100644 --- a/modules/db/test/blaze/db/impl/search_param/composite_test.clj +++ b/modules/db/test/blaze/db/impl/search_param/composite_test.clj @@ -159,7 +159,8 @@ [_ k12] [_ k13] [_ k14] [_ k15] [_ k16] [_ k17]] (search-param/index-entries (code-value-quantity-param search-param-registry) - [] hash observation)] + (constantly nil) + [] 201853 hash observation)] (testing "`code` followed by `value`" (testing "SearchParamValueResource key" @@ -168,13 +169,13 @@ :type := "Observation" [:v-hash split-value 0] := observation-code [:v-hash split-value 1] := value - :id := "id-155558" + :did := 201853 :hash-prefix := (hash/prefix hash))) (testing "ResourceSearchParamValue key" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "Observation" - :id := "id-155558" + :did := 201853 :hash-prefix := (hash/prefix hash) :code := "code-value-quantity" [:v-hash split-value 0] := observation-code @@ -280,7 +281,8 @@ ::x ::y})] (given (search-param/index-entries (code-value-quantity-param search-param-registry) - [] hash resource) + (constantly nil) + [] 201932 hash resource) ::anom/category := ::anom/fault ::x := ::y))) @@ -293,7 +295,8 @@ ::x ::y}))] (given (anom-vec (search-param/index-entries (code-value-quantity-param search-param-registry) - [] hash resource)) + (constantly nil) + [] 201948 hash resource)) ::anom/category := ::anom/fault ::x := ::y))) @@ -314,7 +317,8 @@ ::x ::y}))] (given (anom-vec (search-param/index-entries (code-value-quantity-param search-param-registry) - [] hash resource)) + (constantly nil) + [] 201954 hash resource)) ::anom/category := ::anom/fault ::x := ::y))))) @@ -329,7 +333,8 @@ ::x ::y})] (given (search-param/index-entries (code-value-concept-param search-param-registry) - [] hash resource) + (constantly nil) + [] 202002 hash resource) ::anom/category := ::anom/fault ::x := ::y)))))))) diff --git a/modules/db/test/blaze/db/impl/search_param/date_test.clj b/modules/db/test/blaze/db/impl/search_param/date_test.clj index 9649c60c8..21b190036 100644 --- a/modules/db/test/blaze/db/impl/search_param/date_test.clj +++ b/modules/db/test/blaze/db/impl/search_param/date_test.clj @@ -90,7 +90,9 @@ hash (hash/generate patient) [[_ k0]] (search-param/index-entries - (birth-date-param search-param-registry) [] hash patient)] + (birth-date-param search-param-registry) + (constantly nil) + [] 202016 hash patient)] (testing "the entry is about both bounds of `2020-02-04`" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) @@ -99,7 +101,7 @@ :v-hash := (codec/date-lb-ub (date-lb (LocalDate/of 2020 2 4)) (date-ub (LocalDate/of 2020 2 4))) - :id := "id-142629" + :did := 202016 :hash-prefix := (hash/prefix hash))))) (testing "death-date" @@ -111,7 +113,8 @@ [[_ k0]] (search-param/index-entries (sr/get search-param-registry "death-date" "Patient") - [] hash patient)] + (constantly nil) + [] 202030 hash patient)] (testing "the entry is about both bounds of `2020-01-01T00:00:00Z`" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) @@ -124,7 +127,7 @@ (date-ub (OffsetDateTime/of 2019 11 17 0 14 29 0 (ZoneOffset/ofHours 1)))) - :id := "id-142629" + :did := 202030 :hash-prefix := (hash/prefix hash)))))) (testing "Encounter" @@ -139,7 +142,8 @@ [[_ k0]] (search-param/index-entries (sr/get search-param-registry "date" "Encounter") - [] hash encounter)] + (constantly nil) + [] 202044 hash encounter)] (testing "the entry is about the lower bound of the start and the upper bound of the end of the period" @@ -153,7 +157,7 @@ (date-ub (OffsetDateTime/of 2019 11 17 0 44 29 0 (ZoneOffset/ofHours 1)))) - :id := "id-160224" + :did := 202044 :hash-prefix := (hash/prefix hash)))) (testing "without start" @@ -166,7 +170,8 @@ [[_ k0]] (search-param/index-entries (sr/get search-param-registry "date" "Encounter") - [] hash encounter)] + (constantly nil) + [] 202100 hash encounter)] (testing "the entry is about the min bound as lower bound and the upper bound of the end of the period" @@ -176,7 +181,7 @@ :v-hash := (codec/date-lb-ub codec/date-min-bound (date-ub (LocalDate/of 2019 11 17))) - :id := "id-160224" + :did := 202100 :hash-prefix := (hash/prefix hash))))) (testing "Encounter date without end" @@ -189,7 +194,8 @@ [[_ k0]] (search-param/index-entries (sr/get search-param-registry "date" "Encounter") - [] hash encounter)] + (constantly nil) + [] 202117 hash encounter)] (testing "the entry is about the lower bound of the start and the max upper bound" @@ -201,7 +207,7 @@ (OffsetDateTime/of 2019 11 17 0 14 29 0 (ZoneOffset/ofHours 1))) codec/date-max-bound) - :id := "id-160224" + :did := 202117 :hash-prefix := (hash/prefix hash))))))) (testing "DiagnosticReport" @@ -213,7 +219,8 @@ [[_ k0]] (search-param/index-entries (sr/get search-param-registry "issued" "DiagnosticReport") - [] hash patient)] + (constantly nil) + [] 202130 hash patient)] (testing "the entry is about both bounds of `2019-11-17T00:14:29.917+01:00`" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) @@ -226,7 +233,7 @@ (date-ub (OffsetDateTime/of 2019 11 17 0 14 29 917 (ZoneOffset/ofHours 1)))) - :id := "id-155607" + :did := 202130 :hash-prefix := (hash/prefix hash)))))) (testing "FHIRPath evaluation problem" @@ -236,7 +243,8 @@ (with-redefs [fhir-path/eval (fn [_ _ _] {::anom/category ::anom/fault})] (given (search-param/index-entries (sr/get search-param-registry "issued" "DiagnosticReport") - [] hash resource) + (constantly nil) + [] 202141 hash resource) ::anom/category := ::anom/fault))))) (testing "skip warning" diff --git a/modules/db/test/blaze/db/impl/search_param/number_test.clj b/modules/db/test/blaze/db/impl/search_param/number_test.clj index 551e8ef7f..355191022 100644 --- a/modules/db/test/blaze/db/impl/search_param/number_test.clj +++ b/modules/db/test/blaze/db/impl/search_param/number_test.clj @@ -103,20 +103,21 @@ [[_ k0] [_ k1]] (search-param/index-entries (sr/get search-param-registry "probability" "RiskAssessment") - [] hash risk-assessment)] + (constantly nil) + [] 202156 hash risk-assessment)] (testing "first SearchParamValueResource key is about `value`" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "probability" :type := "RiskAssessment" :v-hash := (codec/number 0.9M) - :id := "id-163630" + :did := 202156 :hash-prefix := (hash/prefix hash))) (testing "first ResourceSearchParamValue key is about `value`" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "RiskAssessment" - :id := "id-163630" + :did := 202156 :hash-prefix := (hash/prefix hash) :code := "probability" :v-hash := (codec/number 0.9M))))) @@ -132,20 +133,21 @@ [[_ k0] [_ k1]] (search-param/index-entries (sr/get search-param-registry "variant-start" "MolecularSequence") - [] hash risk-assessment)] + (constantly nil) + [] 202219 hash risk-assessment)] (testing "first SearchParamValueResource key is about `value`" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "variant-start" :type := "MolecularSequence" :v-hash := (codec/number 1M) - :id := "id-170736" + :did := 202219 :hash-prefix := (hash/prefix hash))) (testing "first ResourceSearchParamValue key is about `value`" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "MolecularSequence" - :id := "id-170736" + :did := 202219 :hash-prefix := (hash/prefix hash) :code := "variant-start" :v-hash := (codec/number 1M))))) @@ -157,7 +159,8 @@ (with-redefs [fhir-path/eval (fn [_ _ _] {::anom/category ::anom/fault})] (given (search-param/index-entries (sr/get search-param-registry "probability" "RiskAssessment") - [] hash resource) + (constantly nil) + [] 202240 hash resource) ::anom/category := ::anom/fault))))) (testing "skip warning" diff --git a/modules/db/test/blaze/db/impl/search_param/quantity_spec.clj b/modules/db/test/blaze/db/impl/search_param/quantity_spec.clj index 79aa64526..7fde8eeb6 100644 --- a/modules/db/test/blaze/db/impl/search_param/quantity_spec.clj +++ b/modules/db/test/blaze/db/impl/search_param/quantity_spec.clj @@ -17,7 +17,7 @@ :tid :blaze.db/tid :prefix-length nat-int? :value ::spq/value - :start-id (s/? :blaze.db/id-byte-string))) + :start-did (s/? :blaze.db/did))) (s/fdef spq/matches? diff --git a/modules/db/test/blaze/db/impl/search_param/quantity_test.clj b/modules/db/test/blaze/db/impl/search_param/quantity_test.clj index f2f117a34..5bac0f07b 100644 --- a/modules/db/test/blaze/db/impl/search_param/quantity_test.clj +++ b/modules/db/test/blaze/db/impl/search_param/quantity_test.clj @@ -54,7 +54,7 @@ (catch Exception e (is (= "No matching clause: :foo" (ex-message e))))) - (testing "with start-id" + (testing "with start-did" (try (spq/resource-keys! {} (codec/c-hash "value-quantity") 0 0 {:op :foo} 0) (catch Exception e @@ -155,20 +155,21 @@ [[_ k0] [_ k1] [_ k2] [_ k3] [_ k4] [_ k5]] (search-param/index-entries (sr/get search-param-registry "value-quantity" "Observation") - [] hash observation)] + (constantly nil) + [] 153511 hash observation)] (testing "first SearchParamValueResource key is about `value`" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "value-quantity" :type := "Observation" :v-hash := (codec/quantity nil 140M) - :id := "id-155558" + :did := 153511 :hash-prefix := (hash/prefix hash))) (testing "first ResourceSearchParamValue key is about `value`" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "Observation" - :id := "id-155558" + :did := 153511 :hash-prefix := (hash/prefix hash) :code := "value-quantity" :v-hash := (codec/quantity nil 140M))) @@ -178,13 +179,13 @@ :code := "value-quantity" :type := "Observation" :v-hash := (codec/quantity "mm[Hg]" 140M) - :id := "id-155558" + :did := 153511 :hash-prefix := (hash/prefix hash))) (testing "second ResourceSearchParamValue key is about `code value`" (given (r-sp-v-tu/decode-key-human (bb/wrap k3)) :type := "Observation" - :id := "id-155558" + :did := 153511 :hash-prefix := (hash/prefix hash) :code := "value-quantity" :v-hash := (codec/quantity "mm[Hg]" 140M))) @@ -194,13 +195,13 @@ :code := "value-quantity" :type := "Observation" :v-hash := (codec/quantity "http://unitsofmeasure.org|mm[Hg]" 140M) - :id := "id-155558" + :did := 153511 :hash-prefix := (hash/prefix hash))) (testing "third ResourceSearchParamValue key is about `system|code value`" (given (r-sp-v-tu/decode-key-human (bb/wrap k5)) :type := "Observation" - :id := "id-155558" + :did := 153511 :hash-prefix := (hash/prefix hash) :code := "value-quantity" :v-hash := (codec/quantity "http://unitsofmeasure.org|mm[Hg]" 140M))))) @@ -218,20 +219,21 @@ [[_ k0] [_ k1] [_ k2] [_ k3]] (search-param/index-entries (sr/get search-param-registry "value-quantity" "Observation") - [] hash observation)] + (constantly nil) + [] 153548 hash observation)] (testing "first SearchParamValueResource key is about `value`" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "value-quantity" :type := "Observation" :v-hash := (codec/quantity nil 140M) - :id := "id-155558" + :did := 153548 :hash-prefix := (hash/prefix hash))) (testing "first ResourceSearchParamValue key is about `value`" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "Observation" - :id := "id-155558" + :did := 153548 :hash-prefix := (hash/prefix hash) :code := "value-quantity" :v-hash := (codec/quantity nil 140M))) @@ -241,13 +243,13 @@ :code := "value-quantity" :type := "Observation" :v-hash := (codec/quantity "mmHg" 140M) - :id := "id-155558" + :did := 153548 :hash-prefix := (hash/prefix hash))) (testing "second ResourceSearchParamValue key is about `unit value`" (given (r-sp-v-tu/decode-key-human (bb/wrap k3)) :type := "Observation" - :id := "id-155558" + :did := 153548 :hash-prefix := (hash/prefix hash) :code := "value-quantity" :v-hash := (codec/quantity "mmHg" 140M))))) @@ -266,20 +268,21 @@ [[_ k0] [_ k1] [_ k2] [_ k3]] (search-param/index-entries (sr/get search-param-registry "value-quantity" "Observation") - [] hash observation)] + (constantly nil) + [] 153606 hash observation)] (testing "first SearchParamValueResource key is about `value`" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "value-quantity" :type := "Observation" :v-hash := (codec/quantity nil 120M) - :id := "id-155558" + :did := 153606 :hash-prefix := (hash/prefix hash))) (testing "first ResourceSearchParamValue key is about `value`" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "Observation" - :id := "id-155558" + :did := 153606 :hash-prefix := (hash/prefix hash) :code := "value-quantity" :v-hash := (codec/quantity nil 120M))) @@ -289,13 +292,13 @@ :code := "value-quantity" :type := "Observation" :v-hash := (codec/quantity "mm[Hg]" 120M) - :id := "id-155558" + :did := 153606 :hash-prefix := (hash/prefix hash))) (testing "second ResourceSearchParamValue key is about `code value`" (given (r-sp-v-tu/decode-key-human (bb/wrap k3)) :type := "Observation" - :id := "id-155558" + :did := 153606 :hash-prefix := (hash/prefix hash) :code := "value-quantity" :v-hash := (codec/quantity "mm[Hg]" 120M))))) @@ -314,20 +317,21 @@ [[_ k0] [_ k1] [_ k2] [_ k3] [_ k4] [_ k5]] (search-param/index-entries (sr/get search-param-registry "value-quantity" "Observation") - [] hash observation)] + (constantly nil) + [] 153622 hash observation)] (testing "first SearchParamValueResource key is about `value`" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "value-quantity" :type := "Observation" :v-hash := (codec/quantity nil 120M) - :id := "id-155558" + :did := 153622 :hash-prefix := (hash/prefix hash))) (testing "first ResourceSearchParamValue key is about `value`" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "Observation" - :id := "id-155558" + :did := 153622 :hash-prefix := (hash/prefix hash) :code := "value-quantity" :v-hash := (codec/quantity nil 120M))) @@ -337,13 +341,13 @@ :code := "value-quantity" :type := "Observation" :v-hash := (codec/quantity "mm[Hg]" 120M) - :id := "id-155558" + :did := 153622 :hash-prefix := (hash/prefix hash))) (testing "second ResourceSearchParamValue key is about `code value`" (given (r-sp-v-tu/decode-key-human (bb/wrap k3)) :type := "Observation" - :id := "id-155558" + :did := 153622 :hash-prefix := (hash/prefix hash) :code := "value-quantity" :v-hash := (codec/quantity "mm[Hg]" 120M))) @@ -353,13 +357,13 @@ :code := "value-quantity" :type := "Observation" :v-hash := (codec/quantity "mmHg" 120M) - :id := "id-155558" + :did := 153622 :hash-prefix := (hash/prefix hash))) (testing "third ResourceSearchParamValue key is about `unit value`" (given (r-sp-v-tu/decode-key-human (bb/wrap k5)) :type := "Observation" - :id := "id-155558" + :did := 153622 :hash-prefix := (hash/prefix hash) :code := "value-quantity" :v-hash := (codec/quantity "mmHg" 120M)))))) @@ -371,7 +375,8 @@ (with-redefs [fhir-path/eval (fn [_ _ _] {::anom/category ::anom/fault})] (given (search-param/index-entries (sr/get search-param-registry "value-quantity" "Observation") - [] hash resource) + (constantly nil) + [] 153644 hash resource) ::anom/category := ::anom/fault)))) (testing "skip warning" diff --git a/modules/db/test/blaze/db/impl/search_param/spec.clj b/modules/db/test/blaze/db/impl/search_param/spec.clj index 6ce8a1dd7..bb3c5adb6 100644 --- a/modules/db/test/blaze/db/impl/search_param/spec.clj +++ b/modules/db/test/blaze/db/impl/search_param/spec.clj @@ -5,4 +5,4 @@ (s/def :blaze.db/compartment - (s/tuple :blaze.db/c-hash :blaze.db/id-byte-string)) + (s/tuple :blaze.db/c-hash :blaze.db/did)) diff --git a/modules/db/test/blaze/db/impl/search_param/string_test.clj b/modules/db/test/blaze/db/impl/search_param/string_test.clj index 8dcae54fc..547743664 100644 --- a/modules/db/test/blaze/db/impl/search_param/string_test.clj +++ b/modules/db/test/blaze/db/impl/search_param/string_test.clj @@ -65,7 +65,9 @@ hash (hash/generate patient)] (is (empty? (search-param/index-entries - (phonetic-param search-param-registry) [] hash + (phonetic-param search-param-registry) + (constantly nil) + [] 153659 hash patient))))) (let [patient {:fhir/type :fhir/Patient @@ -74,20 +76,22 @@ hash (hash/generate patient) [[_ k0] [_ k1]] (search-param/index-entries - (phonetic-param search-param-registry) [] hash patient)] + (phonetic-param search-param-registry) + (constantly nil) + [] 153708 hash patient)] (testing "SearchParamValueResource key" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "phonetic" :type := "Patient" :v-hash := (codec/string (phonetics/soundex "family-102508")) - :id := "id-122929" + :did := 153708 :hash-prefix := (hash/prefix hash))) (testing "ResourceSearchParamValue key" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "Patient" - :id := "id-122929" + :did := 153708 :hash-prefix := (hash/prefix hash) :code := "phonetic" :v-hash := (codec/string (phonetics/soundex "family-102508")))))) @@ -102,7 +106,8 @@ [[_ k0] [_ k1] [_ k2] [_ k3]] (search-param/index-entries (sr/get search-param-registry "address" "Patient") - [] hash patient)] + (constantly nil) + [] 153730 hash patient)] (testing "first entry is about `line`" (testing "SearchParamValueResource key" @@ -110,13 +115,13 @@ :code := "address" :type := "Patient" :v-hash := (codec/string "line 120252") - :id := "id-122929" + :did := 153730 :hash-prefix := (hash/prefix hash))) (testing "ResourceSearchParamValue key" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "Patient" - :id := "id-122929" + :did := 153730 :hash-prefix := (hash/prefix hash) :code := "address" :v-hash := (codec/string "line 120252")))) @@ -127,13 +132,13 @@ :code := "address" :type := "Patient" :v-hash := (codec/string "city 105431") - :id := "id-122929" + :did := 153730 :hash-prefix := (hash/prefix hash))) (testing "ResourceSearchParamValue key" (given (r-sp-v-tu/decode-key-human (bb/wrap k3)) :type := "Patient" - :id := "id-122929" + :did := 153730 :hash-prefix := (hash/prefix hash) :code := "address" :v-hash := (codec/string "city 105431")))))) @@ -146,20 +151,21 @@ [[_ k0] [_ k1]] (search-param/index-entries (sr/get search-param-registry "description" "ActivityDefinition") - [] hash resource)] + (constantly nil) + [] 153757 hash resource)] (testing "SearchParamValueResource key" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "description" :type := "ActivityDefinition" :v-hash := (codec/string "desc 121328") - :id := "id-121344" + :did := 153757 :hash-prefix := (hash/prefix hash))) (testing "ResourceSearchParamValue key" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "ActivityDefinition" - :id := "id-121344" + :did := 153757 :hash-prefix := (hash/prefix hash) :code := "description" :v-hash := (codec/string "desc 121328"))))) @@ -171,7 +177,8 @@ (with-redefs [fhir-path/eval (fn [_ _ _] {::anom/category ::anom/fault})] (given (search-param/index-entries (sr/get search-param-registry "description" "ActivityDefinition") - [] hash resource) + (constantly nil) + [] 153816 hash resource) ::anom/category := ::anom/fault)))) (testing "skip warning" diff --git a/modules/db/test/blaze/db/impl/search_param/token_spec.clj b/modules/db/test/blaze/db/impl/search_param/token_spec.clj index 6048e3e86..20d361e11 100644 --- a/modules/db/test/blaze/db/impl/search_param/token_spec.clj +++ b/modules/db/test/blaze/db/impl/search_param/token_spec.clj @@ -13,4 +13,4 @@ :c-hash :blaze.db/c-hash :tid :blaze.db/tid :value byte-string? - :start-id (s/? :blaze.db/id-byte-string))) + :start-did (s/? :blaze.db/did))) diff --git a/modules/db/test/blaze/db/impl/search_param/token_test.clj b/modules/db/test/blaze/db/impl/search_param/token_test.clj index d8b48137a..45b398f51 100644 --- a/modules/db/test/blaze/db/impl/search_param/token_test.clj +++ b/modules/db/test/blaze/db/impl/search_param/token_test.clj @@ -65,20 +65,21 @@ [[_ k0] [_ k1]] (search-param/index-entries (sr/get search-param-registry "_id" "Observation") - [] hash observation)] + (constantly nil) + [] 153828 hash observation)] (testing "SearchParamValueResource key" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "_id" :type := "Observation" :v-hash := (codec/v-hash "id-161849") - :id := "id-161849" + :did := 153828 :hash-prefix := (hash/prefix hash))) (testing "ResourceSearchParamValue key" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "Observation" - :id := "id-161849" + :did := 153828 :hash-prefix := (hash/prefix hash) :code := "_id" :v-hash := (codec/v-hash "id-161849"))))) @@ -96,20 +97,22 @@ hash (hash/generate observation) [[_ k0] [_ k1] [_ k2] [_ k3] [_ k4] [_ k5]] (search-param/index-entries - (code-param search-param-registry) [] hash observation)] + (code-param search-param-registry) + (constantly nil) + [] 153911 hash observation)] (testing "first SearchParamValueResource key is about `code`" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "code" :type := "Observation" :v-hash := (codec/v-hash "code-171327") - :id := "id-183201" + :did := 153911 :hash-prefix := (hash/prefix hash))) (testing "first ResourceSearchParamValue key is about `code`" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "Observation" - :id := "id-183201" + :did := 153911 :hash-prefix := (hash/prefix hash) :code := "code" :v-hash := (codec/v-hash "code-171327"))) @@ -119,13 +122,13 @@ :code := "code" :type := "Observation" :v-hash := (codec/v-hash "system-171339|") - :id := "id-183201" + :did := 153911 :hash-prefix := (hash/prefix hash))) (testing "second ResourceSearchParamValue key is about `system|`" (given (r-sp-v-tu/decode-key-human (bb/wrap k3)) :type := "Observation" - :id := "id-183201" + :did := 153911 :hash-prefix := (hash/prefix hash) :code := "code" :v-hash := (codec/v-hash "system-171339|"))) @@ -135,13 +138,13 @@ :code := "code" :type := "Observation" :v-hash := (codec/v-hash "system-171339|code-171327") - :id := "id-183201" + :did := 153911 :hash-prefix := (hash/prefix hash))) (testing "third ResourceSearchParamValue key is about `system|code`" (given (r-sp-v-tu/decode-key-human (bb/wrap k5)) :type := "Observation" - :id := "id-183201" + :did := 153911 :hash-prefix := (hash/prefix hash) :code := "code" :v-hash := (codec/v-hash "system-171339|code-171327"))))) @@ -158,20 +161,22 @@ hash (hash/generate observation) [[_ k0] [_ k1] [_ k2] [_ k3]] (search-param/index-entries - (code-param search-param-registry) [] hash observation)] + (code-param search-param-registry) + (constantly nil) + [] 153954 hash observation)] (testing "first SearchParamValueResource key is about `code`" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "code" :type := "Observation" :v-hash := (codec/v-hash "code-134035") - :id := "id-183201" + :did := 153954 :hash-prefix := (hash/prefix hash))) (testing "first ResourceSearchParamValue key is about `code`" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "Observation" - :id := "id-183201" + :did := 153954 :hash-prefix := (hash/prefix hash) :code := "code" :v-hash := (codec/v-hash "code-134035"))) @@ -181,13 +186,13 @@ :code := "code" :type := "Observation" :v-hash := (codec/v-hash "|code-134035") - :id := "id-183201" + :did := 153954 :hash-prefix := (hash/prefix hash))) (testing "second ResourceSearchParamValue key is about `|code`" (given (r-sp-v-tu/decode-key-human (bb/wrap k3)) :type := "Observation" - :id := "id-183201" + :did := 153954 :hash-prefix := (hash/prefix hash) :code := "code" :v-hash := (codec/v-hash "|code-134035"))))) @@ -204,20 +209,22 @@ hash (hash/generate observation) [[_ k0] [_ k1]] (search-param/index-entries - (code-param search-param-registry) [] hash observation)] + (code-param search-param-registry) + (constantly nil) + [] 154050 hash observation)] (testing "first SearchParamValueResource key is about `system|`" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "code" :type := "Observation" :v-hash := (codec/v-hash "system-171339|") - :id := "id-183201" + :did := 154050 :hash-prefix := (hash/prefix hash))) (testing "first ResourceSearchParamValue key is about `system|`" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "Observation" - :id := "id-183201" + :did := 154050 :hash-prefix := (hash/prefix hash) :code := "code" :v-hash := (codec/v-hash "system-171339|"))))) @@ -233,20 +240,21 @@ [[_ k0] [_ k1] [_ k2] [_ k3] [_ k4] [_ k5]] (search-param/index-entries (sr/get search-param-registry "identifier" "Patient") - [] hash patient)] + (constantly nil) + [] 154101 hash patient)] (testing "first SearchParamValueResource key is about `value`" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "identifier" :type := "Patient" :v-hash := (codec/v-hash "value-123005") - :id := "id-122929" + :did := 154101 :hash-prefix := (hash/prefix hash))) (testing "first ResourceSearchParamValue key is about `value`" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "Patient" - :id := "id-122929" + :did := 154101 :hash-prefix := (hash/prefix hash) :code := "identifier" :v-hash := (codec/v-hash "value-123005"))) @@ -256,13 +264,13 @@ :code := "identifier" :type := "Patient" :v-hash := (codec/v-hash "system-123000|") - :id := "id-122929" + :did := 154101 :hash-prefix := (hash/prefix hash))) (testing "second ResourceSearchParamValue key is about `system|`" (given (r-sp-v-tu/decode-key-human (bb/wrap k3)) :type := "Patient" - :id := "id-122929" + :did := 154101 :hash-prefix := (hash/prefix hash) :code := "identifier" :v-hash := (codec/v-hash "system-123000|"))) @@ -272,13 +280,13 @@ :code := "identifier" :type := "Patient" :v-hash := (codec/v-hash "system-123000|value-123005") - :id := "id-122929" + :did := 154101 :hash-prefix := (hash/prefix hash))) (testing "third ResourceSearchParamValue key is about `system|value`" (given (r-sp-v-tu/decode-key-human (bb/wrap k5)) :type := "Patient" - :id := "id-122929" + :did := 154101 :hash-prefix := (hash/prefix hash) :code := "identifier" :v-hash := (codec/v-hash "system-123000|value-123005"))))) @@ -293,20 +301,21 @@ [[_ k0] [_ k1] [_ k2] [_ k3]] (search-param/index-entries (sr/get search-param-registry "identifier" "Patient") - [] hash patient)] + (constantly nil) + [] 154139 hash patient)] (testing "first SearchParamValueResource key is about `value`" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "identifier" :type := "Patient" :v-hash := (codec/v-hash "value-140132") - :id := "id-122929" + :did := 154139 :hash-prefix := (hash/prefix hash))) (testing "first ResourceSearchParamValue key is about `value`" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "Patient" - :id := "id-122929" + :did := 154139 :hash-prefix := (hash/prefix hash) :code := "identifier" :v-hash := (codec/v-hash "value-140132"))) @@ -316,13 +325,13 @@ :code := "identifier" :type := "Patient" :v-hash := (codec/v-hash "|value-140132") - :id := "id-122929" + :did := 154139 :hash-prefix := (hash/prefix hash))) (testing "third ResourceSearchParamValue key is about `|value`" (given (r-sp-v-tu/decode-key-human (bb/wrap k3)) :type := "Patient" - :id := "id-122929" + :did := 154139 :hash-prefix := (hash/prefix hash) :code := "identifier" :v-hash := (codec/v-hash "|value-140132"))))) @@ -337,20 +346,21 @@ [[_ k0] [_ k1]] (search-param/index-entries (sr/get search-param-registry "identifier" "Patient") - [] hash patient)] + (constantly nil) + [] 154210 hash patient)] (testing "second SearchParamValueResource key is about `system|`" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "identifier" :type := "Patient" :v-hash := (codec/v-hash "system-140316|") - :id := "id-122929" + :did := 154210 :hash-prefix := (hash/prefix hash))) (testing "second ResourceSearchParamValue key is about `system|`" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "Patient" - :id := "id-122929" + :did := 154210 :hash-prefix := (hash/prefix hash) :code := "identifier" :v-hash := (codec/v-hash "system-140316|"))))) @@ -362,20 +372,21 @@ [[_ k0] [_ k1]] (search-param/index-entries (sr/get search-param-registry "deceased" "Patient") - [] hash patient)] + (constantly nil) + [] 154245 hash patient)] (testing "SearchParamValueResource key" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "deceased" :type := "Patient" :v-hash := (codec/v-hash "false") - :id := "id-142629" + :did := 154245 :hash-prefix := (hash/prefix hash))) (testing "ResourceSearchParamValue key" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "Patient" - :id := "id-142629" + :did := 154245 :hash-prefix := (hash/prefix hash) :code := "deceased" :v-hash := (codec/v-hash "false"))))) @@ -388,20 +399,21 @@ [[_ k0] [_ k1]] (search-param/index-entries (sr/get search-param-registry "deceased" "Patient") - [] hash patient)] + (constantly nil) + [] 154258 hash patient)] (testing "SearchParamValueResource key" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "deceased" :type := "Patient" :v-hash := (codec/v-hash "true") - :id := "id-142629" + :did := 154258 :hash-prefix := (hash/prefix hash))) (testing "ResourceSearchParamValue key" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "Patient" - :id := "id-142629" + :did := 154258 :hash-prefix := (hash/prefix hash) :code := "deceased" :v-hash := (codec/v-hash "true"))))) @@ -415,20 +427,21 @@ [[_ k0] [_ k1]] (search-param/index-entries (sr/get search-param-registry "deceased" "Patient") - [] hash patient)] + (constantly nil) + [] 154315 hash patient)] (testing "SearchParamValueResource key" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "deceased" :type := "Patient" :v-hash := (codec/v-hash "true") - :id := "id-142629" + :did := 154315 :hash-prefix := (hash/prefix hash))) (testing "ResourceSearchParamValue key" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "Patient" - :id := "id-142629" + :did := 154315 :hash-prefix := (hash/prefix hash) :code := "deceased" :v-hash := (codec/v-hash "true")))))) @@ -448,20 +461,21 @@ [[_ k0] [_ k1] [_ k2] [_ k3] [_ k4] [_ k5]] (search-param/index-entries (sr/get search-param-registry "bodysite" "Specimen") - [] hash specimen)] + (constantly nil) + [] 154350 hash specimen)] (testing "first SearchParamValueResource key is about `code`" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "bodysite" :type := "Specimen" :v-hash := (codec/v-hash "code-103812") - :id := "id-105153" + :did := 154350 :hash-prefix := (hash/prefix hash))) (testing "first ResourceSearchParamValue key is about `code`" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "Specimen" - :id := "id-105153" + :did := 154350 :hash-prefix := (hash/prefix hash) :code := "bodysite" :v-hash := (codec/v-hash "code-103812"))) @@ -471,13 +485,13 @@ :code := "bodysite" :type := "Specimen" :v-hash := (codec/v-hash "system-103824|") - :id := "id-105153" + :did := 154350 :hash-prefix := (hash/prefix hash))) (testing "second ResourceSearchParamValue key is about `system|`" (given (r-sp-v-tu/decode-key-human (bb/wrap k3)) :type := "Specimen" - :id := "id-105153" + :did := 154350 :hash-prefix := (hash/prefix hash) :code := "bodysite" :v-hash := (codec/v-hash "system-103824|"))) @@ -487,13 +501,13 @@ :code := "bodysite" :type := "Specimen" :v-hash := (codec/v-hash "system-103824|code-103812") - :id := "id-105153" + :did := 154350 :hash-prefix := (hash/prefix hash))) (testing "third ResourceSearchParamValue key is about `system|code`" (given (r-sp-v-tu/decode-key-human (bb/wrap k5)) :type := "Specimen" - :id := "id-105153" + :did := 154350 :hash-prefix := (hash/prefix hash) :code := "bodysite" :v-hash := (codec/v-hash "system-103824|code-103812"))))) @@ -509,20 +523,21 @@ [[_ k0] [_ k1] [_ k2] [_ k3] [_ k4] [_ k5]] (search-param/index-entries (sr/get search-param-registry "class" "Encounter") - [] hash specimen)] + (constantly nil) + [] 154421 hash specimen)] (testing "first SearchParamValueResource key is about `code`" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "class" :type := "Encounter" :v-hash := (codec/v-hash "AMB") - :id := "id-105153" + :did := 154421 :hash-prefix := (hash/prefix hash))) (testing "first ResourceSearchParamValue key is about `code`" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "Encounter" - :id := "id-105153" + :did := 154421 :hash-prefix := (hash/prefix hash) :code := "class" :v-hash := (codec/v-hash "AMB"))) @@ -532,13 +547,13 @@ :code := "class" :type := "Encounter" :v-hash := (codec/v-hash "http://terminology.hl7.org/CodeSystem/v3-ActCode|") - :id := "id-105153" + :did := 154421 :hash-prefix := (hash/prefix hash))) (testing "second ResourceSearchParamValue key is about `system|`" (given (r-sp-v-tu/decode-key-human (bb/wrap k3)) :type := "Encounter" - :id := "id-105153" + :did := 154421 :hash-prefix := (hash/prefix hash) :code := "class" :v-hash := (codec/v-hash "http://terminology.hl7.org/CodeSystem/v3-ActCode|"))) @@ -548,13 +563,13 @@ :code := "class" :type := "Encounter" :v-hash := (codec/v-hash "http://terminology.hl7.org/CodeSystem/v3-ActCode|AMB") - :id := "id-105153" + :did := 154421 :hash-prefix := (hash/prefix hash))) (testing "third ResourceSearchParamValue key is about `system|code`" (given (r-sp-v-tu/decode-key-human (bb/wrap k5)) :type := "Encounter" - :id := "id-105153" + :did := 154421 :hash-prefix := (hash/prefix hash) :code := "class" :v-hash := (codec/v-hash "http://terminology.hl7.org/CodeSystem/v3-ActCode|AMB"))))) @@ -569,20 +584,21 @@ [[_ k0] [_ k1]] (search-param/index-entries (sr/get search-param-registry "series" "ImagingStudy") - [] hash specimen)] + (constantly nil) + [] 154455 hash specimen)] (testing "SearchParamValueResource key is about `id`" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "series" :type := "ImagingStudy" :v-hash := (codec/v-hash "1.2.840.99999999.1.59354388.1582528879516") - :id := "id-105153" + :did := 154455 :hash-prefix := (hash/prefix hash))) (testing "ResourceSearchParamValue key" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "ImagingStudy" - :id := "id-105153" + :did := 154455 :hash-prefix := (hash/prefix hash) :code := "series" :v-hash := (codec/v-hash "1.2.840.99999999.1.59354388.1582528879516"))))) @@ -595,20 +611,21 @@ [[_ k0] [_ k1]] (search-param/index-entries (sr/get search-param-registry "version" "CodeSystem") - [] hash resource)] + (constantly nil) + [] 154546 hash resource)] (testing "SearchParamValueResource key" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "version" :type := "CodeSystem" :v-hash := (codec/v-hash "version-122621") - :id := "id-111846" + :did := 154546 :hash-prefix := (hash/prefix hash))) (testing "ResourceSearchParamValue key" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "CodeSystem" - :id := "id-111846" + :did := 154546 :hash-prefix := (hash/prefix hash) :code := "version" :v-hash := (codec/v-hash "version-122621"))))) @@ -620,11 +637,12 @@ (with-redefs [fhir-path/eval (fn [_ _ _] {::anom/category ::anom/fault})] (given (search-param/index-entries (sr/get search-param-registry "_id" "Patient") - [] hash resource) + (constantly nil) + [] 154605 hash resource) ::anom/category := ::anom/fault)))) (testing "skip warning" - (is (nil? (spt/index-entries "" nil)))))) + (is (nil? (spt/index-entries (constantly nil) "" nil)))))) (defn subject-param [search-param-registry] diff --git a/modules/db/test/blaze/db/impl/search_param/util_spec.clj b/modules/db/test/blaze/db/impl/search_param/util_spec.clj index ce83f70e0..ecd2cae0a 100644 --- a/modules/db/test/blaze/db/impl/search_param/util_spec.clj +++ b/modules/db/test/blaze/db/impl/search_param/util_spec.clj @@ -8,6 +8,7 @@ [blaze.db.impl.search-param.util :as u] [blaze.db.kv.spec] [blaze.db.spec] + [blaze.fhir.spec.spec] [clojure.spec.alpha :as s])) @@ -16,6 +17,11 @@ :ret (s/tuple keyword? string?)) +(s/fdef u/non-deleted-resource-handle + :args (s/cat :resource-handle fn? :tid :blaze.db/tid :did :blaze.db/did) + :ret (s/nilable :blaze.db/resource-handle)) + + (s/fdef u/resource-handle-mapper :args (s/cat :context :blaze.db.impl.batch-db/context :tid :blaze.db/tid)) diff --git a/modules/db/test/blaze/db/impl/search_param_spec.clj b/modules/db/test/blaze/db/impl/search_param_spec.clj index bd6dabf27..fed16f503 100644 --- a/modules/db/test/blaze/db/impl/search_param_spec.clj +++ b/modules/db/test/blaze/db/impl/search_param_spec.clj @@ -39,7 +39,7 @@ :tid :blaze.db/tid :modifier (s/nilable :blaze.db.search-param/modifier) :values (s/coll-of some? :min-count 1) - :start-id (s/? :blaze.db/id-byte-string)) + :start-did (s/? :blaze.db/did)) :ret (s/coll-of :blaze.db/resource-handle :kind sequential?)) @@ -69,7 +69,9 @@ (s/fdef search-param/index-entries :args (s/cat :search-param :blaze.db/search-param - :linked-compartments (s/nilable (s/coll-of (s/tuple string? string?))) + :resource-id fn? + :linked-compartments (s/nilable (s/coll-of :blaze.db/compartment)) + :did :blaze.db/did :hash :blaze.resource/hash :resource :blaze/resource) :ret (s/or :entries (s/coll-of :blaze.db.kv/put-entry-w-cf :kind sequential?) diff --git a/modules/db/test/blaze/db/impl/search_param_test.clj b/modules/db/test/blaze/db/impl/search_param_test.clj index 19d0082ee..ad27b3ece 100644 --- a/modules/db/test/blaze/db/impl/search_param_test.clj +++ b/modules/db/test/blaze/db/impl/search_param_test.clj @@ -73,20 +73,21 @@ [[_ k0] [_ k1]] (search-param/index-entries (sr/get search-param-registry "_profile" "Patient") - [] hash patient)] + (constantly nil) + [] 201411 hash patient)] (testing "SearchParamValueResource key" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "_profile" :type := "Patient" :v-hash := (codec/v-hash "profile-uri-141443") - :id := "id-140855" + :did := 201411 :hash-prefix := (hash/prefix hash))) (testing "ResourceSearchParamValue key" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "Patient" - :id := "id-140855" + :did := 201411 :hash-prefix := (hash/prefix hash) :code := "_profile" :v-hash := (codec/v-hash "profile-uri-141443"))))) @@ -99,7 +100,8 @@ (empty? (search-param/index-entries (sr/get search-param-registry "patient" "Specimen") - [] hash specimen))))) + (constantly nil) + [] 201646 hash specimen))))) (testing "ActivityDefinition url" (let [resource {:fhir/type :fhir/ActivityDefinition @@ -109,20 +111,21 @@ [[_ k0] [_ k1]] (search-param/index-entries (sr/get search-param-registry "url" "ActivityDefinition") - [] hash resource)] + (constantly nil) + [] 201658 hash resource)] (testing "SearchParamValueResource key" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "url" :type := "ActivityDefinition" :v-hash := (codec/v-hash "url-111854") - :id := "id-111846" + :did := 201658 :hash-prefix := (hash/prefix hash))) (testing "ResourceSearchParamValue key" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "ActivityDefinition" - :id := "id-111846" + :did := 201658 :hash-prefix := (hash/prefix hash) :code := "url" :v-hash := (codec/v-hash "url-111854"))))) @@ -137,20 +140,23 @@ [[_ k0] [_ k1] [_ k2] [_ k3] [_ k4] [_ k5]] (search-param/index-entries (sr/get search-param-registry "item" "List") - [] hash resource)] + (fn [tid id] + (when (and (= (codec/tid "Patient") tid) (= "0" id)) + 181705)) + [] 201714 hash resource)] (testing "first SearchParamValueResource key is about `id`" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "item" :type := "List" :v-hash := (codec/v-hash "0") - :id := "id-121825" + :did := 201714 :hash-prefix := (hash/prefix hash))) (testing "first ResourceSearchParamValue key is about `id`" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "List" - :id := "id-121825" + :did := 201714 :hash-prefix := (hash/prefix hash) :code := "item" :v-hash := (codec/v-hash "0"))) @@ -160,34 +166,32 @@ :code := "item" :type := "List" :v-hash := (codec/v-hash "Patient/0") - :id := "id-121825" + :did := 201714 :hash-prefix := (hash/prefix hash))) (testing "second ResourceSearchParamValue key is about `type/id`" (given (r-sp-v-tu/decode-key-human (bb/wrap k3)) :type := "List" - :id := "id-121825" + :did := 201714 :hash-prefix := (hash/prefix hash) :code := "item" :v-hash := (codec/v-hash "Patient/0"))) - (testing "third SearchParamValueResource key is about `tid` and `id`" + (testing "third SearchParamValueResource key is about `tid` and `did`" (given (sp-vr-tu/decode-key-human (bb/wrap k4)) :code := "item" :type := "List" - :v-hash := (codec/tid-id (codec/tid "Patient") - (codec/id-byte-string "0")) - :id := "id-121825" + :v-hash := (codec/tid-did (codec/tid "Patient") 181705) + :did := 201714 :hash-prefix := (hash/prefix hash))) - (testing "third ResourceSearchParamValue key is about `tid` and `id`" + (testing "third ResourceSearchParamValue key is about `tid` and `did`" (given (r-sp-v-tu/decode-key-human (bb/wrap k5)) :type := "List" - :id := "id-121825" + :did := 201714 :hash-prefix := (hash/prefix hash) :code := "item" - :v-hash := (codec/tid-id (codec/tid "Patient") - (codec/id-byte-string "0")))))) + :v-hash := (codec/tid-did (codec/tid "Patient") 181705))))) (testing "with identifier reference" (let [resource {:fhir/type :fhir/List :id "id-123058" @@ -203,20 +207,21 @@ [[_ k0] [_ k1] [_ k2] [_ k3] [_ k4] [_ k5]] (search-param/index-entries (sr/get search-param-registry "item" "List") - [] hash resource)] + (constantly nil) + [] 201800 hash resource)] (testing "first SearchParamValueResource key is about `value`" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "item:identifier" :type := "List" :v-hash := (codec/v-hash "value-122931") - :id := "id-123058" + :did := 201800 :hash-prefix := (hash/prefix hash))) (testing "first ResourceSearchParamValue key is about `value`" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "List" - :id := "id-123058" + :did := 201800 :hash-prefix := (hash/prefix hash) :code := "item:identifier" :v-hash := (codec/v-hash "value-122931"))) @@ -226,13 +231,13 @@ :code := "item:identifier" :type := "List" :v-hash := (codec/v-hash "system-122917|") - :id := "id-123058" + :did := 201800 :hash-prefix := (hash/prefix hash))) (testing "second ResourceSearchParamValue key is about `system|`" (given (r-sp-v-tu/decode-key-human (bb/wrap k3)) :type := "List" - :id := "id-123058" + :did := 201800 :hash-prefix := (hash/prefix hash) :code := "item:identifier" :v-hash := (codec/v-hash "system-122917|"))) @@ -242,13 +247,13 @@ :code := "item:identifier" :type := "List" :v-hash := (codec/v-hash "system-122917|value-122931") - :id := "id-123058" + :did := 201800 :hash-prefix := (hash/prefix hash))) (testing "third ResourceSearchParamValue key is about `system|value`" (given (r-sp-v-tu/decode-key-human (bb/wrap k5)) :type := "List" - :id := "id-123058" + :did := 201800 :hash-prefix := (hash/prefix hash) :code := "item:identifier" :v-hash := (codec/v-hash "system-122917|value-122931"))))) @@ -264,20 +269,21 @@ [[_ k0] [_ k1]] (search-param/index-entries (sr/get search-param-registry "item" "List") - [] hash resource)] + (constantly nil) + [] 201820 hash resource)] (testing "first SearchParamValueResource key is about `id`" (given (sp-vr-tu/decode-key-human (bb/wrap k0)) :code := "item" :type := "List" :v-hash := (codec/v-hash "http://foo.com/bar-141221") - :id := "id-121825" + :did := 201820 :hash-prefix := (hash/prefix hash))) (testing "first ResourceSearchParamValue key is about `id`" (given (r-sp-v-tu/decode-key-human (bb/wrap k1)) :type := "List" - :id := "id-121825" + :did := 201820 :hash-prefix := (hash/prefix hash) :code := "item" :v-hash := (codec/v-hash "http://foo.com/bar-141221")))))))) diff --git a/modules/db/test/blaze/db/node/resource_indexer_test.clj b/modules/db/test/blaze/db/node/resource_indexer_test.clj index b2a4b3daf..6247345e2 100644 --- a/modules/db/test/blaze/db/node/resource_indexer_test.clj +++ b/modules/db/test/blaze/db/node/resource_indexer_test.clj @@ -6,6 +6,7 @@ [blaze.db.impl.index.compartment.resource-test-util :as cr-tu] [blaze.db.impl.index.compartment.search-param-value-resource-test-util :as c-sp-vr-tu] + [blaze.db.impl.index.resource-id :as ri] [blaze.db.impl.index.resource-search-param-value-test-util :as r-sp-v-tu] [blaze.db.impl.index.search-param-value-resource-test-util :as sp-vr-tu] [blaze.db.kv :as kv] @@ -139,7 +140,8 @@ :resource-value-index nil :compartment-search-param-value-index nil :compartment-resource-type-index nil - :active-search-params nil}} + :active-search-params nil + :resource-id-index nil}} ::rs/kv {:kv-store (ig/ref :blaze.db/resource-kv-store) @@ -178,6 +180,7 @@ [{:op "put" :type "Patient" :id "0" + :did 193734 :hash hash}] :local-payload {hash patient}}) @@ -201,6 +204,7 @@ [{:op "put" :type "Observation" :id "0" + :did 193724 :hash hash}] :local-payload {hash observation}})) @@ -212,6 +216,7 @@ (with-system [{kv-store [::kv/mem :blaze.db/index-kv-store] resource-store ::rs/kv :blaze.db.node/keys [resource-indexer]} system] + (kv/put! kv-store [(ri/index-entry (codec/tid "Patient") "id-145552" 155044)]) (let [resource {:fhir/type :fhir/Condition :id "id-204446" :code @@ -238,19 +243,17 @@ [{:op "put" :type "Condition" :id "id-204446" + :did 142201 :hash hash}]}) (testing "SearchParamValueResource index" - (is (every? #{["Condition" "id-204446" #blaze/hash-prefix"4AB29C7B"]} + (is (every? #{["Condition" 142201 #blaze/hash-prefix"4AB29C7B"]} (sp-vr-tu/decode-index-entries - kv-store :type :id :hash-prefix))) + kv-store :type :did :hash-prefix))) (is (= (sp-vr-tu/decode-index-entries kv-store :code :v-hash) [["patient" (codec/v-hash "Patient/id-145552")] ["patient" (codec/v-hash "id-145552")] - ["patient" (codec/tid-id - (codec/tid "Patient") - (codec/id-byte-string "id-145552"))] - + ["patient" (codec/tid-did (codec/tid "Patient") 155044)] ["code" (codec/v-hash "system-204435|code-204441")] ["code" (codec/v-hash "system-204435|")] ["code" (codec/v-hash "code-204441")] @@ -263,23 +266,19 @@ (LocalDate/of 2020 1 30)))] ["subject" (codec/v-hash "Patient/id-145552")] ["subject" (codec/v-hash "id-145552")] - ["subject" (codec/tid-id - (codec/tid "Patient") - (codec/id-byte-string "id-145552"))] + ["subject" (codec/tid-did (codec/tid "Patient") 155044)] ["_profile" (codec/v-hash "url-164445")] ["_id" (codec/v-hash "id-204446")] ["_lastUpdated" #blaze/byte-string"80008001"]]))) (testing "ResourceSearchParamValue index" - (is (every? #{["Condition" "id-204446" #blaze/hash-prefix"4AB29C7B"]} + (is (every? #{["Condition" 142201 #blaze/hash-prefix"4AB29C7B"]} (r-sp-v-tu/decode-index-entries - kv-store :type :id :hash-prefix))) + kv-store :type :did :hash-prefix))) (is (= (r-sp-v-tu/decode-index-entries kv-store :code :v-hash) [["patient" (codec/v-hash "Patient/id-145552")] ["patient" (codec/v-hash "id-145552")] - ["patient" (codec/tid-id - (codec/tid "Patient") - (codec/id-byte-string "id-145552"))] + ["patient" (codec/tid-did (codec/tid "Patient") 155044)] ["code" (codec/v-hash "system-204435|code-204441")] ["code" (codec/v-hash "system-204435|")] ["code" (codec/v-hash "code-204441")] @@ -292,28 +291,24 @@ (LocalDate/of 2020 1 30)))] ["subject" (codec/v-hash "Patient/id-145552")] ["subject" (codec/v-hash "id-145552")] - ["subject" (codec/tid-id - (codec/tid "Patient") - (codec/id-byte-string "id-145552"))] + ["subject" (codec/tid-did (codec/tid "Patient") 155044)] ["_profile" (codec/v-hash "url-164445")] ["_id" (codec/v-hash "id-204446")] ["_lastUpdated" #blaze/byte-string"80008001"]]))) (testing "CompartmentResource index" - (is (= (cr-tu/decode-index-entries kv-store :compartment :type :id) - [[["Patient" "id-145552"] "Condition" "id-204446"]]))) + (is (= (cr-tu/decode-index-entries kv-store :compartment :type :did) + [[["Patient" 155044] "Condition" 142201]]))) (testing "CompartmentSearchParamValueResource index" - (is (every? #{[["Patient" "id-145552"] "Condition" "id-204446" + (is (every? #{[["Patient" 155044] "Condition" 142201 #blaze/hash-prefix"4AB29C7B"]} (c-sp-vr-tu/decode-index-entries - kv-store :compartment :type :id :hash-prefix))) + kv-store :compartment :type :did :hash-prefix))) (is (= (c-sp-vr-tu/decode-index-entries kv-store :code :v-hash) [["patient" (codec/v-hash "Patient/id-145552")] ["patient" (codec/v-hash "id-145552")] - ["patient" (codec/tid-id - (codec/tid "Patient") - (codec/id-byte-string "id-145552"))] + ["patient" (codec/tid-did (codec/tid "Patient") 155044)] ["code" (codec/v-hash "system-204435|code-204441")] ["code" (codec/v-hash "system-204435|")] ["code" (codec/v-hash "code-204441")] @@ -326,9 +321,7 @@ (LocalDate/of 2020 1 30)))] ["subject" (codec/v-hash "Patient/id-145552")] ["subject" (codec/v-hash "id-145552")] - ["subject" (codec/tid-id - (codec/tid "Patient") - (codec/id-byte-string "id-145552"))] + ["subject" (codec/tid-did (codec/tid "Patient") 155044)] ["_profile" (codec/v-hash "url-164445")] ["_id" (codec/v-hash "id-204446")] ["_lastUpdated" #blaze/byte-string"80008001"]])))))) @@ -338,6 +331,7 @@ (with-system [{kv-store [::kv/mem :blaze.db/index-kv-store] resource-store ::rs/kv :blaze.db.node/keys [resource-indexer]} system] + (kv/put! kv-store [(ri/index-entry (codec/tid "Patient") "id-180857" 174950)]) (let [resource {:fhir/type :fhir/Observation :id "id-192702" :status #fhir/code"status-193613" :category @@ -371,12 +365,13 @@ [{:op "put" :type "Observation" :id "id-192702" + :did 193644 :hash hash}]}) (testing "SearchParamValueResource index" - (is (every? #{["Observation" "id-192702" #blaze/hash-prefix"651D1F37"]} + (is (every? #{["Observation" 193644 #blaze/hash-prefix"651D1F37"]} (sp-vr-tu/decode-index-entries - kv-store :type :id :hash-prefix))) + kv-store :type :did :hash-prefix))) (is (= (sp-vr-tu/decode-index-entries kv-store :code :v-hash) [["code-value-quantity" (bs/concat (codec/v-hash "code-193824") @@ -411,9 +406,7 @@ ["category" (codec/v-hash "code-193603")] ["category" (codec/v-hash "system-193558|code-193603")] ["patient" (codec/v-hash "id-180857")] - ["patient" (codec/tid-id - (codec/tid "Patient") - (codec/id-byte-string "id-180857"))] + ["patient" (codec/tid-did (codec/tid "Patient") 174950)] ["patient" (codec/v-hash "Patient/id-180857")] ["code" (codec/v-hash "code-193824")] ["code" (codec/v-hash "system-193821|")] @@ -451,9 +444,7 @@ ["combo-code-value-quantity" #blaze/byte-string"D47C56F6D0C25BA3F35972C2DDEDDFE6900926"] ["subject" (codec/v-hash "id-180857")] - ["subject" (codec/tid-id - (codec/tid "Patient") - (codec/id-byte-string "id-180857"))] + ["subject" (codec/tid-did (codec/tid "Patient") 174950)] ["subject" (codec/v-hash "Patient/id-180857")] ["status" (codec/v-hash "status-193613")] ["_id" (codec/v-hash "id-192702")] @@ -470,6 +461,7 @@ :tx-cmds [{:op "delete" :type "Patient" + :did 193708 :id "0"}]}) (testing "doesn't index anything" diff --git a/modules/db/test/blaze/db/node/tx_indexer/verify_spec.clj b/modules/db/test/blaze/db/node/tx_indexer/verify_spec.clj index 6e1bc81f4..fc16aa2db 100644 --- a/modules/db/test/blaze/db/node/tx_indexer/verify_spec.clj +++ b/modules/db/test/blaze/db/node/tx_indexer/verify_spec.clj @@ -3,6 +3,7 @@ [blaze.byte-string-spec] [blaze.db.impl.index-spec] [blaze.db.impl.index.resource-as-of-spec] + [blaze.db.impl.index.resource-id-spec] [blaze.db.impl.index.rts-as-of-spec] [blaze.db.impl.index.system-stats-spec] [blaze.db.impl.index.type-stats-spec] @@ -16,5 +17,5 @@ (s/fdef verify/verify-tx-cmds :args (s/cat :db-before :blaze.db/db :t :blaze.db/t :cmds :blaze.db/tx-cmds) - :ret (s/or :entries (s/coll-of :blaze.db.kv/put-entry) + :ret (s/or :res (s/tuple (s/coll-of :blaze.db.kv/put-entry) :blaze.db/tx-cmds) :anomaly ::anom/anomaly)) diff --git a/modules/db/test/blaze/db/node/tx_indexer/verify_test.clj b/modules/db/test/blaze/db/node/tx_indexer/verify_test.clj index 3cdb53391..75b62c55f 100644 --- a/modules/db/test/blaze/db/node/tx_indexer/verify_test.clj +++ b/modules/db/test/blaze/db/node/tx_indexer/verify_test.clj @@ -1,18 +1,12 @@ (ns blaze.db.node.tx-indexer.verify-test (:require - [blaze.byte-string :as bs] [blaze.db.api :as d] [blaze.db.impl.codec :as codec] - [blaze.db.impl.index.resource-as-of :as rao] [blaze.db.impl.index.resource-as-of-test-util :as rao-tu] - [blaze.db.impl.index.rts-as-of :as rts] - [blaze.db.impl.index.system-as-of :as sao] + [blaze.db.impl.index.resource-id-test-util :as ri-tu] [blaze.db.impl.index.system-as-of-test-util :as sao-tu] - [blaze.db.impl.index.system-stats :as system-stats] [blaze.db.impl.index.system-stats-test-util :as ss-tu] - [blaze.db.impl.index.type-as-of :as tao] [blaze.db.impl.index.type-as-of-test-util :as tao-tu] - [blaze.db.impl.index.type-stats :as type-stats] [blaze.db.impl.index.type-stats-test-util :as ts-tu] [blaze.db.kv.mem] [blaze.db.kv.mem-spec] @@ -29,16 +23,16 @@ [blaze.fhir.spec.type] [blaze.fhir.structure-definition-repo] [blaze.log] - [blaze.test-util :refer [with-system]] + [blaze.test-util :as tu :refer [with-system]] [clojure.spec.test.alpha :as st] - [clojure.test :as test :refer [deftest is testing]] - [clojure.walk :as walk] + [clojure.test :as test :refer [deftest testing]] [cognitect.anomalies :as anom] [juxt.iota :refer [given]] [taoensso.timbre :as log])) (st/instrument) +(tu/init-fhir-specs) (log/set-level! :trace) @@ -51,195 +45,243 @@ (test/use-fixtures :each fixture) -(def tid-patient (codec/tid "Patient")) +(def ^:private patient-0 + {:fhir/type :fhir/Patient :id "0"}) -(def patient-0 {:fhir/type :fhir/Patient :id "0"}) -(def patient-0-v2 {:fhir/type :fhir/Patient :id "0" :gender #fhir/code"male"}) -(def patient-1 {:fhir/type :fhir/Patient :id "1"}) -(def patient-2 {:fhir/type :fhir/Patient :id "2"}) -(def patient-3 {:fhir/type :fhir/Patient :id "3" - :identifier [#fhir/Identifier{:value "120426"}]}) +(def ^:private patient-1 + {:fhir/type :fhir/Patient :id "1"}) -(defn bytes->vec [x] - (if (bytes? x) (vec x) x)) +(deftest verify-tx-cmds-test + (testing "adding one patient to an empty store" + (let [did (codec/did 1 0) + hash (hash/generate patient-0)] + (doseq [op [:create :put]] + (with-system [{:blaze.db/keys [node]} system] + (given (verify/verify-tx-cmds + (d/db node) 1 + [{:op (name op) :type "Patient" :id "0" :hash hash}]) + [0 0 0] := :resource-as-of-index + [0 0 1 rao-tu/decode-key] := {:type "Patient" :did did :t 1} + [0 0 2 rao-tu/decode-val] := {:hash hash :num-changes 1 :op op :id "0"} + + [0 1 0] := :type-as-of-index + [0 1 1 tao-tu/decode-key] := {:type "Patient" :t 1 :did did} + [0 1 2 tao-tu/decode-val] := {:hash hash :num-changes 1 :op op :id "0"} + + [0 2 0] := :system-as-of-index + [0 2 1 sao-tu/decode-key] := {:t 1 :type "Patient" :did did} + [0 2 2 sao-tu/decode-val] := {:hash hash :num-changes 1 :op op :id "0"} + + [0 3 0] := :resource-id-index + [0 3 1 ri-tu/decode-key] := {:type "Patient" :id "0"} + [0 3 2 ri-tu/decode-val] := {:did did} + + [0 4 0] := :type-stats-index + [0 4 1 ts-tu/decode-key] := {:type "Patient" :t 1} + [0 4 2 ts-tu/decode-val] := {:total 1 :num-changes 1} + + [0 5 0] := :system-stats-index + [0 5 1 ss-tu/decode-key] := {:t 1} + [0 5 2 ss-tu/decode-val] := {:total 1 :num-changes 1} + + [1 0 :did] := did))))) -(defmacro is-entries= [a b] - `(is (= (walk/postwalk bytes->vec ~a) (walk/postwalk bytes->vec ~b)))) + (testing "adding a second version of a patient to a store containing it already" + (let [did (codec/did 1 0) + hash (hash/generate patient-0)] + (with-system [{:blaze.db/keys [node]} system] + @(d/transact node [[:put {:fhir/type :fhir/Patient :id "0"}]]) + (given (verify/verify-tx-cmds + (d/db node) 2 + [{:op "put" :type "Patient" :id "0" :hash hash}]) -(def ^:private deleted-hash - "The hash of a deleted version of a resource." - (bs/from-byte-array (byte-array 32))) + [0 0 0] := :resource-as-of-index + [0 0 1 rao-tu/decode-key] := {:type "Patient" :did did :t 2} + [0 0 2 rao-tu/decode-val] := {:hash hash :num-changes 2 :op :put :id "0"} + [0 1 0] := :type-as-of-index + [0 1 1 tao-tu/decode-key] := {:type "Patient" :t 2 :did did} + [0 1 2 tao-tu/decode-val] := {:hash hash :num-changes 2 :op :put :id "0"} -(deftest verify-tx-cmds-test - (testing "adding one patient to an empty store" - (with-system [{:blaze.db/keys [node]} system] - (is-entries= - (verify/verify-tx-cmds - (d/db node) 1 - [{:op "put" :type "Patient" :id "0" :hash (hash/generate patient-0)}]) - (let [value (rts/encode-value (hash/generate patient-0) 1 :put)] - [[:resource-as-of-index - (rao/encode-key tid-patient (codec/id-byte-string "0") 1) - value] - [:type-as-of-index - (tao/encode-key tid-patient 1 (codec/id-byte-string "0")) - value] - [:system-as-of-index - (sao/encode-key 1 tid-patient (codec/id-byte-string "0")) - value] - (type-stats/index-entry tid-patient 1 {:total 1 :num-changes 1}) - (system-stats/index-entry 1 {:total 1 :num-changes 1})])))) + [0 2 0] := :system-as-of-index + [0 2 1 sao-tu/decode-key] := {:t 2 :type "Patient" :did did} + [0 2 2 sao-tu/decode-val] := {:hash hash :num-changes 2 :op :put :id "0"} - (testing "adding a second version of a patient to a store containing it already" - (with-system [{:blaze.db/keys [node]} system] - @(d/transact node [[:put {:fhir/type :fhir/Patient :id "0"}]]) + [0 3 0] := :type-stats-index + [0 3 1 ts-tu/decode-key] := {:type "Patient" :t 2} + [0 3 2 ts-tu/decode-val] := {:total 1 :num-changes 2} - (is-entries= - (verify/verify-tx-cmds - (d/db node) 2 - [{:op "put" :type "Patient" :id "0" :hash (hash/generate patient-0-v2)}]) - (let [value (rts/encode-value (hash/generate patient-0-v2) 2 :put)] - [[:resource-as-of-index - (rao/encode-key tid-patient (codec/id-byte-string "0") 2) - value] - [:type-as-of-index - (tao/encode-key tid-patient 2 (codec/id-byte-string "0")) - value] - [:system-as-of-index - (sao/encode-key 2 tid-patient (codec/id-byte-string "0")) - value] - (type-stats/index-entry tid-patient 2 {:total 1 :num-changes 2}) - (system-stats/index-entry 2 {:total 1 :num-changes 2})])))) - - (testing "adding a second version of a patient to a store containing it already incl. matcher" - (with-system [{:blaze.db/keys [node]} system] - @(d/transact node [[:put {:fhir/type :fhir/Patient :id "0"}]]) + [0 4 0] := :system-stats-index + [0 4 1 ss-tu/decode-key] := {:t 2} + [0 4 2 ss-tu/decode-val] := {:total 1 :num-changes 2} - (is-entries= - (verify/verify-tx-cmds - (d/db node) 2 - [{:op "put" :type "Patient" :id "0" :hash (hash/generate patient-0-v2) - :if-match 1}]) - (let [value (rts/encode-value (hash/generate patient-0-v2) 2 :put)] - [[:resource-as-of-index - (rao/encode-key tid-patient (codec/id-byte-string "0") 2) - value] - [:type-as-of-index - (tao/encode-key tid-patient 2 (codec/id-byte-string "0")) - value] - [:system-as-of-index - (sao/encode-key 2 tid-patient (codec/id-byte-string "0")) - value] - (type-stats/index-entry tid-patient 2 {:total 1 :num-changes 2}) - (system-stats/index-entry 2 {:total 1 :num-changes 2})])))) + [1 0 :did] := did))) + + (testing "incl. matcher" + (let [did (codec/did 1 0) + hash (hash/generate patient-0)] + (with-system [{:blaze.db/keys [node]} system] + @(d/transact node [[:put {:fhir/type :fhir/Patient :id "0"}]]) + + (given (verify/verify-tx-cmds + (d/db node) 2 + [{:op "put" :type "Patient" :id "0" :hash hash :if-match 1}]) + + [0 0 0] := :resource-as-of-index + [0 0 1 rao-tu/decode-key] := {:type "Patient" :did did :t 2} + [0 0 2 rao-tu/decode-val] := {:hash hash :num-changes 2 :op :put :id "0"} + + [0 1 0] := :type-as-of-index + [0 1 1 tao-tu/decode-key] := {:type "Patient" :t 2 :did did} + [0 1 2 tao-tu/decode-val] := {:hash hash :num-changes 2 :op :put :id "0"} + + [0 2 0] := :system-as-of-index + [0 2 1 sao-tu/decode-key] := {:t 2 :type "Patient" :did did} + [0 2 2 sao-tu/decode-val] := {:hash hash :num-changes 2 :op :put :id "0"} + + [0 3 0] := :type-stats-index + [0 3 1 ts-tu/decode-key] := {:type "Patient" :t 2} + [0 3 2 ts-tu/decode-val] := {:total 1 :num-changes 2} + + [0 4 0] := :system-stats-index + [0 4 1 ss-tu/decode-key] := {:t 2} + [0 4 2 ss-tu/decode-val] := {:total 1 :num-changes 2} + + [1 0 :did] := did))))) (testing "deleting a patient from an empty store" - (with-system [{:blaze.db/keys [node]} system] - (given (verify/verify-tx-cmds - (d/db node) 1 - [{:op "delete" :type "Patient" :id "0"}]) - [0 #(drop 1 %) rao-tu/decode-index-entry] := - [{:type "Patient" :id "0" :t 1} - {:hash deleted-hash :num-changes 1 :op :delete}] + (let [did (codec/did 1 0)] + (with-system [{:blaze.db/keys [node]} system] + (given (verify/verify-tx-cmds + (d/db node) 1 + [{:op "delete" :type "Patient" :id "0"}]) + + [0 0 0] := :resource-as-of-index + [0 0 1 rao-tu/decode-key] := {:type "Patient" :did did :t 1} + [0 0 2 rao-tu/decode-val] := {:hash hash/deleted-hash :num-changes 1 :op :delete :id "0"} + + [0 1 0] := :type-as-of-index + [0 1 1 tao-tu/decode-key] := {:type "Patient" :t 1 :did did} + [0 1 2 tao-tu/decode-val] := {:hash hash/deleted-hash :num-changes 1 :op :delete :id "0"} - [1 #(drop 1 %) tao-tu/decode-index-entry] := - [{:type "Patient" :t 1 :id "0"} - {:hash deleted-hash :num-changes 1 :op :delete}] + [0 2 0] := :system-as-of-index + [0 2 1 sao-tu/decode-key] := {:t 1 :type "Patient" :did did} + [0 2 2 sao-tu/decode-val] := {:hash hash/deleted-hash :num-changes 1 :op :delete :id "0"} - [2 #(drop 1 %) sao-tu/decode-index-entry] := - [{:t 1 :type "Patient" :id "0"} - {:hash deleted-hash :num-changes 1 :op :delete}] + [0 3 0] := :resource-id-index + [0 3 1 ri-tu/decode-key] := {:type "Patient" :id "0"} + [0 3 2 ri-tu/decode-val] := {:did did} - [3 #(drop 1 %) ts-tu/decode-index-entry] := - [{:type "Patient" :t 1} - {:total 0 :num-changes 1}] + [0 4 0] := :type-stats-index + [0 4 1 ts-tu/decode-key] := {:type "Patient" :t 1} + [0 4 2 ts-tu/decode-val] := {:total 0 :num-changes 1} - [4 #(drop 1 %) ss-tu/decode-index-entry] := - [{:t 1} - {:total 0 :num-changes 1}]))) + [0 5 0] := :system-stats-index + [0 5 1 ss-tu/decode-key] := {:t 1} + [0 5 2 ss-tu/decode-val] := {:total 0 :num-changes 1} + + [1] :? empty?)))) (testing "deleting an already deleted patient" - (with-system [{:blaze.db/keys [node]} system] - @(d/transact node [[:delete "Patient" "0"]]) + (let [did (codec/did 1 0)] + (with-system [{:blaze.db/keys [node]} system] + @(d/transact node [[:delete "Patient" "0"]]) - (given - (verify/verify-tx-cmds - (d/db node) 2 - [{:op "delete" :type "Patient" :id "0"}]) + (given (verify/verify-tx-cmds + (d/db node) 2 + [{:op "delete" :type "Patient" :id "0"}]) - [0 #(drop 1 %) rao-tu/decode-index-entry] := - [{:type "Patient" :id "0" :t 2} - {:hash deleted-hash :num-changes 2 :op :delete}] + [0 0 0] := :resource-as-of-index + [0 0 1 rao-tu/decode-key] := {:type "Patient" :did did :t 2} + [0 0 2 rao-tu/decode-val] := {:hash hash/deleted-hash :num-changes 2 :op :delete :id "0"} - [1 #(drop 1 %) tao-tu/decode-index-entry] := - [{:type "Patient" :t 2 :id "0"} - {:hash deleted-hash :num-changes 2 :op :delete}] + [0 1 0] := :type-as-of-index + [0 1 1 tao-tu/decode-key] := {:type "Patient" :t 2 :did did} + [0 1 2 tao-tu/decode-val] := {:hash hash/deleted-hash :num-changes 2 :op :delete :id "0"} - [2 #(drop 1 %) sao-tu/decode-index-entry] := - [{:t 2 :type "Patient" :id "0"} - {:hash deleted-hash :num-changes 2 :op :delete}] + [0 2 0] := :system-as-of-index + [0 2 1 sao-tu/decode-key] := {:t 2 :type "Patient" :did did} + [0 2 2 sao-tu/decode-val] := {:hash hash/deleted-hash :num-changes 2 :op :delete :id "0"} - [3 #(drop 1 %) ts-tu/decode-index-entry] := - [{:type "Patient" :t 2} - {:total 0 :num-changes 2}] + [0 3 0] := :type-stats-index + [0 3 1 ts-tu/decode-key] := {:type "Patient" :t 2} + [0 3 2 ts-tu/decode-val] := {:total 0 :num-changes 2} - [4 #(drop 1 %) ss-tu/decode-index-entry] := - [{:t 2} - {:total 0 :num-changes 2}]))) + [0 4 0] := :system-stats-index + [0 4 1 ss-tu/decode-key] := {:t 2} + [0 4 2 ss-tu/decode-val] := {:total 0 :num-changes 2} + + [1] :? empty?)))) (testing "deleting an existing patient" - (with-system [{:blaze.db/keys [node]} system] - @(d/transact node [[:put {:fhir/type :fhir/Patient :id "0"}]]) + (let [did (codec/did 1 0)] + (with-system [{:blaze.db/keys [node]} system] + @(d/transact node [[:delete "Patient" "0"]]) - (given - (verify/verify-tx-cmds - (d/db node) 2 - [{:op "delete" :type "Patient" :id "0"}]) + (given (verify/verify-tx-cmds + (d/db node) 2 + [{:op "delete" :type "Patient" :id "0"}]) + + [0 0 0] := :resource-as-of-index + [0 0 1 rao-tu/decode-key] := {:type "Patient" :did did :t 2} + [0 0 2 rao-tu/decode-val] := {:hash hash/deleted-hash :num-changes 2 :op :delete :id "0"} - [0 #(drop 1 %) rao-tu/decode-index-entry] := - [{:type "Patient" :id "0" :t 2} - {:hash deleted-hash :num-changes 2 :op :delete}] + [0 1 0] := :type-as-of-index + [0 1 1 tao-tu/decode-key] := {:type "Patient" :t 2 :did did} + [0 1 2 tao-tu/decode-val] := {:hash hash/deleted-hash :num-changes 2 :op :delete :id "0"} - [1 #(drop 1 %) tao-tu/decode-index-entry] := - [{:type "Patient" :t 2 :id "0"} - {:hash deleted-hash :num-changes 2 :op :delete}] + [0 2 0] := :system-as-of-index + [0 2 1 sao-tu/decode-key] := {:t 2 :type "Patient" :did did} + [0 2 2 sao-tu/decode-val] := {:hash hash/deleted-hash :num-changes 2 :op :delete :id "0"} - [2 #(drop 1 %) sao-tu/decode-index-entry] := - [{:t 2 :type "Patient" :id "0"} - {:hash deleted-hash :num-changes 2 :op :delete}] + [0 3 0] := :type-stats-index + [0 3 1 ts-tu/decode-key] := {:type "Patient" :t 2} + [0 3 2 ts-tu/decode-val] := {:total 0 :num-changes 2} - [3 #(drop 1 %) ts-tu/decode-index-entry] := - [{:type "Patient" :t 2} - {:total 0 :num-changes 2}] + [0 4 0] := :system-stats-index + [0 4 1 ss-tu/decode-key] := {:t 2} + [0 4 2 ss-tu/decode-val] := {:total 0 :num-changes 2} - [4 #(drop 1 %) ss-tu/decode-index-entry] := - [{:t 2} - {:total 0 :num-changes 2}]))) + [1] :? empty?)))) (testing "adding a second patient to a store containing already one" - (with-system [{:blaze.db/keys [node]} system] - @(d/transact node [[:put {:fhir/type :fhir/Patient :id "0"}]]) + (let [did (codec/did 2 0) + hash (hash/generate patient-1)] + (with-system [{:blaze.db/keys [node]} system] + @(d/transact node [[:put {:fhir/type :fhir/Patient :id "0"}]]) - (is-entries= - (verify/verify-tx-cmds - (d/db node) 2 - [{:op "put" :type "Patient" :id "1" :hash (hash/generate patient-1)}]) - (let [value (rts/encode-value (hash/generate patient-1) 1 :put)] - [[:resource-as-of-index - (rao/encode-key tid-patient (codec/id-byte-string "1") 2) - value] - [:type-as-of-index - (tao/encode-key tid-patient 2 (codec/id-byte-string "1")) - value] - [:system-as-of-index - (sao/encode-key 2 tid-patient (codec/id-byte-string "1")) - value] - (type-stats/index-entry tid-patient 2 {:total 2 :num-changes 2}) - (system-stats/index-entry 2 {:total 2 :num-changes 2})])))) + (given (verify/verify-tx-cmds + (d/db node) 2 + [{:op "put" :type "Patient" :id "1" :hash hash}]) + + [0 0 0] := :resource-as-of-index + [0 0 1 rao-tu/decode-key] := {:type "Patient" :did did :t 2} + [0 0 2 rao-tu/decode-val] := {:hash hash :num-changes 1 :op :put :id "1"} + + [0 1 0] := :type-as-of-index + [0 1 1 tao-tu/decode-key] := {:type "Patient" :t 2 :did did} + [0 1 2 tao-tu/decode-val] := {:hash hash :num-changes 1 :op :put :id "1"} + + [0 2 0] := :system-as-of-index + [0 2 1 sao-tu/decode-key] := {:t 2 :type "Patient" :did did} + [0 2 2 sao-tu/decode-val] := {:hash hash :num-changes 1 :op :put :id "1"} + + [0 3 0] := :resource-id-index + [0 3 1 ri-tu/decode-key] := {:type "Patient" :id "1"} + [0 3 2 ri-tu/decode-val] := {:did did} + + [0 4 0] := :type-stats-index + [0 4 1 ts-tu/decode-key] := {:type "Patient" :t 2} + [0 4 2 ts-tu/decode-val] := {:total 2 :num-changes 2} + + [0 5 0] := :system-stats-index + [0 5 1 ss-tu/decode-key] := {:t 2} + [0 5 2 ss-tu/decode-val] := {:total 2 :num-changes 2} + + [1 0 :did] := did)))) (testing "update conflict" (with-system [{:blaze.db/keys [node]} system] @@ -250,6 +292,7 @@ (d/db node) 2 [{:op "put" :type "Patient" :id "0" :hash (hash/generate patient-0) :if-match 0}]) + ::anom/category := ::anom/conflict ::anom/message := "Precondition `W/\"0\"` failed on `Patient/0`." :http/status := 412))) @@ -268,25 +311,30 @@ [{:op "create" :type "Patient" :id "foo" :hash (hash/generate patient-0) :if-none-exist [["birthdate" "2020"]]}]) + ::anom/category := ::anom/conflict ::anom/message := "Conditional create of a Patient with query `birthdate=2020` failed because at least the two matches `Patient/0/_history/1` and `Patient/1/_history/1` were found." :http/status := 412))) (testing "match" (with-system [{:blaze.db/keys [node]} system] - @(d/transact node [[:put patient-3]]) + @(d/transact node [[:put {:fhir/type :fhir/Patient :id "3" + :identifier [#fhir/Identifier{:value "120426"}]}]]) - (is - (empty? - (verify/verify-tx-cmds - (d/db node) 2 - [{:op "create" :type "Patient" :id "0" - :hash (hash/generate patient-0) - :if-none-exist [["identifier" "120426"]]}]))))) + (given + (verify/verify-tx-cmds + (d/db node) 2 + [{:op "create" :type "Patient" :id "0" + :hash (hash/generate patient-0) + :if-none-exist [["identifier" "120426"]]}]) + + [0] :? empty? + [1] :? empty?))) (testing "conflict because matching resource is deleted" (with-system [{:blaze.db/keys [node]} system] - @(d/transact node [[:put patient-3]]) + @(d/transact node [[:put {:fhir/type :fhir/Patient :id "3" + :identifier [#fhir/Identifier{:value "120426"}]}]]) (given (verify/verify-tx-cmds @@ -295,28 +343,39 @@ {:op "create" :type "Patient" :id "0" :hash (hash/generate patient-0) :if-none-exist [["identifier" "120426"]]}]) + ::anom/category := ::anom/conflict ::anom/message := "Duplicate transaction commands `create Patient?identifier=120426 (resolved to id 3)` and `delete Patient/3`."))) (testing "on recreation" - (with-system [{:blaze.db/keys [node]} system] - @(d/transact node [[:put patient-0]]) - @(d/transact node [[:delete "Patient" "0"]]) + (let [did (codec/did 1 0) + hash (hash/generate patient-0)] + (with-system [{:blaze.db/keys [node]} system] + @(d/transact node [[:put patient-0]]) + @(d/transact node [[:delete "Patient" "0"]]) - (is-entries= - (verify/verify-tx-cmds - (d/db node) 3 - [{:op "put" :type "Patient" :id "0" - :hash (hash/generate patient-0)}]) - (let [value (rts/encode-value (hash/generate patient-0) 3 :put)] - [[:resource-as-of-index - (rao/encode-key tid-patient (codec/id-byte-string "0") 3) - value] - [:type-as-of-index - (tao/encode-key tid-patient 3 (codec/id-byte-string "0")) - value] - [:system-as-of-index - (sao/encode-key 3 tid-patient (codec/id-byte-string "0")) - value] - (type-stats/index-entry tid-patient 3 {:total 1 :num-changes 3}) - (system-stats/index-entry 3 {:total 1 :num-changes 3})])))))) + (given (verify/verify-tx-cmds + (d/db node) 3 + [{:op "put" :type "Patient" :id "0" :hash hash}]) + + [0 0 0] := :resource-as-of-index + [0 0 1 rao-tu/decode-key] := {:type "Patient" :did did :t 3} + [0 0 2 rao-tu/decode-val] := {:hash hash :num-changes 3 :op :put :id "0"} + + [0 1 0] := :type-as-of-index + [0 1 1 tao-tu/decode-key] := {:type "Patient" :t 3 :did did} + [0 1 2 tao-tu/decode-val] := {:hash hash :num-changes 3 :op :put :id "0"} + + [0 2 0] := :system-as-of-index + [0 2 1 sao-tu/decode-key] := {:t 3 :type "Patient" :did did} + [0 2 2 sao-tu/decode-val] := {:hash hash :num-changes 3 :op :put :id "0"} + + [0 3 0] := :type-stats-index + [0 3 1 ts-tu/decode-key] := {:type "Patient" :t 3} + [0 3 2 ts-tu/decode-val] := {:total 1 :num-changes 3} + + [0 4 0] := :system-stats-index + [0 4 1 ss-tu/decode-key] := {:t 3} + [0 4 2 ss-tu/decode-val] := {:total 1 :num-changes 3} + + [1 0 :did] := did)))))) diff --git a/modules/db/test/blaze/db/node_test.clj b/modules/db/test/blaze/db/node_test.clj index 2c5ccea7e..fca45d03c 100644 --- a/modules/db/test/blaze/db/node_test.clj +++ b/modules/db/test/blaze/db/node_test.clj @@ -185,7 +185,7 @@ (given-thrown (ig/init (with-index-store-version system 0)) :key := :blaze.db/node :reason := ::ig/build-threw-exception - [:cause-data :expected-version] := 1 + [:cause-data :expected-version] := 2 [:cause-data :actual-version] := 0))) @@ -280,10 +280,10 @@ (deftest existing-data-with-compatible-version - (with-system [{:blaze.db/keys [node]} (with-index-store-version system 1)] + (with-system [{:blaze.db/keys [node]} (with-index-store-version system 2)] (is node))) (deftest sets-db-version-on-startup (with-system [{kv-store [::kv/mem :blaze.db/index-kv-store]} system] - (is (= 1 (version/get kv-store))))) + (is (= 2 (version/get kv-store))))) diff --git a/modules/db/test/blaze/db/test_util.clj b/modules/db/test/blaze/db/test_util.clj index 82e32ef68..94b519a08 100644 --- a/modules/db/test/blaze/db/test_util.clj +++ b/modules/db/test/blaze/db/test_util.clj @@ -37,7 +37,8 @@ :blaze.test/clock {} - :blaze.db/resource-handle-cache {} + :blaze.db/resource-handle-cache + {:max-size 1000} :blaze.db/tx-cache {:kv-store (ig/ref :blaze.db/index-kv-store)} @@ -54,6 +55,7 @@ :tx-success-index {:reverse-comparator? true} :tx-error-index nil :t-by-instant-index {:reverse-comparator? true} + :resource-id-index nil :resource-as-of-index nil :type-as-of-index nil :system-as-of-index nil diff --git a/modules/interaction/test/blaze/interaction/search/include_test.clj b/modules/interaction/test/blaze/interaction/search/include_test.clj index ba44222e5..b9812d332 100644 --- a/modules/interaction/test/blaze/interaction/search/include_test.clj +++ b/modules/interaction/test/blaze/interaction/search/include_test.clj @@ -45,7 +45,9 @@ count := 1 [0 fhir-spec/fhir-type] := :fhir/Patient)))) - (testing "not enforcing referential integrity" + ;; TODO: we have to add to the ResourceSearchParamValue index of the observation in the transaction creating the patient + ;; TODO: add this test to blaze.db.api-test + #_(testing "not enforcing referential integrity" (with-system-data [{:blaze.db/keys [node]} non-ref-int-system] [[[:put {:fhir/type :fhir/Observation :id "0" :subject @@ -99,8 +101,8 @@ observations (d/type-list db "Observation")] (given (include/add-includes db include-defs observations) count := 2 - [0 fhir-spec/fhir-type] := :fhir/Patient - [1 fhir-spec/fhir-type] := :fhir/Encounter)))) + [0 fhir-spec/fhir-type] := :fhir/Encounter + [1 fhir-spec/fhir-type] := :fhir/Patient)))) (testing "one direct reverse include" (with-system-data [{:blaze.db/keys [node]} mem-node-system] diff --git a/modules/interaction/test/blaze/interaction/search_type_test.clj b/modules/interaction/test/blaze/interaction/search_type_test.clj index 407f7ac03..fd17e158d 100644 --- a/modules/interaction/test/blaze/interaction/search_type_test.clj +++ b/modules/interaction/test/blaze/interaction/search_type_test.clj @@ -1785,13 +1785,13 @@ [:search :mode] := #fhir/code"match")) (testing "the second entry is the included Encounter" - (given (-> body :entry (nth 2)) + (given (-> body :entry second) :fullUrl := #fhir/uri"base-url-113047/Encounter/1" [:resource :fhir/type] := :fhir/Encounter [:search :mode] := #fhir/code"include")) (testing "the third entry is the included Patient" - (given (-> body :entry second) + (given (-> body :entry (nth 2)) :fullUrl := #fhir/uri"base-url-113047/Patient/0" [:resource :fhir/type] := :fhir/Patient [:search :mode] := #fhir/code"include"))))) diff --git a/profiling/blaze/profiling.clj b/profiling/blaze/profiling.clj index da2830ee6..34fe9339c 100644 --- a/profiling/blaze/profiling.clj +++ b/profiling/blaze/profiling.clj @@ -73,6 +73,7 @@ (rocksdb/get-property index-db :tx-success-index "rocksdb.stats") (rocksdb/get-property index-db :tx-error-index "rocksdb.stats") (rocksdb/get-property index-db :t-by-instant-index "rocksdb.stats") + (rocksdb/get-property index-db :resource-id-index "rocksdb.stats") (rocksdb/get-property index-db :resource-as-of-index "rocksdb.stats") (rocksdb/get-property index-db :type-as-of-index "rocksdb.stats") (rocksdb/get-property index-db :system-as-of-index "rocksdb.stats") diff --git a/resources/blaze.edn b/resources/blaze.edn index 6fbf37a09..a3607ac82 100644 --- a/resources/blaze.edn +++ b/resources/blaze.edn @@ -280,6 +280,7 @@ :tx-success-index {:reverse-comparator? true} :tx-error-index nil :t-by-instant-index {:reverse-comparator? true} + :resource-id-index nil :resource-as-of-index nil :type-as-of-index nil :system-as-of-index nil @@ -403,6 +404,12 @@ :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384] :reverse-comparator? true} + :resource-id-index + {:write-buffer-size-in-mb 4 + :max-bytes-for-level-base-in-mb 16 + :target-file-size-base-in-mb 4 + :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} + :resource-as-of-index {:write-buffer-size-in-mb 8 :max-bytes-for-level-base-in-mb 32 @@ -597,6 +604,12 @@ :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384] :reverse-comparator? true} + :resource-id-index + {:write-buffer-size-in-mb 4 + :max-bytes-for-level-base-in-mb 16 + :target-file-size-base-in-mb 4 + :block-size #blaze/cfg ["DB_BLOCK_SIZE" int? 16384]} + :resource-as-of-index {:write-buffer-size-in-mb 8 :max-bytes-for-level-base-in-mb 32