Add property index #529

Author: joepio

CHANGELOG.md

@@ -6,6 +6,7 @@ Changes to JS assets are not included here, but in [`atomic-data-browser`'s CHAN
 
 ## UNRELEASED
 
+- Improve query performance, refactor indexes #529
 - Improved error handling for HTTPS initialization #530
 
 ## [v0.34.0] - 2022-10-31

lib/src/atoms.rs

@@ -2,7 +2,7 @@
 
 use crate::{
     errors::AtomicResult,
-    values::{ReferenceString, Value},
+    values::{ReferenceString, SortableValue, Value},
 };
 
 /// The Atom is the smallest meaningful piece of data.
@@ -34,13 +34,15 @@ impl Atom {
 
     /// Converts one Atom to a series of stringified values that can be indexed.
     pub fn to_indexable_atoms(&self) -> Vec<IndexAtom> {
+        let sort_value = self.value.to_sortable_string();
         let index_atoms = match &self.value.to_reference_index_strings() {
             Some(v) => v,
             None => return vec![],
         }
         .iter()
         .map(|v| IndexAtom {
-            value: v.into(),
+            ref_value: v.into(),
+            sort_value: sort_value.clone(),
             subject: self.subject.clone(),
             property: self.property.clone(),
         })
@@ -56,7 +58,8 @@ impl Atom {
 pub struct IndexAtom {
     pub subject: String,
     pub property: String,
-    pub value: ReferenceString,
+    pub ref_value: ReferenceString,
+    pub sort_value: SortableValue,
 }
 
 impl std::fmt::Display for Atom {

lib/src/db.rs

@@ -4,26 +4,27 @@
 mod migrations;
 mod prop_val_sub_index;
 mod query_index;
-mod reference_index;
 #[cfg(test)]
 pub mod test;
+mod val_prop_sub_index;
 
 use std::{
     collections::{HashMap, HashSet},
     sync::{Arc, Mutex},
 };
 
-use tracing::{info, instrument, trace};
+use tracing::{info, instrument};
 
 use crate::{
     atoms::IndexAtom,
     commit::CommitResponse,
-    db::reference_index::key_to_atom,
+    db::val_prop_sub_index::find_in_val_prop_sub_index,
     endpoints::{default_endpoints, Endpoint},
     errors::{AtomicError, AtomicResult},
     resources::PropVals,
     storelike::{Query, QueryResult, Storelike},
-    Atom, Resource, Value,
+    values::SortableValue,
+    Atom, Resource,
 };
 
 use self::{
@@ -34,9 +35,9 @@ use self::{
     },
     query_index::{
         check_if_atom_matches_watched_query_filters, query_indexed, update_indexed_member,
-        watch_collection, IndexIterator, QueryFilter,
+        IndexIterator, QueryFilter,
     },
-    reference_index::{add_atom_to_reference_index, remove_atom_from_reference_index},
+    val_prop_sub_index::{add_atom_to_reference_index, remove_atom_from_reference_index},
 };
 
 // A function called by the Store when a Commit is accepted
@@ -474,29 +475,45 @@ impl Storelike for Db {
     #[instrument(skip(self))]
     fn query(&self, q: &Query) -> AtomicResult<QueryResult> {
         if let Ok(res) = query_indexed(self, q) {
+            // TODO: Maybe this is not the best check.
+            // If nothing is found, this may indicate both that the query is not indexed,
+            // or that there are simply no results.
+            // Probably should use `q_filter.is_watched` to check if the query is indexed.
             if res.count > 0 {
                 // Yay, we have a cache hit!
-                // We don't have to perform a (more expansive) TPF query + sorting
+                // We don't have to create the indexes, so we can return early.
                 return Ok(res);
             }
         }
 
         let q_filter: QueryFilter = q.into();
 
         // Maybe make this optional?
-        watch_collection(self, &q_filter)?;
+        q_filter.watch(self)?;
 
         info!(filter = ?q_filter, "Building query index");
 
         let atoms: IndexIterator = match (&q.property, q.value.as_ref()) {
             (Some(prop), val) => find_in_prop_val_sub_index(self, prop, val),
             (None, None) => self.all_index_atoms(q.include_external),
-            (None, Some(_)) => todo!(),
+            (None, Some(val)) => find_in_val_prop_sub_index(self, val, None),
         };
 
         for a in atoms {
             let atom = a?;
-            update_indexed_member(self, &q_filter, &atom.subject, &atom.value, false)?;
+            let sort_val: SortableValue = if let Some(sort) = &q_filter.sort_by {
+                if &atom.property == sort {
+                    atom.sort_value
+                } else {
+                    // Find the sort value in the store
+                    let sort_atom = self.get_value(&atom.subject, sort)?;
+                    sort_atom.to_sortable_string()
+                }
+            } else {
+                atom.sort_value
+            };
+
+            update_indexed_member(self, &q_filter, &atom.subject, &sort_val, false)?;
         }
 
         // Retry the same query!
@@ -562,111 +579,6 @@ impl Storelike for Db {
     fn set_default_agent(&self, agent: crate::agents::Agent) {
         self.default_agent.lock().unwrap().replace(agent);
     }
-
-    // TPF implementation that used the index_value cache, far more performant than the StoreLike implementation
-    #[instrument(skip(self))]
-    fn tpf(
-        &self,
-        q_subject: Option<&str>,
-        q_property: Option<&str>,
-        q_value: Option<&Value>,
-        // Whether resources from outside the store should be searched through
-        include_external: bool,
-    ) -> AtomicResult<Vec<Atom>> {
-        trace!("tpf");
-        let mut vec: Vec<Atom> = Vec::new();
-
-        let hassub = q_subject.is_some();
-        let hasprop = q_property.is_some();
-        let hasval = q_value.is_some();
-
-        // Simply return all the atoms
-        if !hassub && !hasprop && !hasval {
-            for resource in self.all_resources(include_external) {
-                for (property, value) in resource.get_propvals() {
-                    vec.push(Atom::new(
-                        resource.get_subject().clone(),
-                        property.clone(),
-                        value.clone(),
-                    ))
-                }
-            }
-            return Ok(vec);
-        }
-
-        // If the value is a resourcearray, check if it is inside
-        let val_equals = |val: &str| {
-            let q = q_value.unwrap().to_sortable_string();
-            val == q || {
-                if val.starts_with('[') {
-                    match crate::parse::parse_json_array(val) {
-                        Ok(vec) => return vec.contains(&q),
-                        Err(_) => return val == q,
-                    }
-                }
-                false
-            }
-        };
-
-        // Find atoms matching the TPF query in a single resource
-        let mut find_in_resource = |resource: &Resource| {
-            let subj = resource.get_subject();
-            for (prop, val) in resource.get_propvals().iter() {
-                if hasprop && q_property.as_ref().unwrap() == prop {
-                    if hasval {
-                        if val_equals(&val.to_string()) {
-                            vec.push(Atom::new(subj.into(), prop.into(), val.clone()))
-                        }
-                        break;
-                    } else {
-                        vec.push(Atom::new(subj.into(), prop.into(), val.clone()))
-                    }
-                    break;
-                } else if hasval && !hasprop && val_equals(&val.to_string()) {
-                    vec.push(Atom::new(subj.into(), prop.into(), val.clone()))
-                }
-            }
-        };
-
-        match q_subject {
-            Some(sub) => match self.get_resource(sub) {
-                Ok(resource) => {
-                    if hasprop | hasval {
-                        find_in_resource(&resource);
-                        Ok(vec)
-                    } else {
-                        Ok(resource.to_atoms())
-                    }
-                }
-                Err(_) => Ok(vec),
-            },
-            None => {
-                if hasval {
-                    let key_prefix = if hasprop {
-                        format!("{}\n{}\n", q_value.unwrap(), q_property.unwrap())
-                    } else {
-                        format!("{}\n", q_value.unwrap())
-                    };
-                    for item in self.reference_index.scan_prefix(key_prefix) {
-                        let (k, _v) = item?;
-                        let key_string = String::from_utf8(k.to_vec())?;
-                        // WARNING: Converts all Atoms to Strings, the datatype is lost here
-                        let atom = key_to_atom(&key_string)?;
-                        // NOTE: This means we'll include random values that start with the current server URL, including paragraphs for example.
-                        if include_external || atom.subject.starts_with(self.get_server_url()) {
-                            vec.push(atom)
-                        }
-                    }
-                    return Ok(vec);
-                }
-                // TODO: Add an index for searching only by property
-                for resource in self.all_resources(include_external) {
-                    find_in_resource(&resource);
-                }
-                Ok(vec)
-            }
-        }
-    }
 }
 
 fn corrupt_db_message(subject: &str) -> String {

lib/src/db/prop_val_sub_index.rs

@@ -44,7 +44,9 @@ fn key_from_atom(atom: &IndexAtom) -> Vec<u8> {
     [
         atom.property.as_bytes(),
         &[SEPARATION_BIT],
-        atom.value.as_bytes(),
+        atom.ref_value.as_bytes(),
+        &[SEPARATION_BIT],
+        atom.sort_value.as_bytes(),
         &[SEPARATION_BIT],
         atom.subject.as_bytes(),
     ]
@@ -57,13 +59,16 @@ fn key_to_index_atom(key: &[u8]) -> AtomicResult<IndexAtom> {
     let mut parts = key.split(|b| b == &SEPARATION_BIT);
     let prop = std::str::from_utf8(parts.next().ok_or("Invalid key for prop_val_sub_index")?)
         .map_err(|_| "Can't parse prop into string")?;
-    let val = std::str::from_utf8(parts.next().ok_or("Invalid key for prop_val_sub_index")?)
-        .map_err(|_| "Can't parse val into string")?;
+    let ref_val = std::str::from_utf8(parts.next().ok_or("Invalid key for prop_val_sub_index")?)
+        .map_err(|_| "Can't parse ref_val into string")?;
+    let sort_val = std::str::from_utf8(parts.next().ok_or("Invalid key for prop_val_sub_index")?)
+        .map_err(|_| "Can't parse sort_val into string")?;
     let sub = std::str::from_utf8(parts.next().ok_or("Invalid key for prop_val_sub_index")?)
         .map_err(|_| "Can't parse subject into string")?;
     Ok(IndexAtom {
         property: prop.into(),
-        value: val.into(),
+        ref_value: ref_val.into(),
+        sort_value: sort_val.into(),
         subject: sub.into(),
     })
 }
@@ -76,7 +81,8 @@ mod test {
     fn round_trip() {
         let atom = IndexAtom {
             property: "http://example.com/prop".into(),
-            value: "http://example.com/val \n hello \n".into(),
+            ref_value: "http://example.com/val \n hello \n".into(),
+            sort_value: "2".into(),
             subject: "http://example.com/subj".into(),
         };
         let key = key_from_atom(&atom);