Remove TPF #529

Author: joepio

CHANGELOG.md

@@ -6,7 +6,7 @@ Changes to JS assets are not included here, but in [`atomic-data-browser`'s CHAN
 
 ## UNRELEASED
 
-- Improve query performance, refactor indexes #529
+- Improve query performance, refactor indexes. The `.tpf` API is deprecated in favor of the more powerful `.query`. #529
 - Improved error handling for HTTPS initialization #530
 
 ## [v0.34.0] - 2022-10-31

lib/src/db/query_index.rs

@@ -5,7 +5,7 @@ use crate::{
     atoms::IndexAtom,
     errors::AtomicResult,
     storelike::{Query, QueryResult},
-    values::{query_value_compare, SortableValue},
+    values::SortableValue,
     Atom, Db, Resource, Storelike, Value,
 };
 use serde::{Deserialize, Serialize};
@@ -112,7 +112,6 @@ pub fn query_indexed(store: &Db, q: &Query) -> AtomicResult<QueryResult> {
         if in_selection {
             let (k, _v) = kv.map_err(|_e| "Unable to parse query_cached")?;
             let (_q_filter, _val, subject) = parse_collection_members_key(&k)?;
-            println!("Found subject: {} with val :{_val}", subject);
 
             // If no external resources should be included, skip this one if it's an external resource
             if !q.include_external && !subject.starts_with(&self_url) {
@@ -179,7 +178,7 @@ fn check_resource_query_filter_property(
         }
     } else if let Some(filter_val) = &q_filter.value {
         for (prop, val) in resource.get_propvals() {
-            if query_value_compare(val, filter_val) {
+            if val.contains_value(filter_val) {
                 return Some(prop.to_string());
             }
         }
@@ -350,7 +349,6 @@ pub fn create_query_index_key(
     };
     value_bytes.push(SEPARATION_BIT);
 
-    println!("Create key subject {:?} value {:?}", subject, value);
     let subject_bytes = if let Some(sub) = subject {
         sub.as_bytes().to_vec()
     } else {
@@ -383,7 +381,6 @@ pub fn parse_collection_members_key(bytes: &[u8]) -> AtomicResult<(QueryFilter,
     } else {
         return Err("Can't parse subject in members_key".into());
     };
-    println!("Parsed key: {:?} {:?} {:?}", q_filter, value, subject);
     Ok((q_filter, value, subject))
 }
 

lib/src/hierarchy.rs

@@ -4,7 +4,7 @@
 
 use core::fmt;
 
-use crate::{errors::AtomicResult, urls, AtomicError, Resource, Storelike, Value};
+use crate::{errors::AtomicResult, storelike::Query, urls, AtomicError, Resource, Storelike};
 
 #[derive(Debug)]
 pub enum Right {
@@ -32,16 +32,8 @@ impl fmt::Display for Right {
 
 /// Looks for children relations, adds to the resource. Performs a TPF query, might be expensive.
 pub fn add_children(store: &impl Storelike, resource: &mut Resource) -> AtomicResult<Resource> {
-    let atoms = store.tpf(
-        None,
-        Some(urls::PARENT),
-        Some(&Value::AtomicUrl(resource.get_subject().into())),
-        false,
-    )?;
-    let mut children: Vec<String> = Vec::new();
-    for atom in atoms {
-        children.push(atom.subject)
-    }
+    let results = store.query(&Query::new_prop_val(urls::PARENT, resource.get_subject()))?;
+    let mut children = results.subjects;
     children.sort();
     resource.set_propval(urls::CHILDREN.into(), children.into(), store)?;
     Ok(resource.to_owned())

lib/src/plugins/bookmark.rs

@@ -121,7 +121,7 @@ impl Parser {
     pub fn serialize(node: NodeRef) -> Result<String, FromUtf8Error> {
         let mut stream = Vec::new();
         if let Err(e) = node.serialize(&mut stream) {
-            println!("{}", e);
+            tracing::error!("Serialize error: {}", e);
         }
 
         String::from_utf8(stream)

lib/src/plugins/versioning.rs

@@ -1,8 +1,8 @@
 use tracing::warn;
 
 use crate::{
-    collections::CollectionBuilder, endpoints::Endpoint, errors::AtomicResult, urls, AtomicError,
-    Commit, Resource, Storelike, Value,
+    collections::CollectionBuilder, endpoints::Endpoint, errors::AtomicResult, storelike::Query,
+    urls, AtomicError, Commit, Resource, Storelike,
 };
 
 pub fn version_endpoint() -> Endpoint {
@@ -91,37 +91,16 @@ fn handle_all_versions_request(
 /// Searches the local store for all commits with this subject, returns sorted from old to new.
 #[tracing::instrument(skip(store))]
 fn get_commits_for_resource(subject: &str, store: &impl Storelike) -> AtomicResult<Vec<Commit>> {
-    let commit_atoms = store.tpf(
-        None,
-        Some(urls::SUBJECT),
-        Some(&Value::AtomicUrl(subject.into())),
-        false,
-    )?;
-    let mut commits = Vec::new();
-    for atom in commit_atoms {
-        let resource = if let Ok(r) = store.get_resource(&atom.subject) {
-            r
-        } else {
-            // https://github.com/atomicdata-dev/atomic-data-rust/issues/488
-            warn!("Could not find commit for {} , skipping", atom.subject);
-            continue;
-        };
-        let mut is_commit = false;
-        // If users use the `subject` field for a non-commit, we prevent using it as a commit here.
-        for c in resource.get(urls::IS_A)?.to_subjects(None)?.iter() {
-            if c == urls::COMMIT {
-                is_commit = true
-            }
-        }
-        if is_commit {
-            let commit = crate::Commit::from_resource(resource)?;
-            commits.push(commit)
-        }
-    }
-    // Sort all commits by date
-    commits.sort_by(|a, b| a.created_at.cmp(&b.created_at));
+    let mut q = Query::new_prop_val(urls::SUBJECT, subject);
+    q.sort_by = Some(urls::CREATED_AT.into());
+    let result = store.query(&q)?;
+    let filtered: Vec<Commit> = result
+        .resources
+        .iter()
+        .filter_map(|r| crate::Commit::from_resource(r.clone()).ok())
+        .collect();
 
-    Ok(commits)
+    Ok(filtered)
 }
 
 #[tracing::instrument(skip(store))]

lib/src/populate.rs

@@ -8,6 +8,7 @@ use crate::{
     errors::AtomicResult,
     parse::ParseOpts,
     schema::{Class, Property},
+    storelike::Query,
     urls, Resource, Storelike, Value,
 };
 
@@ -212,18 +213,13 @@ pub fn populate_default_store(store: &impl Storelike) -> AtomicResult<()> {
 /// Generates some nice collections for classes, such as `/agent` and `/collection`.
 /// Requires a `self_url` to be set in the store.
 pub fn populate_collections(store: &impl Storelike) -> AtomicResult<()> {
-    let classes_atoms = store.tpf(
-        None,
-        Some("https://atomicdata.dev/properties/isA"),
-        Some(&Value::AtomicUrl(
-            "https://atomicdata.dev/classes/Class".into(),
-        )),
-        true,
-    )?;
+    let mut query = Query::new_class(urls::CLASS);
+    query.include_external = true;
+    let result = store.query(&query)?;
 
-    for atom in classes_atoms {
+    for subject in result.subjects {
         let mut collection =
-            crate::collections::create_collection_resource_for_class(store, &atom.subject)?;
+            crate::collections::create_collection_resource_for_class(store, &subject)?;
         collection.save_locally(store)?;
     }
 

lib/src/store.rs

@@ -1,6 +1,8 @@
 //! In-memory store of Atomic data.
 //! This provides many methods for finding, changing, serializing and parsing Atomic Data.
 
+use crate::storelike::QueryResult;
+use crate::Value;
 use crate::{atoms::Atom, storelike::Storelike};
 use crate::{errors::AtomicResult, Resource};
 use std::{collections::HashMap, sync::Arc, sync::Mutex};
@@ -24,6 +26,97 @@ impl Store {
         crate::populate::populate_base_models(&store)?;
         Ok(store)
     }
+
+    /// Triple Pattern Fragments interface.
+    /// Use this for most queries, e.g. finding all items with some property / value combination.
+    /// Returns an empty array if nothing is found.
+    ///
+    /// # Example
+    ///
+    /// For example, if I want to view all Resources that are instances of the class "Property", I'd do:
+    ///
+    /// ```
+    /// use atomic_lib::Storelike;
+    /// let mut store = atomic_lib::Store::init().unwrap();
+    /// store.populate();
+    /// let atoms = store.tpf(
+    ///     None,
+    ///     Some("https://atomicdata.dev/properties/isA"),
+    ///     Some(&atomic_lib::Value::AtomicUrl("https://atomicdata.dev/classes/Class".into())),
+    ///     true
+    /// ).unwrap();
+    /// assert!(atoms.len() > 11)
+    /// ```
+    // Very costly, slow implementation.
+    // Does not assume any indexing.
+    fn tpf(
+        &self,
+        q_subject: Option<&str>,
+        q_property: Option<&str>,
+        q_value: Option<&Value>,
+        // Whether resources from outside the store should be searched through
+        include_external: bool,
+    ) -> AtomicResult<Vec<Atom>> {
+        let mut vec: Vec<Atom> = Vec::new();
+
+        let hassub = q_subject.is_some();
+        let hasprop = q_property.is_some();
+        let hasval = q_value.is_some();
+
+        // Simply return all the atoms
+        if !hassub && !hasprop && !hasval {
+            for resource in self.all_resources(include_external) {
+                for (property, value) in resource.get_propvals() {
+                    vec.push(Atom::new(
+                        resource.get_subject().clone(),
+                        property.clone(),
+                        value.clone(),
+                    ))
+                }
+            }
+            return Ok(vec);
+        }
+
+        // Find atoms matching the TPF query in a single resource
+        let mut find_in_resource = |resource: &Resource| {
+            let subj = resource.get_subject();
+            for (prop, val) in resource.get_propvals().iter() {
+                if hasprop && q_property.as_ref().unwrap() == prop {
+                    if hasval {
+                        if val.contains_value(q_value.unwrap()) {
+                            vec.push(Atom::new(subj.into(), prop.into(), val.clone()))
+                        }
+                        break;
+                    } else {
+                        vec.push(Atom::new(subj.into(), prop.into(), val.clone()))
+                    }
+                    break;
+                } else if hasval && !hasprop && val.contains_value(q_value.unwrap()) {
+                    vec.push(Atom::new(subj.into(), prop.into(), val.clone()))
+                }
+            }
+        };
+
+        match q_subject {
+            Some(sub) => match self.get_resource(sub) {
+                Ok(resource) => {
+                    if hasprop | hasval {
+                        find_in_resource(&resource);
+                        Ok(vec)
+                    } else {
+                        Ok(resource.to_atoms())
+                    }
+                }
+                Err(_) => Ok(vec),
+            },
+            None => {
+                for resource in self.all_resources(include_external) {
+                    find_in_resource(&resource);
+                }
+                Ok(vec)
+            }
+        }
+    }
 }
 
 impl Storelike for Store {
@@ -126,6 +219,62 @@ impl Storelike for Store {
     fn set_default_agent(&self, agent: crate::agents::Agent) {
         self.default_agent.lock().unwrap().replace(agent);
     }
+
+    fn query(&self, q: &crate::storelike::Query) -> AtomicResult<crate::storelike::QueryResult> {
+        let atoms = self.tpf(
+            None,
+            q.property.as_deref(),
+            q.value.as_ref(),
+            q.include_external,
+        )?;
+
+        // Remove duplicate subjects
+        let mut subjects_deduplicated: Vec<String> = atoms
+            .iter()
+            .map(|atom| atom.subject.clone())
+            .collect::<std::collections::HashSet<String>>()
+            .into_iter()
+            .collect();
+
+        // Sort by subject, better than no sorting
+        subjects_deduplicated.sort();
+
+        // WARNING: Entering expensive loop!
+        // This is needed for sorting, authorization and including nested resources.
+        // It could be skipped if there is no authorization and sorting requirement.
+        let mut resources = Vec::new();
+        for subject in subjects_deduplicated.iter() {
+            // These nested resources are not fully calculated - they will be presented as -is
+            match self.get_resource_extended(subject, true, q.for_agent.as_deref()) {
+                Ok(resource) => {
+                    resources.push(resource);
+                }
+                Err(e) => match &e.error_type {
+                    crate::AtomicErrorType::NotFoundError => {}
+                    crate::AtomicErrorType::UnauthorizedError => {}
+                    _other => {
+                        return Err(
+                            format!("Error when getting resource in collection: {}", e).into()
+                        )
+                    }
+                },
+            }
+        }
+
+        if let Some(sort) = &q.sort_by {
+            resources = crate::collections::sort_resources(resources, sort, q.sort_desc);
+        }
+        let mut subjects = Vec::new();
+        for r in resources.iter() {
+            subjects.push(r.get_subject().clone())
+        }
+
+        Ok(QueryResult {
+            count: atoms.len(),
+            subjects,
+            resources,
+        })
+    }
 }
 
 #[cfg(test)]