Upgrades to Oxigraph v0.3

.pre-commit-config.yaml

@@ -28,7 +28,7 @@ repos:
     hooks:
       - id: isort
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.31.0
+    rev: v2.31.1
     hooks:
       - id: pyupgrade
         args: [ --py37-plus ]
@@ -41,14 +41,14 @@ repos:
     hooks:
       - id: flake8
   - repo: https://github.com/PyCQA/bandit
-    rev: 1.7.2
+    rev: 1.7.4
     hooks:
       - id: bandit
   - repo: https://github.com/PyCQA/pydocstyle
     rev: 6.1.1
     hooks:
       - id: pydocstyle
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.931'
+    rev: v0.941
     hooks:
       - id: mypy

README.md

@@ -7,31 +7,31 @@ Oxrdflib
 [![actions status](https://github.com/oxigraph/oxrdflib/workflows/build/badge.svg)](https://github.com/oxigraph/oxrdflib/actions)
 [![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 
-Oxrdflib provides [rdflib](https://rdflib.readthedocs.io/) stores using [pyoxigraph](https://oxigraph.org/pyoxigraph/).
+Oxrdflib provides an [rdflib](https://rdflib.readthedocs.io/) store based on [pyoxigraph](https://oxigraph.org/pyoxigraph/).
+This store is named `"Oxigraph"`.
 
-The stores could be used as drop-in replacements of the rdflib default ones. They support context but not formulas.
+This store can be used as drop-in replacement of the rdflib default one. It support context but not formulas.
 Transaction support is not implemented yet.
 
-SPARQL query evaluation is done by pyoxigraph instead of rdflib if an oxrdflib store is used.
-
-Two stores are currently provided:
-* An in-memory store, named `"OxMemory"`.
-* A disk-based store based on the [Sled key-value store](https://sled.rs/), named `"OxSled"`.
+SPARQL query evaluation is done by pyoxigraph instead of rdflib if the Oxigraph store is used.
+SPARQL update evaluation is still done using rdflib because of [a limitation in rdflib context management](https://github.com/RDFLib/rdflib/issues/1396).
 
 Oxrdflib is [available on Pypi](https://pypi.org/project/oxrdflib/) and installable with:
 ```bash
 pip install oxrdflib
 ```
 
-The oxrdflib stores are automatically registered as rdflib store plugins by setuptools.
+The oxrdflib store is automatically registered as an rdflib store plugin by setuptools.
 
-## API
+*Warning:* Oxigraph is not stable yet and its storage format might change in the future.
+To migrate to future version you might have to dump and load the store content.
+However, Oxigraph should be in a good enough shape to power most of use cases if you are not afraid of down time and data loss.
 
-### `"OxMemory"`, an in-memory store
+## API
 
-To create a rdflib graph with pyoxigraph in memory store use
+To create a rdflib graph using the Oxigraph store use
 ```python
-rdflib.Graph(store="OxMemory")
+rdflib.Graph(store="Oxigraph")
 ```
 instead of the usual
 ```python
@@ -40,34 +40,39 @@ rdflib.Graph()
 
 Similarly, to get a conjunctive graph, use
 ```python
-rdflib.ConjunctiveGraph(store="OxMemory")
+rdflib.ConjunctiveGraph(store="Oxigraph")
 ```
 instead of the usual
 ```python
 rdflib.ConjunctiveGraph()
 ```
+and to get a dataset, use
 
-### `"OxSled"`, a disk-based store
-
-The disk-based store is based on the [Sled key-value store](https://sled.rs/).
-Sled is not stable yet and its storage system might change in the future.
+```python
+rdflib.Dataset(store="Oxigraph")
+```
+instead of the usual
+```python
+rdflib.Dataset()
+```
 
-To open Sled based graph in the directory `test_dir` use
+If you want to get the store data persisted on disk, use the `open` method on the `Graph` object (or `ConjunctiveGraph` or `Dataset`) with the directory where data should be persisted. For example:
 ```python
-graph = rdflib.Graph(store="OxSled")
+graph = rdflib.Graph(store="Oxigraph")
 graph.open("test_dir")
 ```
 The store is closed with the `close()` method or automatically when Python garbage collector collects the store object.
 
-It is also possible to not provide a directory name.
-In this case, a temporary directory will be created and deleted when the store is closed.
-For example, this code uses a temporary directory:
-```python
-rdflib.Graph(store="OxSled")
-```
+If the `open` method is not called Oxigraph will automatically use a ramdisk on Linux and a temporary file in the other operating systems.
+
+To do anything else, use the usual rdflib python API.
 
-`rdflib.ConjunctiveGraph` is also usable with `"OxSled"`.
+## Migration guide
 
+### From 0.2 to 0.3
+* The 0.2 stores named `"OxSled"` and `"OxMemory"` have been merged into the `"Oxigraph"` store.
+* The on-disk storage system provided by `"OxSled"` has been dropped and replaced by a new storage system based on [RocksDB](https://rocksdb.org/).
+  To migrate you need to first dump your data in RDF using `oxrdflib` 0.2 and the `serialize` method, then upgrade to `oxrdflib` 0.3, and finally reload the data using the `parse` method.
 
 ## Development
 

oxrdflib/__init__.py

@@ -1,4 +1,4 @@
-from abc import ABC, abstractmethod
+import shutil
 
 import pyoxigraph as ox
 from rdflib import Graph
@@ -7,23 +7,40 @@
 from rdflib.store import VALID_STORE, Store
 from rdflib.term import BNode, Literal, Node, URIRef, Variable
 
-__all__ = ["MemoryOxStore", "SledOxStore"]
+__all__ = ["OxigraphStore"]
 
 
-class _BaseOxStore(Store, ABC):
+class OxigraphStore(Store):
     context_aware = True
     formula_aware = False
     transaction_aware = False
     graph_aware = True
 
-    @property
-    @abstractmethod
-    def _inner(self):
-        pass
+    def __init__(self, configuration=None, identifier=None):
+        self._store = None
+        super().__init__(configuration, identifier)
+
+    def open(self, configuration, create=False):
+        if self._store is not None:
+            raise ValueError("The open function should be called before any RDF operation")
+        self._store = ox.Store(configuration)
+        return VALID_STORE
+
+    def close(self, commit_pending_transaction=False):
+        del self._store
+
+    def destroy(self, configuration):
+        shutil.rmtree(configuration)
 
     def gc(self):
         pass
 
+    @property
+    def _inner(self):
+        if self._store is None:
+            self._store = ox.Store()
+        return self._store
+
     def add(self, triple, context, quoted=False):
         if quoted:
             raise ValueError("Oxigraph stores are not formula aware")
@@ -96,38 +113,6 @@ def remove_graph(self, graph):
         self._inner.remove_graph(_to_ox(graph))
 
 
-class MemoryOxStore(_BaseOxStore):
-    def __init__(self, configuration=None, identifier=None):
-        self._store = ox.MemoryStore()
-        super().__init__(configuration, identifier)
-
-    @property
-    def _inner(self):
-        return self._store
-
-
-class SledOxStore(_BaseOxStore):
-    def __init__(self, configuration=None, identifier=None):
-        self._store = None
-        super().__init__(configuration, identifier)
-
-    def open(self, configuration, create=False):
-        self._store = ox.SledStore(configuration)
-        return VALID_STORE
-
-    def close(self, commit_pending_transaction=False):
-        del self._store
-
-    def destroy(self, configuration):
-        raise NotImplementedError("destroy is not implemented yet for the Sled based store")
-
-    @property
-    def _inner(self):
-        if self._store is None:
-            self._store = ox.SledStore()
-        return self._store
-
-
 def _to_ox(term, context=None):
     if term is None:
         return None

setup.py

@@ -4,7 +4,7 @@
 
 setup(
     name="oxrdflib",
-    version="0.2.0",
+    version="0.3.0",
     description="rdflib stores based on pyoxigraph",
     long_description=(pathlib.Path(__file__).parent / "README.md").read_text(),
     long_description_content_type="text/markdown",
@@ -31,7 +31,13 @@
         "Tracker": "https://github.com/oxigraph/oxrdflib/issues",
     },
     packages=["oxrdflib"],
-    install_requires=["pyoxigraph~=0.2.0", "rdflib~=6.0"],
-    entry_points={"rdf.plugins.store": ["OxMemory = oxrdflib:MemoryOxStore", "OxSled = oxrdflib:SledOxStore"]},
+    install_requires=["pyoxigraph~=0.3.0", "rdflib~=6.0"],
+    entry_points={
+        "rdf.plugins.store": [
+            "Oxigraph = oxrdflib:OxigraphStore",
+            "OxMemory = oxrdflib:OxigraphStore",
+            "OxSled = oxrdflib:OxigraphStore",
+        ]
+    },
     include_package_data=True,
 )

tests/test_dataset.py

@@ -44,7 +44,7 @@
 
 class DatasetTestCase(unittest.TestCase):
     def setUp(self):
-        self.graph = Dataset(store="OxMemory")
+        self.graph = Dataset(store="Oxigraph")
         self.michel = URIRef("urn:michel")
         self.tarek = URIRef("urn:tarek")
         self.bob = URIRef("urn:bob")
@@ -68,7 +68,6 @@ def tearDown(self):
         self.graph.close()
 
     def testGraphAware(self):
-
         if not self.graph.store.graph_aware:
             return
 

tests/test_graph.py

@@ -43,7 +43,7 @@
 
 class GraphTestCase(unittest.TestCase):
     def setUp(self):
-        self.graph = Graph(store="OxMemory")
+        self.graph = Graph(store="Oxigraph")
         self.michel = URIRef("urn:michel")
         self.tarek = URIRef("urn:tarek")
         self.bob = URIRef("urn:bob")

tests/test_graph_context.py

@@ -43,7 +43,7 @@
 
 class ContextTestCase(unittest.TestCase):
     def setUp(self):
-        self.graph = ConjunctiveGraph(store="OxMemory")
+        self.graph = ConjunctiveGraph(store="Oxigraph")
         self.michel = URIRef("urn:michel")
         self.tarek = URIRef("urn:tarek")
         self.bob = URIRef("urn:bob")

tests/test_sparql.py

@@ -8,7 +8,7 @@
 
 class SparqlTestCase(unittest.TestCase):
     def test_ask_query(self):
-        g = ConjunctiveGraph("OxMemory")
+        g = ConjunctiveGraph("Oxigraph")
         g.add((EX.foo, RDF.type, EX.Entity))
 
         # basic
@@ -23,13 +23,13 @@ def test_ask_query(self):
         self.assertFalse(g.query("ASK { ?s ?p ?o }", initBindings={"o": EX.NotExists}))
 
         # in specific graph
-        g = ConjunctiveGraph("OxMemory")
+        g = ConjunctiveGraph("Oxigraph")
         g1 = Graph(store=g.store, identifier=EX.g1)
         g1.add((EX.foo, RDF.type, EX.Entity))
         self.assertTrue(g1.query("ASK { ?s ?p ?o }"))
 
     def test_select_query_graph(self):
-        g = Graph("OxMemory")
+        g = Graph("Oxigraph")
         g.add((EX.foo, RDF.type, EX.Entity))
         result = g.query("SELECT ?s WHERE { ?s ?p ?o }")
         self.assertEqual(len(result), 1)
@@ -42,7 +42,7 @@ def test_select_query_graph(self):
         )
 
     def test_select_query_conjunctive(self):
-        g = ConjunctiveGraph("OxMemory")
+        g = ConjunctiveGraph("Oxigraph")
         g.add((EX.foo, RDF.type, EX.Entity))
         result = g.query("SELECT ?s WHERE { ?s ?p ?o }")
         self.assertEqual(len(result), 1)
@@ -55,7 +55,7 @@ def test_select_query_conjunctive(self):
         )
 
     def test_construct_query(self):
-        g = ConjunctiveGraph("OxMemory")
+        g = ConjunctiveGraph("Oxigraph")
         g.add((EX.foo, RDF.type, EX.Entity))
         result = g.query("CONSTRUCT WHERE { ?s ?p ?o }")
         self.assertEqual(len(result), 1)

tests/test_sled.py → tests/test_store.py

@@ -1,4 +1,4 @@
-import shutil
+import os
 import unittest
 
 from rdflib import RDF, XSD, BNode, ConjunctiveGraph, Graph, Literal, Namespace
@@ -7,26 +7,32 @@
 
 
 class StoreTestCase(unittest.TestCase):
-    def test_sled_store_default(self):
-        g = ConjunctiveGraph("OxSled")
+    def test_store_without_open(self):
+        g = ConjunctiveGraph("Oxigraph")
         self._fill_graph(g)
         self._test_graph(g)
         self.assertEqual(len(list(iter(g))), 4)
 
-    def test_sled_store_open(self):
-        g = ConjunctiveGraph("OxSled")
-        g.open("test_sled")
+    def test_store_with_open(self):
+        g = ConjunctiveGraph("Oxigraph")
+        g.open("test_store")
         self._fill_graph(g)
         g.close()
         del g
+        self.assertTrue(os.path.exists("test_store"))
 
-        g = ConjunctiveGraph("OxSled")
-        g.open("test_sled")
+        g = ConjunctiveGraph("Oxigraph")
+        g.open("test_store")
         self._test_graph(g)
         g.close()
-        del g
+        g.destroy("test_store")
+        self.assertFalse(os.path.exists("test_store"))
 
-        shutil.rmtree("test_sled")
+    def test_store_with_late_open(self):
+        g = ConjunctiveGraph("Oxigraph")
+        g.add((EX.foo, RDF.type, EX.Entity))
+        with self.assertRaises(Exception) as _:
+            g.open("test_store")
 
     def _fill_graph(self, g: Graph):
         g.add((EX.foo, RDF.type, EX.Entity))