diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b168560..2420a76 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,7 +28,7 @@ repos: hooks: - id: isort - repo: https://github.com/asottile/pyupgrade - rev: v2.31.0 + rev: v2.31.1 hooks: - id: pyupgrade args: [ --py37-plus ] @@ -41,7 +41,7 @@ repos: hooks: - id: flake8 - repo: https://github.com/PyCQA/bandit - rev: 1.7.2 + rev: 1.7.4 hooks: - id: bandit - repo: https://github.com/PyCQA/pydocstyle @@ -49,6 +49,6 @@ repos: hooks: - id: pydocstyle - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v0.931' + rev: v0.941 hooks: - id: mypy diff --git a/README.md b/README.md index 719acbf..aa4e0be 100644 --- a/README.md +++ b/README.md @@ -7,31 +7,31 @@ Oxrdflib [![actions status](https://github.com/oxigraph/oxrdflib/workflows/build/badge.svg)](https://github.com/oxigraph/oxrdflib/actions) [![Gitter](https://badges.gitter.im/oxigraph/community.svg)](https://gitter.im/oxigraph/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) -Oxrdflib provides [rdflib](https://rdflib.readthedocs.io/) stores using [pyoxigraph](https://oxigraph.org/pyoxigraph/). +Oxrdflib provides an [rdflib](https://rdflib.readthedocs.io/) store based on [pyoxigraph](https://oxigraph.org/pyoxigraph/). +This store is named `"Oxigraph"`. -The stores could be used as drop-in replacements of the rdflib default ones. They support context but not formulas. +This store can be used as drop-in replacement of the rdflib default one. It support context but not formulas. Transaction support is not implemented yet. -SPARQL query evaluation is done by pyoxigraph instead of rdflib if an oxrdflib store is used. - -Two stores are currently provided: -* An in-memory store, named `"OxMemory"`. -* A disk-based store based on the [Sled key-value store](https://sled.rs/), named `"OxSled"`. +SPARQL query evaluation is done by pyoxigraph instead of rdflib if the Oxigraph store is used. +SPARQL update evaluation is still done using rdflib because of [a limitation in rdflib context management](https://github.com/RDFLib/rdflib/issues/1396). Oxrdflib is [available on Pypi](https://pypi.org/project/oxrdflib/) and installable with: ```bash pip install oxrdflib ``` -The oxrdflib stores are automatically registered as rdflib store plugins by setuptools. +The oxrdflib store is automatically registered as an rdflib store plugin by setuptools. -## API +*Warning:* Oxigraph is not stable yet and its storage format might change in the future. +To migrate to future version you might have to dump and load the store content. +However, Oxigraph should be in a good enough shape to power most of use cases if you are not afraid of down time and data loss. -### `"OxMemory"`, an in-memory store +## API -To create a rdflib graph with pyoxigraph in memory store use +To create a rdflib graph using the Oxigraph store use ```python -rdflib.Graph(store="OxMemory") +rdflib.Graph(store="Oxigraph") ``` instead of the usual ```python @@ -40,34 +40,39 @@ rdflib.Graph() Similarly, to get a conjunctive graph, use ```python -rdflib.ConjunctiveGraph(store="OxMemory") +rdflib.ConjunctiveGraph(store="Oxigraph") ``` instead of the usual ```python rdflib.ConjunctiveGraph() ``` +and to get a dataset, use -### `"OxSled"`, a disk-based store - -The disk-based store is based on the [Sled key-value store](https://sled.rs/). -Sled is not stable yet and its storage system might change in the future. +```python +rdflib.Dataset(store="Oxigraph") +``` +instead of the usual +```python +rdflib.Dataset() +``` -To open Sled based graph in the directory `test_dir` use +If you want to get the store data persisted on disk, use the `open` method on the `Graph` object (or `ConjunctiveGraph` or `Dataset`) with the directory where data should be persisted. For example: ```python -graph = rdflib.Graph(store="OxSled") +graph = rdflib.Graph(store="Oxigraph") graph.open("test_dir") ``` The store is closed with the `close()` method or automatically when Python garbage collector collects the store object. -It is also possible to not provide a directory name. -In this case, a temporary directory will be created and deleted when the store is closed. -For example, this code uses a temporary directory: -```python -rdflib.Graph(store="OxSled") -``` +If the `open` method is not called Oxigraph will automatically use a ramdisk on Linux and a temporary file in the other operating systems. + +To do anything else, use the usual rdflib python API. -`rdflib.ConjunctiveGraph` is also usable with `"OxSled"`. +## Migration guide +### From 0.2 to 0.3 +* The 0.2 stores named `"OxSled"` and `"OxMemory"` have been merged into the `"Oxigraph"` store. +* The on-disk storage system provided by `"OxSled"` has been dropped and replaced by a new storage system based on [RocksDB](https://rocksdb.org/). + To migrate you need to first dump your data in RDF using `oxrdflib` 0.2 and the `serialize` method, then upgrade to `oxrdflib` 0.3, and finally reload the data using the `parse` method. ## Development diff --git a/oxrdflib/__init__.py b/oxrdflib/__init__.py index 4e795cc..6ced35e 100644 --- a/oxrdflib/__init__.py +++ b/oxrdflib/__init__.py @@ -1,4 +1,4 @@ -from abc import ABC, abstractmethod +import shutil import pyoxigraph as ox from rdflib import Graph @@ -7,23 +7,40 @@ from rdflib.store import VALID_STORE, Store from rdflib.term import BNode, Literal, Node, URIRef, Variable -__all__ = ["MemoryOxStore", "SledOxStore"] +__all__ = ["OxigraphStore"] -class _BaseOxStore(Store, ABC): +class OxigraphStore(Store): context_aware = True formula_aware = False transaction_aware = False graph_aware = True - @property - @abstractmethod - def _inner(self): - pass + def __init__(self, configuration=None, identifier=None): + self._store = None + super().__init__(configuration, identifier) + + def open(self, configuration, create=False): + if self._store is not None: + raise ValueError("The open function should be called before any RDF operation") + self._store = ox.Store(configuration) + return VALID_STORE + + def close(self, commit_pending_transaction=False): + del self._store + + def destroy(self, configuration): + shutil.rmtree(configuration) def gc(self): pass + @property + def _inner(self): + if self._store is None: + self._store = ox.Store() + return self._store + def add(self, triple, context, quoted=False): if quoted: raise ValueError("Oxigraph stores are not formula aware") @@ -96,38 +113,6 @@ def remove_graph(self, graph): self._inner.remove_graph(_to_ox(graph)) -class MemoryOxStore(_BaseOxStore): - def __init__(self, configuration=None, identifier=None): - self._store = ox.MemoryStore() - super().__init__(configuration, identifier) - - @property - def _inner(self): - return self._store - - -class SledOxStore(_BaseOxStore): - def __init__(self, configuration=None, identifier=None): - self._store = None - super().__init__(configuration, identifier) - - def open(self, configuration, create=False): - self._store = ox.SledStore(configuration) - return VALID_STORE - - def close(self, commit_pending_transaction=False): - del self._store - - def destroy(self, configuration): - raise NotImplementedError("destroy is not implemented yet for the Sled based store") - - @property - def _inner(self): - if self._store is None: - self._store = ox.SledStore() - return self._store - - def _to_ox(term, context=None): if term is None: return None diff --git a/setup.py b/setup.py index e0a7baf..8cf3f7b 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setup( name="oxrdflib", - version="0.2.0", + version="0.3.0", description="rdflib stores based on pyoxigraph", long_description=(pathlib.Path(__file__).parent / "README.md").read_text(), long_description_content_type="text/markdown", @@ -31,7 +31,13 @@ "Tracker": "https://github.com/oxigraph/oxrdflib/issues", }, packages=["oxrdflib"], - install_requires=["pyoxigraph~=0.2.0", "rdflib~=6.0"], - entry_points={"rdf.plugins.store": ["OxMemory = oxrdflib:MemoryOxStore", "OxSled = oxrdflib:SledOxStore"]}, + install_requires=["pyoxigraph~=0.3.0", "rdflib~=6.0"], + entry_points={ + "rdf.plugins.store": [ + "Oxigraph = oxrdflib:OxigraphStore", + "OxMemory = oxrdflib:OxigraphStore", + "OxSled = oxrdflib:OxigraphStore", + ] + }, include_package_data=True, ) diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 6ac7f6e..d68d7f9 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -44,7 +44,7 @@ class DatasetTestCase(unittest.TestCase): def setUp(self): - self.graph = Dataset(store="OxMemory") + self.graph = Dataset(store="Oxigraph") self.michel = URIRef("urn:michel") self.tarek = URIRef("urn:tarek") self.bob = URIRef("urn:bob") @@ -68,7 +68,6 @@ def tearDown(self): self.graph.close() def testGraphAware(self): - if not self.graph.store.graph_aware: return diff --git a/tests/test_graph.py b/tests/test_graph.py index 50bd5f2..f20e680 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -43,7 +43,7 @@ class GraphTestCase(unittest.TestCase): def setUp(self): - self.graph = Graph(store="OxMemory") + self.graph = Graph(store="Oxigraph") self.michel = URIRef("urn:michel") self.tarek = URIRef("urn:tarek") self.bob = URIRef("urn:bob") diff --git a/tests/test_graph_context.py b/tests/test_graph_context.py index b33589c..d058f16 100644 --- a/tests/test_graph_context.py +++ b/tests/test_graph_context.py @@ -43,7 +43,7 @@ class ContextTestCase(unittest.TestCase): def setUp(self): - self.graph = ConjunctiveGraph(store="OxMemory") + self.graph = ConjunctiveGraph(store="Oxigraph") self.michel = URIRef("urn:michel") self.tarek = URIRef("urn:tarek") self.bob = URIRef("urn:bob") diff --git a/tests/test_sparql.py b/tests/test_sparql.py index 9300577..5410fa9 100644 --- a/tests/test_sparql.py +++ b/tests/test_sparql.py @@ -8,7 +8,7 @@ class SparqlTestCase(unittest.TestCase): def test_ask_query(self): - g = ConjunctiveGraph("OxMemory") + g = ConjunctiveGraph("Oxigraph") g.add((EX.foo, RDF.type, EX.Entity)) # basic @@ -23,13 +23,13 @@ def test_ask_query(self): self.assertFalse(g.query("ASK { ?s ?p ?o }", initBindings={"o": EX.NotExists})) # in specific graph - g = ConjunctiveGraph("OxMemory") + g = ConjunctiveGraph("Oxigraph") g1 = Graph(store=g.store, identifier=EX.g1) g1.add((EX.foo, RDF.type, EX.Entity)) self.assertTrue(g1.query("ASK { ?s ?p ?o }")) def test_select_query_graph(self): - g = Graph("OxMemory") + g = Graph("Oxigraph") g.add((EX.foo, RDF.type, EX.Entity)) result = g.query("SELECT ?s WHERE { ?s ?p ?o }") self.assertEqual(len(result), 1) @@ -42,7 +42,7 @@ def test_select_query_graph(self): ) def test_select_query_conjunctive(self): - g = ConjunctiveGraph("OxMemory") + g = ConjunctiveGraph("Oxigraph") g.add((EX.foo, RDF.type, EX.Entity)) result = g.query("SELECT ?s WHERE { ?s ?p ?o }") self.assertEqual(len(result), 1) @@ -55,7 +55,7 @@ def test_select_query_conjunctive(self): ) def test_construct_query(self): - g = ConjunctiveGraph("OxMemory") + g = ConjunctiveGraph("Oxigraph") g.add((EX.foo, RDF.type, EX.Entity)) result = g.query("CONSTRUCT WHERE { ?s ?p ?o }") self.assertEqual(len(result), 1) diff --git a/tests/test_sled.py b/tests/test_store.py similarity index 61% rename from tests/test_sled.py rename to tests/test_store.py index 0009ef1..a2cdf5c 100644 --- a/tests/test_sled.py +++ b/tests/test_store.py @@ -1,4 +1,4 @@ -import shutil +import os import unittest from rdflib import RDF, XSD, BNode, ConjunctiveGraph, Graph, Literal, Namespace @@ -7,26 +7,32 @@ class StoreTestCase(unittest.TestCase): - def test_sled_store_default(self): - g = ConjunctiveGraph("OxSled") + def test_store_without_open(self): + g = ConjunctiveGraph("Oxigraph") self._fill_graph(g) self._test_graph(g) self.assertEqual(len(list(iter(g))), 4) - def test_sled_store_open(self): - g = ConjunctiveGraph("OxSled") - g.open("test_sled") + def test_store_with_open(self): + g = ConjunctiveGraph("Oxigraph") + g.open("test_store") self._fill_graph(g) g.close() del g + self.assertTrue(os.path.exists("test_store")) - g = ConjunctiveGraph("OxSled") - g.open("test_sled") + g = ConjunctiveGraph("Oxigraph") + g.open("test_store") self._test_graph(g) g.close() - del g + g.destroy("test_store") + self.assertFalse(os.path.exists("test_store")) - shutil.rmtree("test_sled") + def test_store_with_late_open(self): + g = ConjunctiveGraph("Oxigraph") + g.add((EX.foo, RDF.type, EX.Entity)) + with self.assertRaises(Exception) as _: + g.open("test_store") def _fill_graph(self, g: Graph): g.add((EX.foo, RDF.type, EX.Entity))