Merge pull request #61 from scossu/term_query

Term query
scossu · Apr 23, 2018 · 6572a4a · 6572a4a
2 parents 1ad6251 + 7201489
commit 6572a4a
Show file tree

Hide file tree

Showing 24 changed files with 566 additions and 9,830 deletions.
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.0.0a13
+1.0.0a14
diff --git a/docs/discovery.rst b/docs/discovery.rst
@@ -66,7 +66,23 @@ or a native Python method if applicable.
 Term Search
 -----------
 
-This feature has not yet been implemented. It is meant to provide a discovery
-tool based on simple term match, and possibly comparison. It should be more
-efficient and predictable than SPARQL.
+.. figure:: assets/lsup_term_search.png
+   :alt: LAKEsuperior Term Search Window
+
+   LAKEsuperior Term Search Window
+
+This feature provides a discovery tool focused on resource subjects and based
+on individual term match and comparison. It tends to be more manageable than
+SPARQL but also uses some SPARQL syntax for the terms.
+
+Multiple search conditions can be entered and processed with AND or OR logic.
+
+The obtained results are resource URIs relative to the endpoint.
+
+Please consult the search page itself for detailed instructions on how to enter
+query terms.
+
+The term search is also available via REST API. E.g.::
+
+    curl -i -XPOST http://localhost:8000/query/term_search -d '{"terms": [{"pred": "rdf:type", "op": "_id", "val": "ldp:Container"}], "logic": "and"}' -H'Content-Type:application/json'
 
diff --git a/lakesuperior/api/query.py b/lakesuperior/api/query.py
@@ -2,6 +2,8 @@
 
 from io import BytesIO
 
+from rdflib import URIRef
+
 from lakesuperior import env
 from lakesuperior.dictionaries.namespaces import ns_collection as nsc
 from lakesuperior.dictionaries.namespaces import ns_mgr as nsm
@@ -12,6 +14,103 @@
 rdfly = env.app_globals.rdfly
 rdf_store = env.app_globals.rdf_store
 
+operands = ('_id', '=', '!=', '<', '>', '<=', '>=')
+"""
+Available term comparators for term query.
+
+The ``_uri`` term is used to match URIRef terms, all other comparators are
+used against literals.
+"""
+
+
+def triple_match(s=None, p=None, o=None, return_full=False):
+    """
+    Query store by matching triple patterns.
+
+    Any of the ``s``, ``p`` or ``o`` terms can be None to represent a wildcard.
+
+    This method is for triple matching only; it does not allow to query, nor
+    exposes to the caller, any context.
+
+    :param rdflib.term.Identifier s: Subject term.
+    :param rdflib.term.Identifier p: Predicate term.
+    :param rdflib.term.Identifier o: Object term.
+    :param bool return_full: if ``False`` (the default), the returned values
+        in the set are the URIs of the resources found. If True, the full set
+        of matching triples is returned.
+
+    :rtype: set(tuple(rdflib.term.Identifier){3}) or set(rdflib.URIRef)
+    :return: Matching resource URIs if ``return_full`` is false, or
+        matching triples otherwise.
+    """
+    with TxnManager(rdf_store) as txn:
+        matches = rdf_store.triples((s, p, o), None)
+        # Strip contexts and de-duplicate.
+        qres = (
+            {match[0] for match in matches} if return_full
+            else {match[0][0] for match in matches})
+
+    return qres
+
+
+def term_query(terms, or_logic=False):
+    """
+    Query resources by predicates, comparators and values.
+
+    Comparators can be against literal or URIRef objects. For a list of
+    comparators and their meanings, see the documentation and source for
+    :py:data:`~lakesuperior.api.query.operands`.
+
+    :param list(tuple{3}) terms: List of 3-tuples containing:
+
+        - Predicate URI (rdflib.URIRef)
+        - Comparator value (str)
+        - Value to compare to (rdflib.URIRef or rdflib.Literal or str)
+
+    :param bool or_logic: Whether to concatenate multiple query terms with OR
+        logic (uses SPARQL ``UNION`` statements). The default is False (i.e.
+        terms are concatenated as standard SPARQL statements).
+    """
+    qry_term_ls = []
+    for i, term in enumerate(terms):
+        if term['op'] not in operands:
+            raise ValueError('Not a valid operand: {}'.format(term['op']))
+
+        if term['op'] == '_id':
+            qry_term = '?s {} {} .'.format(term['pred'], term['val'])
+        else:
+            oname = '?o_{}'.format(i)
+            qry_term = '?s {0} {1}\nFILTER (str({1}) {2} "{3}") .'.format(
+                    term['pred'], oname, term['op'], term['val'])
+
+        qry_term_ls.append(qry_term)
+
+    if or_logic:
+        qry_terms = '{\n' + '\n} UNION {\n'.join(qry_term_ls) + '\n}'
+    else:
+        qry_terms = '\n'.join(qry_term_ls)
+    qry_str = '''
+    SELECT ?s WHERE {{
+      {}
+    }}
+    '''.format(qry_terms)
+    logger.debug('Query: {}'.format(qry_str))
+
+    with TxnManager(rdf_store) as txn:
+        qres = rdfly.raw_query(qry_str)
+        return {row[0] for row in qres}
+
+
+def fulltext_lookup(pattern):
+    """
+    Look up one term by partial match.
+
+    *TODO: reserved for future use. A `Whoosh
+    <https://whoosh.readthedocs.io/>`__ or similar full-text index is
+    necessary for this.*
+    """
+    pass
+
 
 def sparql_query(qry_str, fmt):
     """

diff --git a/lakesuperior/dictionaries/namespaces.py b/lakesuperior/dictionaries/namespaces.py
@@ -34,7 +34,8 @@
 }
 
 ns_collection = core_namespaces.copy()
-ns_collection.update(config['namespaces'])
+custom_ns = {pfx: Namespace(ns) for pfx, ns in config['namespaces'].items()}
+ns_collection.update(custom_ns)
 
 ns_mgr = NamespaceManager(Graph())
 ns_pfx_sparql = {}

diff --git a/lakesuperior/endpoints/query.py b/lakesuperior/endpoints/query.py
@@ -1,11 +1,16 @@
 import logging
 
-from flask import Blueprint, current_app, request, render_template, send_file
+from flask import (
+        Blueprint, current_app, jsonify, request, make_response,
+        render_template, send_file)
+from rdflib import URIRef
 from rdflib.plugin import PluginException
 
 from lakesuperior import env
-from lakesuperior.dictionaries.namespaces import ns_mgr as nsm
 from lakesuperior.api import query as query_api
+from lakesuperior.dictionaries.namespaces import ns_collection as nsc
+from lakesuperior.dictionaries.namespaces import ns_mgr as nsm
+from lakesuperior.toolbox import Toolbox
 
 # Query endpoint. raw SPARQL queries exposing the underlying layout can be made
 # available. Also convenience methods that allow simple lookups based on simple
@@ -18,24 +23,37 @@
 query = Blueprint('query', __name__)
 
 
-@query.route('/term_search', methods=['GET'])
+@query.route('/term_search', methods=['GET', 'POST'])
 def term_search():
     """
     Search by entering a search term and optional property and comparison term.
     """
-    valid_operands = (
-        ('=', 'Equals'),
-        ('>', 'Greater Than'),
-        ('<', 'Less Than'),
-        ('<>', 'Not Equal'),
-        ('a', 'RDF Type'),
+    operands = (
+        ('_id', 'Matches Term'),
+        ('=', 'Is Equal To'),
+        ('!=', 'Is Not Equal To'),
+        ('<', 'Is Less Than'),
+        ('>', 'Is Greater Than'),
+        ('<=', 'Is Less Than Or Equal To'),
+        ('>=', 'Is Greater Than Or Equal To'),
     )
+    qres = term_list = []
 
-    term = request.args.get('term')
-    prop = request.args.get('prop', default=1)
-    cmp = request.args.get('cmp', default='=')
+    if request.method == 'POST':
+        terms = request.json.get('terms', {})
+        or_logic = request.json.get('logic', 'and') == 'or'
+        logger.info('Form: {}'.format(request.json))
+        logger.info('Terms: {}'.format(terms))
+        logger.info('Logic: {}'.format(or_logic))
+        qres = query_api.term_query(terms, or_logic)
 
-    return render_template('term_search.html')
+        rsp = [
+            uri.replace(nsc['fcres'], request.host_url.rstrip('/') + '/ldp')
+            for uri in qres]
+        return jsonify(rsp), 200
+    else:
+        return render_template(
+            'term_search.html', operands=operands, qres=qres, nsm=nsm)
 
 
 @query.route('/sparql', methods=['GET', 'POST'])

diff --git a/lakesuperior/endpoints/templates/base.html b/lakesuperior/endpoints/templates/base.html
@@ -14,49 +14,16 @@
     {% endblock %}
     </head>
     <body>
-    {% block navbar%}
-        <nav class="navbar navbar-default">
-          <div class="container-fluid">
-            <!-- Brand and toggle get grouped for better mobile display -->
-            <div class="navbar-header">
-              <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false">
-                <span class="sr-only">Toggle navigation</span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-                <span class="icon-bar"></span>
-              </button>
-              <a class="navbar-brand" href="/">LAKEsuperior</a>
-            </div>
-            <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
-              <ul class="nav navbar-nav">
-                <li><a href="/ldp">Browse Resources<span class="sr-only">(current)</span></a></li>
-                <li class="dropdown">
-                  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Query<span class="caret"></span></a>
-                  <ul class="dropdown-menu">
-                    <li><a href="/query/term_search">Term Search</a></li>
-                    <li><a href="/query/sparql">SPARQL Query</a></li>
-                  </ul>
-                </li>
-                <li class="dropdown">
-                  <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Administration<span class="caret"></span></a>
-                  <ul class="dropdown-menu">
-                    <li><a href="/admin/stats">Statistics</a></li>
-                    <li><a href="/admin/tools">Tools</a></li>
-                  </ul>
-                </li>
-              </ul>
-            </div><!-- /.navbar-collapse -->
-          </div><!-- /.container-fluid -->
-        </nav>
-    {% endblock %}
-        <main class="container">
+        {% include 'navbar.html' %}
+        <div class="container">
             {% block breadcrumbs %}{% endblock %}
             <h1>{{ self.title() }}</h1>
             {% block content %}{% endblock %}
-        </main>
+        </div>
         <!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
         <script src="{{url_for('ldp.static', filename='assets/js/jquery-3.2.1.min.js')}}"></script>
         <!-- Include all compiled plugins (below), or include individual files as needed -->
-        <script src="{{url_for('ldp.static', filename='assets/js/bootstrap.min.js')}}"></script>
+        <script src="{{url_for('ldp.static', filename='assets/js/bootstrap.bundle.min.js')}}"></script>
+        {% block tail_js %}{% endblock %}
     </body>
 </html>
diff --git a/lakesuperior/endpoints/templates/namespaces.html b/lakesuperior/endpoints/templates/namespaces.html
@@ -1,7 +1,9 @@
 <h2>Namespaces</h2>
- <button class="btn btn-primary" type="button" data-toggle="collapse" data-target="#nslist" aria-expanded="false" aria-controls="nsList">
+<div class="my-sm-3">
+  <button class="btn btn-primary" type="button" data-toggle="collapse" data-target="#nslist" aria-expanded="false" aria-controls="nsList">
     Expand/Collapse
-</button>
+  </button>
+</div>
 <div class="collapse" id="nslist">
     <div class="card card-body">
         <table class="table table-striped">

diff --git a/lakesuperior/endpoints/templates/navbar.html b/lakesuperior/endpoints/templates/navbar.html
@@ -0,0 +1,32 @@
+<nav class="navbar navbar-expand-lg navbar-light bg-light">
+  <a class="navbar-brand" href="/">LAKEsuperior</a>
+  <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
+    <span class="navbar-toggler-icon"></span>
+  </button>
+
+  <div class="collapse navbar-collapse" id="navbarSupportedContent">
+    <ul class="navbar-nav mr-auto">
+      <li class="nav-item">
+        <a class="nav-link" href="/ldp">Browse Resources<span class="sr-only">(current)</span></a>
+      </li>
+      <li class="nav-item dropdown">
+        <a class="nav-link dropdown-toggle" href="#" id="navbarDropdown" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
+          Query
+        </a>
+        <div class="dropdown-menu" aria-labelledby="navbarDropdown">
+          <a class="dropdown-item" href="/query/term_search">Term Search</a>
+          <a class="dropdown-item" href="/query/sparql">SPARQL Query</a>
+        </div>
+      </li>
+      <li class="nav-item dropdown">
+        <a class="nav-link dropdown-toggle" href="#" id="navbarDropdown" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
+          Administration
+        </a>
+        <div class="dropdown-menu" aria-labelledby="navbarDropdown">
+          <a class="dropdown-item" href="/admin/stats">Statistics</a>
+          <a class="dropdown-item" href="/admin/tools">Tools</a>
+        </div>
+      </li>
+    </ul>
+  </div><!-- /#navbarSupportedContent -->
+</nav>
diff --git a/lakesuperior/endpoints/templates/resource.html b/lakesuperior/endpoints/templates/resource.html
@@ -40,7 +40,7 @@
 ({{updated_ts.humanize() }})</p>
 <p><strong>Types:</strong>
 {% for t in gr[gr.identifier : nsc['rdf'].type :] | sort %}
-    <span class="label label-primary">{{ t.n3(namespace_manager=nsm) }}</span>
+    <span class="badge badge-info">{{ t.n3(namespace_manager=nsm) }}</span>
 {% endfor %}
 </p>
 <h2>Properties</h2>
@@ -74,7 +74,7 @@ <h2>Properties</h2>
             {% if 'Literal' in t[2].__class__.__name__ %}
             "{{ t[2] }}"
             {% if t[2].datatype %}
-                <span class="label label-primary">
+                <span class="badge badge-info">
                     {{ t[2].datatype.n3(namespace_manager=nsm) }}
                 </span>
             {% endif %}
@@ -117,7 +117,7 @@ <h2>Other subjects</h2>
             {% if 'Literal' in t[2].__class__.__name__ %}
             "{{ t[2] }}"
             {% if t[2].datatype %}
-            <span class="label label-primary">{{ t[2].datatype.n3(namespace_manager=nsm) }}</span>
+            <span class="badge badge-info">{{ t[2].datatype.n3(namespace_manager=nsm) }}</span>
             {% endif %}
             {% else %}
                 <a href="{{ t[2] }}">
@@ -130,31 +130,5 @@ <h2>Other subjects</h2>
     {% endfor %}
     </tbody>
 </table>
-{% block namespaces %}
-            <h2>Namespaces</h2>
-             <button class="btn btn-primary" type="button" data-toggle="collapse" data-target="#nslist" aria-expanded="false" aria-controls="nsList">
-                Expand/Collapse
-            </button>
-            <div class="collapse" id="nslist">
-                <div class="card card-body">
-                    <table class="table table-striped">
-                        <thead>
-                            <tr>
-                                <td>Prefix</td>
-                                <td>URI</td>
-                            </tr>
-                        </thead>
-                        <tbody>
-                        {% for ns in nsm.namespaces() | sort %}
-                            <tr>
-                                <td>{{ ns[0] }}</td>
-                                <td>{{ ns[1] }}</td>
-                            </tr>
-                        {% endfor %}
-                        </tbody>
-                    </table>
-                </div>
-            </div>
-
-{% endblock %}
+{% include 'namespaces.html' %}
 {% endblock %}