[docker] Optionally connect to a running REST API container (#3)

* Update installation instructions

* Change behavior of search when no results found

Handle cases where wanting to connect to an existing docker service

* Change step name in workflow

* Updates to docs

* Correct import

* Updates to example

* Make doc, sentence, and fields hashable

* Test property access

* Pass token attributes at runtime

* Tests for Sentence + convenience methods

* Set ENV default for token attributes

* Fix for update endpoint

* Support for grammars

* Add /api/healthcheck endpoint

* PR template

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,6 @@
+## Summary of Changes
+
+
+### Related issues
+
+Resolves ???

.github/workflows/docker.yml

@@ -35,8 +35,8 @@ jobs:
         echo "app_version=$app_version"
         echo "commit=${{github.sha}}"
 
-  text_reading:
-    name: "docker image for text reading component"
+  rest:
+    name: "docker image for REST API"
     needs: [app_version]
     runs-on: ubuntu-latest
     steps:

README.md

@@ -100,5 +100,4 @@ Please see [CITATION.cff](./CITATION.cff)
 
 ## Authors
 
-- [Gus Hahn-Powell](https://parsertongue.org/about)
-- [Dane Bell](http://danebell.info)
+- [Gus Hahn-Powell](https://parsertongue.org/about)

app/ai/lum/odinson/rest/requests/GrammarRequest.scala

@@ -4,8 +4,9 @@ import play.api.libs.json._
 
 case class GrammarRequest(
   grammar: String,
-  pageSize: Option[Int] = None,
+  maxDocs: Option[Int] = None,
   allowTriggerOverlaps: Option[Boolean] = None,
+  metadataQuery: Option[String] = None,
   pretty: Option[Boolean] = None
 )
 

app/ai/lum/odinson/rest/utils/OdinsonConfigUtils.scala

@@ -0,0 +1,38 @@
+package ai.lum.odinson.rest.utils
+
+import ai.lum.common.ConfigFactory
+import scala.collection.JavaConverters._
+import com.typesafe.config.{ Config, ConfigRenderOptions, ConfigValueFactory }
+import ai.lum.common.ConfigUtils._
+import ai.lum.common.FileUtils._
+import ai.lum.odinson.{ Document => OdinsonDocument, StringField => OdinsonStringField }
+import ai.lum.odinson.utils.exceptions.OdinsonException
+import com.typesafe.config.Config
+import java.io.File
+
+object OdinsonConfigUtils {
+
+  /**
+    * Replaces odinson.compiler.allTokenFields with env var's ODINSON_TOKEN_ATTRIBUTES (if set) 
+    *
+    * @param config
+    */
+  def injectTokenAttributes(config: Config): Config = {
+    val ODINSON_TOKEN_ATTRIBUTES = "ODINSON_TOKEN_ATTRIBUTES"
+    val TARGET_CONFIG_PROPERTY = "odinson.compiler.allTokenFields"
+    //println(s"Looking for ${ODINSON_TOKEN_ATTRIBUTES}")
+    //println(config.getValue(TARGET_CONFIG_PROPERTY))
+    sys.env.get(ODINSON_TOKEN_ATTRIBUTES) match {
+      case None => 
+        println(s"${ODINSON_TOKEN_ATTRIBUTES} not set.  Using defaults")
+        config
+      case Some(envVar) =>
+        val tokenAttributes = envVar.split(",").toList.asJava
+        //println(s"overriding ${TARGET_CONFIG_PROPERTY} with ${ODINSON_TOKEN_ATTRIBUTES}=${envVar}")
+        config.withValue(
+          TARGET_CONFIG_PROPERTY,
+          ConfigValueFactory.fromIterable(tokenAttributes)
+        )
+    }
+  }
+}

app/controllers/OdinsonController.scala

@@ -11,6 +11,7 @@ import com.typesafe.config.{ Config, ConfigRenderOptions, ConfigValueFactory }
 import ai.lum.odinson.rest.BuildInfo
 import ai.lum.odinson.rest.requests._
 import ai.lum.odinson.rest.responses._
+import ai.lum.odinson.rest.utils.{ OdinsonConfigUtils }
 //import org.apache.lucene.document.{ Document => LuceneDocument }
 import org.apache.lucene.store.FSDirectory
 //import play.api.Configuration
@@ -29,7 +30,7 @@ import scala.concurrent.{ ExecutionContext, Future }
 
 @Singleton
 class OdinsonController @Inject() (
-  config: Config = ConfigFactory.load(),
+  _config: Config = ConfigFactory.load(),
   // playConfig: Configuration,
   cc: ControllerComponents
 )(
@@ -41,6 +42,7 @@ class OdinsonController @Inject() (
   import ai.lum.odinson.rest.utils.OdinsonDocumentUtils._
 
   // format: off
+  val config               = OdinsonConfigUtils.injectTokenAttributes(_config)
   val docsDir              = config.apply[File]  ("odinson.docsDir")
   val pageSize             = config.apply[Int]   ("odinson.pageSize")
   val posTagTokenField     = config.apply[String]("odinson.index.posTagTokenField")
@@ -203,6 +205,12 @@ class OdinsonController @Inject() (
     json.format(pretty)
   }
 
+  def healthcheck() = Action.async {
+    Future {
+      Ok(Json.toJson(200))
+    }
+  }
+
   def numDocs = Action.async {
     Future {
       ExtractorEngine.usingEngine(config) { engine =>
@@ -441,17 +449,23 @@ class OdinsonController @Inject() (
       // FIXME: replace .get with validation check
       val gr = request.body.asJson.get.as[GrammarRequest]
       val grammar = gr.grammar
-      val pageSize = gr.pageSize
+      val maxDocs = gr.maxDocs
+      val metadataQuery = gr.metadataQuery
       val allowTriggerOverlaps = gr.allowTriggerOverlaps.getOrElse(false)
       val pretty = gr.pretty
       try {
         // rules -> OdinsonQuery
-        val extractors = engine.ruleReader.compileRuleString(grammar)
+        val extractors = metadataQuery match {
+          case None => engine.ruleReader.compileRuleString(grammar)
+          case Some(raw) => 
+            val mq = engine.compiler.mkParentQuery(raw)
+            engine.compileRuleString(rules=grammar, metadataFilter=mq)
+        }
 
         val start = System.currentTimeMillis()
 
-        val maxSentences: Int = pageSize match {
-          case Some(ps) => ps
+        val maxSentences: Int = maxDocs match {
+          case Some(md) => md
           case None     => engine.numDocs()
         }
 

build.sbt

@@ -56,6 +56,7 @@ lazy val sharedDeps = {
       caffeine,
       "org.scalatest" %% "scalatest" % "3.2.8", //3.2.17",
       "com.typesafe.scala-logging" %%  "scala-logging" % "3.9.5",
+      // "com.typesafe" % "config" % "1.4.3",
       // "ch.qos.logback" %  "logback-classic" % "1.4.11",
       "org.json4s" %% "json4s-core" % json4sVersion,
       "ai.lum"        %% "common"               % "0.1.5",
@@ -135,8 +136,10 @@ lazy val packagerSettings = {
     dockerEnvVars ++= Map(
       "APP_VERSION" -> scala.util.Properties.envOrElse("APP_VERSION", "???"),
       "APPLICATION_SECRET" -> "this-is-not-a-secure-key-please-change-me",
-      // NOTE: bind mount odison dir to /data/odinson
+      // NOTE: bind mount odinson dir to /data/odinson
       "ODINSON_DATA_DIR" -> "/app/data/odinson",
+      // token attributes that should be searchable
+      "ODINSON_TOKEN_ATTRIBUTES" -> "raw,word,norm,lemma,tag,chunk,entity,incoming,outgoing",
       // NOTE: the expected min. RAM requirements
       "_JAVA_OPTIONS" -> "-Xmx2g -Dfile.encoding=UTF-8 -Dplay.server.pidfile.path=/dev/null"
     )

conf/routes

@@ -9,6 +9,9 @@ GET     /                               controllers.Default.redirect(to = "/api"
 GET     /api/buildinfo                  controllers.OdinsonController.buildInfo(pretty: Option[Boolean])
 GET     /api/config                     controllers.OdinsonController.configInfo(pretty: Option[Boolean])
 
+GET    /api/healthcheck                     controllers.OdinsonController.healthcheck()
+HEAD   /api/healthcheck                     controllers.OdinsonController.healthcheck()
+
 # API spec
 GET     /api                            controllers.OpenApiController.openAPI
 

docs/install.md

@@ -24,5 +24,7 @@ We publish releases in the form of docker images:
 ```bash
 # NOTE: replace <local/path/to/data> with the absolute path
 # to the location you want data to be written to on your machine
-docker run -it -p "8000:8000" -v "<local/path/to/data>:/app/data" "lumai/odinson-rest:latest"
+# ensure its permissions allow writing by the docker service
+# i.e., chmod 777 <local/path/to/data>
+docker run -it -p "9000:9000" -v "<local/path/to/data>:/app/data" "lumai/odinson-rest:latest"
 ```

docs/tutorial.md

@@ -19,6 +19,7 @@ Navigate to [localhost:9000/api](http://localhost:9000/api) to interactively exp
 We also provide a Python library as a simple way to build applications that interact with the Odinson REST API.  You can either connect to an existing odinson-rest service or launch one using docker.
 
 ### Launching and interacting with a service using docker
+
 ```python
 from lum.odinson.doc import Document, Fields
 from lum.odinson.rest.docker import DockerBasedOdinsonAPI
@@ -39,6 +40,20 @@ for res in engine.search(odinson_query="[lemma=be]"):
   for span in res.spans():
     print(f"{res.document_id} ({res.sentence_index}):  {span}")
 ```
+### Validating a rule
+
+
+```python
+from lum.odinson.rest.docker import DockerBasedOdinsonAPI
+
+engine = DockerBasedOdinsonAPI()
+# will return False
+engine.validate_rule("[")
+# will return True
+engine.validate_rule("[word=Gonzo]")
+
+engine.close()
+```
 
 <!-- ## API Endpoints and Examples
 

public/schema/odinson.yaml

@@ -139,6 +139,8 @@ paths:
           content:
             "application/json":
               schema:
+                # FIXME: this needs to be updated.
+                # see lum.odinson.rest.responses.GrammarResults
                 $ref: '#/components/schemas/OdinsonGrammarResults'
         '400':
           description: Syntax error in query.
@@ -411,7 +413,8 @@ paths:
           description: |
             An error message.
 
-  /api/update/document/{documentId}:
+  # /api/update/document/{documentId}:
+  /api/update/document:
     post:
       tags:
         - index
@@ -1082,7 +1085,7 @@ components:
     OdinsonGrammarRequest:
       type: object
       required:
-        - rules
+        - grammar
       properties:
         grammar:
           type: string
@@ -1097,7 +1100,9 @@ components:
                 pattern: |
                   trigger = [lemma=have]
                   subject  = >nsubj []
-        pageSize:
+          metadataQuery:
+            $ref: '#/components/schemas/MetadataQuery'
+        maxDocs:
           type: integer
           format: int32
           description: |