Merge pull request #627 from snipsco/release/0.16.0

Release 0.16.0

.appveyor.yml

@@ -23,12 +23,19 @@ install:
 build: false
 
 test_script:
-  - "%PYTHON%\\python.exe -m snips_nlu download de"
-  - "%PYTHON%\\python.exe -m snips_nlu download en"
-  - "%PYTHON%\\python.exe -m snips_nlu download es"
-  - "%PYTHON%\\python.exe -m snips_nlu download fr"
-  - "%PYTHON%\\python.exe -m snips_nlu download ja"
-  - "%PYTHON%\\python.exe -m snips_nlu download ko"
-  - "%PYTHON%\\python.exe -m snips_nlu download zh"
+  - "%PYTHON%\\python.exe -m snips_nlu download snips_nlu_de-0.2.0 --direct"
+  - "%PYTHON%\\python.exe -m snips_nlu download snips_nlu_en-0.2.0 --direct"
+  - "%PYTHON%\\python.exe -m snips_nlu download snips_nlu_es-0.2.0 --direct"
+  - "%PYTHON%\\python.exe -m snips_nlu download snips_nlu_fr-0.2.0 --direct"
+  - "%PYTHON%\\python.exe -m snips_nlu download snips_nlu_ja-0.2.0 --direct"
+  - "%PYTHON%\\python.exe -m snips_nlu download snips_nlu_ko-0.2.0 --direct"
+  - "%PYTHON%\\python.exe -m snips_nlu download snips_nlu_zh-0.2.0 --direct"
+  - "%PYTHON%\\python.exe -m snips_nlu link snips_nlu_de de"
+  - "%PYTHON%\\python.exe -m snips_nlu link snips_nlu_en en"
+  - "%PYTHON%\\python.exe -m snips_nlu link snips_nlu_es es"
+  - "%PYTHON%\\python.exe -m snips_nlu link snips_nlu_fr fr"
+  - "%PYTHON%\\python.exe -m snips_nlu link snips_nlu_ja ja"
+  - "%PYTHON%\\python.exe -m snips_nlu link snips_nlu_ko ko"
+  - "%PYTHON%\\python.exe -m snips_nlu link snips_nlu_zh zh"
   - "%PYTHON%\\python.exe -m unittest discover"
   - "%PYTHON%\\python.exe snips_nlu_samples/sample.py"

.gitignore

@@ -8,6 +8,7 @@
 /venv
 /venv2
 /venv3
+/venv35
 /venv36
 /.tox
 .coverage

CHANGELOG.md

@@ -1,6 +1,23 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
+## [0.16.0] - 2018-07-17
+### Changed
+- The `SnipsNLUEngine` object is now persisted to (and loaded from) a 
+directory, instead of a single json file.
+- The language resources are now persisted along with the `SnipsNLUEngine`, 
+removing the need to download and load the resources when loading a trained engine.
+- The format of language resources has been optimized.
+
+### Added
+- Stemmed gazetteers, computed beforehand. It removes the need to stem 
+gazetteers on the fly.
+- API to persist (and load) a `SnipsNLUEngine` object as a `bytearray`
+
+### Fixed
+- Issue in the `DeterministicIntentParser` when the same slot name was used in 
+multiple intents while referring to different entities
+
 ## [0.15.1] - 2018-07-09
 ### Changed
 - Bump `snips-nlu-ontology` to `0.57.1`
@@ -92,6 +109,7 @@ several commands.
 - Fix compiling issue with `bindgen` dependency when installing from source
 - Fix issue in `CRFSlotFiller` when handling builtin entities
 
+[0.16.0]: https://github.com/snipsco/snips-nlu/compare/0.15.1...0.16.0
 [0.15.1]: https://github.com/snipsco/snips-nlu/compare/0.15.0...0.15.1
 [0.15.0]: https://github.com/snipsco/snips-nlu/compare/0.14.0...0.15.0
 [0.14.0]: https://github.com/snipsco/snips-nlu/compare/0.13.5...0.14.0

README.rst

@@ -29,7 +29,7 @@ Installation
     pip install snips-nlu
 
 We currently have pre-built binaries (wheels) for ``snips-nlu`` and its
-dependencies for MacOS, Linux x86_64 and Windows 64-bit.
+dependencies for MacOS (10.11 and later), Linux x86_64 and Windows.
 
 For any other architecture/os `snips-nlu` can be installed from the source
 distribution. To do so, `Rust <https://www.rust-lang.org/en-US/install.html>`_

debug/debug.py

@@ -34,11 +34,7 @@ def debug_training(dataset_path, config_path=None):
 
 
 def debug_inference(engine_path):
-    with Path(engine_path).open("r", encoding="utf8") as f:
-        engine_dict = json.load(f)
-
-    load_resources(engine_dict["dataset_metadata"]["language_code"])
-    engine = SnipsNLUEngine.from_dict(engine_dict)
+    engine = SnipsNLUEngine.from_path(engine_path)
 
     while True:
         query = input("Enter a query (type 'q' to quit): ").strip()

docs/source/data_model.rst

@@ -6,7 +6,9 @@ Key Concepts & Data Model
 This section is meant to explain the concepts and data model that we use to
 represent input and output data.
 
-The main task that this lib performs is *Information Extraction*, or *Intent Parsing*, to be even more specific. At this point, the output of the engine may still not be very clear to you.
+The main task that this lib performs is *Information Extraction*, or
+*Intent Parsing*, to be even more specific. At this point, the output of the
+engine may still not be very clear to you.
 
 The task of parsing intents is actually two-folds. The first step is to
 understand which intent the sentence is about. The second step is

docs/source/installation.rst

@@ -24,9 +24,9 @@ Snips NLU can be installed via pip with the following command:
 
 
 We currently have pre-built binaries (wheels) for ``snips-nlu`` and its
-dependencies for MacOS and Linux x86_64. If you use a different
-architecture/os you will need to build these dependencies from sources
-which means you will need to install
+dependencies for MacOS (10.11 and later), Linux x86_64 and Windows 64-bit. If
+you use a different architecture/os you will need to build these dependencies
+from sources which means you will need to install
 `setuptools_rust <https://github.com/PyO3/setuptools-rust>`_ and
 `Rust <https://www.rust-lang.org/en-US/install.html>`_ before running the
 ``pip install snips-nlu`` command.

docs/source/quickstart.rst

@@ -110,4 +110,4 @@ You should get something that looks like this:
 Congrats, you parsed your first intent!
 
 
-.. _sample dataset: https://github.com/snipsco/snips-nlu/blob/master/samples/sample_dataset.json
+.. _sample dataset: https://github.com/snipsco/snips-nlu/blob/master/snips_nlu_samples/sample_dataset.json

docs/source/tutorial.rst

@@ -333,33 +333,34 @@ engine we previously created:
 Persisting
 ----------
 
-As a final step, we will persist the engine in a json. That may be useful in
-various contexts, for instance if you want to train on a machine and parse on
-another one.
+As a final step, we will persist the engine into a directory. That may be
+useful in various contexts, for instance if you want to train on a machine and
+parse on another one.
 
 You can persist the engine with the following API:
 
 .. code-block:: python
 
-    engine_json = json.dumps(engine.to_dict())
-    with io.open("trained_engine.json", mode="w") as f:
-        # f.write(engine_json.decode("utf8"))  # Python 2
-        f.write(engine_json)  # Python 3
+    engine.persist("path/to/directory")
 
 
 And load it:
 
 .. code-block:: python
 
+    loaded_engine = SnipsNLUEngine.from_path("path/to/directory")
 
-    with io.open("trained_engine.json") as f:
-        engine_dict = json.load(f)
+    loaded_engine.parse(u"Turn lights on in the bathroom please")
 
-    loaded_engine = SnipsNLUEngine.from_dict(engine_dict)
 
-    loaded_engine.parse(u"Turn lights on in the bathroom please")
+Alternatively, you can persist/load the engine as a ``bytearray``:
+
+.. code-block:: python
+
+    engine_bytes = engine.to_byte_array()
+    loaded_engine = SnipsNLUEngine.from_byte_array(engine_bytes)
 
 
-.. _sample dataset: https://github.com/snipsco/snips-nlu/blob/master/samples/sample_dataset.json
+.. _sample dataset: https://github.com/snipsco/snips-nlu/blob/master/snips_nlu_samples/sample_dataset.json
 .. _default configurations: https://github.com/snipsco/snips-nlu/blob/master/snips_nlu/default_configs
 .. _english one: https://github.com/snipsco/snips-nlu/blob/master/snips_nlu/default_configs/config_en.py

linting_test.py

@@ -2,19 +2,21 @@
 
 import os
 import unittest
+from pathlib import Path
 
 from pylint.lint import Run
 
-ROOT_PATH = os.path.dirname(os.path.abspath(__file__))
+ROOT_PATH = Path(__file__).parent
 
-RCFILEPATH = os.path.join(ROOT_PATH, "tools", "pylintrc")
+RCFILEPATH = ROOT_PATH / "tools" / "pylintrc"
 
-TESTED_PACKAGES = ["snips_nlu", "cli", "debug", "nlu_dataset", "samples"]
+TESTED_PACKAGES = ["snips_nlu", "snips_nlu_samples", "debug"]
+SKIPPED_SUB_PACKAGES = ["tests"]
 
 
 class TestLinting(unittest.TestCase):
     def test_linting(self):
-        args = ["--output-format", "parseable", "--rcfile", RCFILEPATH]
+        args = ["--output-format", "parseable", "--rcfile", str(RCFILEPATH)]
         args += all_python_files()
 
         run = Run(args, exit=False)
@@ -24,10 +26,12 @@ def test_linting(self):
 def all_python_files():
     files = []
     for p in TESTED_PACKAGES:
-        for dirpath, _, filenames in os.walk(os.path.join(ROOT_PATH, p)):
+        for dirpath, _, filenames in os.walk(str(ROOT_PATH / p)):
+            if Path(dirpath).name in SKIPPED_SUB_PACKAGES:
+                continue
             files += [
-                os.sep.join([dirpath, f]) for f in filenames
-                if f.endswith(".py") and "test" not in f
+                os.sep.join([dirpath, f])
+                for f in filenames if f.endswith(".py")
             ]
 
     return files

setup.py

@@ -28,7 +28,7 @@
     "sklearn-crfsuite==0.3.6",
     "semantic_version==2.6.0",
     "snips_nlu_utils==0.6.1",
-    "snips_nlu_ontology==0.57.1",
+    "snips_nlu_ontology==0.57.2",
     "num2words==0.5.6",
     "plac==0.9.6",
     "requests==2.18.4"

snips_nlu/__about__.py

@@ -11,8 +11,8 @@
 __email__ = "[email protected], [email protected]"
 __license__ = "Apache License, Version 2.0"
 
-__version__ = "0.15.1"
-__model_version__ = "0.15.0"
+__version__ = "0.16.0"
+__model_version__ = "0.16.0"
 
 __download_url__ = "https://github.com/snipsco/snips-nlu-language-resources/releases/download"
 __compatibility__ = "https://raw.githubusercontent.com/snipsco/snips-nlu-language-resources/master/compatibility.json"

snips_nlu/__main__.py

@@ -5,31 +5,35 @@
 import plac
 
 from snips_nlu.cli import (
-    cross_val_metrics, train_test_metrics, generate_dataset, download, link)
+    cross_val_metrics, download, download_all_languages, generate_dataset,
+    link, train_test_metrics)
 from snips_nlu.cli.inference import parse
 from snips_nlu.cli.training import train
-from snips_nlu.cli.utils import prints
+from snips_nlu.cli.utils import PrettyPrintLevel, pretty_print
 
 
 def main():
     commands = {
         "train": train,
         "parse": parse,
         "download": download,
+        "download-all-languages": download_all_languages,
         "link": link,
         "generate-dataset": generate_dataset,
         "cross-val-metrics": cross_val_metrics,
         "train-test-metrics": train_test_metrics,
     }
     if len(sys.argv) == 1:
-        prints(', '.join(commands), title="Available commands", exits=1)
+        pretty_print(', '.join(commands), title="Available commands", exits=1,
+                     level=PrettyPrintLevel.INFO)
     command = sys.argv.pop(1)
     sys.argv[0] = 'snips-nlu %s' % command
     if command in commands:
         plac.call(commands[command], sys.argv[1:])
     else:
-        prints("Available: %s" % ', '.join(commands),
-               title="Unknown command: %s" % command, exits=1)
+        pretty_print("Available: %s" % ', '.join(commands),
+                     title="Unknown command: %s" % command, exits=1,
+                     level=PrettyPrintLevel.INFO)
 
 
 if __name__ == "__main__":

snips_nlu/builtin_entities.py

@@ -1,10 +1,9 @@
 from __future__ import unicode_literals
 
-from builtins import object
-from builtins import str
+from builtins import object, str
 
 from snips_nlu_ontology import (
-    get_all_builtin_entities, BuiltinEntityParser as _BuiltinEntityParser,
+    BuiltinEntityParser as _BuiltinEntityParser, get_all_builtin_entities,
     get_supported_entities)
 
 from snips_nlu.utils import LimitedSizeDict

snips_nlu/cli/__init__.py

@@ -1,4 +1,4 @@
-from snips_nlu.cli.download import download
+from snips_nlu.cli.download import download, download_all_languages
 from snips_nlu.cli.generate_dataset import generate_dataset
 from snips_nlu.cli.inference import parse
 from snips_nlu.cli.link import link