Merge pull request #287 from JohnSnowLabs/release/541

Release/541

.github/workflows/nlu_test_flow.yaml

@@ -10,15 +10,15 @@ jobs:
   build:
     runs-on: ubuntu-latest
     env:
-      JSL_LICENSE: ${{ secrets.JSL_LICENSE }}
+      JOHNSNOWLABS_LICENSE_JSON: ${{ secrets.JOHNSNOWLABS_LICENSE_JSON }}
     strategy:
       matrix:
         python-version: [3.7]
     steps:
       - uses: actions/setup-java@v1
         with:
-          java-version: '1.8.0' # The JDK version to make available on the path.
-          java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jd
+          java-version: '1.8.0'
+          java-package: jdk
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
@@ -27,173 +27,17 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install pypandoc scikit-learn
-          pip install  wheel  dataclasses pandas numpy pytest modin[ray] pyspark==3.2.0 spark-nlp
-          java -version
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-          ! echo 2 |  update-alternatives --config java
-      - name: NLU Chunker Component tests
-        if: always()
-        run: |
-          python -m unittest  discover   -s './tests/nlu_core_tests/component_tests/chunker_tests' -p '*tests.py'
-      - name: NLU Classifier Cyber tests -
-        if: always()
-        run: |
-          python -m unittest  'tests/nlu_core_tests/component_tests/classifier_tests/cyber_tests.py'
-      - name: NLU Classifier E2E tests -
-        if: always()
-        run: |
-          python -m unittest  'tests/nlu_core_tests/component_tests/classifier_tests/e2e_tests.py'
-      - name: NLU Classifier Emotion tests -
-        if: always()
-        run: |
-          python -m unittest  'tests/nlu_core_tests/component_tests/classifier_tests/emotion_tests.py'
-      - name: NLU Classifier Language tests -
-        if: always()
-        run: |
-          python -m unittest  'tests/nlu_core_tests/component_tests/classifier_tests/language_tests.py'
-      - name: NLU Classifier NER tests -
-        if: always()
-        run: |
-          python -m unittest  'tests/nlu_core_tests/component_tests/classifier_tests/ner_tests.py'
-      - name: NLU Classifier POS tests -
-        if: always()
-        run: |
-          python -m unittest  'tests/nlu_core_tests/component_tests/classifier_tests/pos_tests.py'
-      - name: NLU Classifier Question tests -
-        if: always()
-        run: |
-          python -m unittest  'tests/nlu_core_tests/component_tests/classifier_tests/question_tests.py'
-      - name: NLU Classifier Sarcasm tests -
-        if: always()
-        run: |
-          python -m unittest  'tests/nlu_core_tests/component_tests/classifier_tests/sarcasm_tests.py'
-      - name: NLU Classifier Sentiment tests -
-        if: always()
-        run: |
-          python -m unittest  'tests/nlu_core_tests/component_tests/classifier_tests/sentiment_tests.py'
-      - name: NLU Classifier Sentiment test Imdb -
-        if: always()
-        run: |
-          python -m unittest  'tests/nlu_core_tests/component_tests/classifier_tests/sentiment_test_imdb.py'
-      - name: NLU Classifier Sentiment test Twitter -
-        if: always()
-        run: |
-          python -m unittest  'tests/nlu_core_tests/component_tests/classifier_tests/sentiment_test_twitter.py'
-      - name: NLU Classifier Spam tests -
-        if: always()
-        run: |
-          python -m unittest  'tests/nlu_core_tests/component_tests/classifier_tests/spam_tests.py'
-      - name: NLU Classifier Toxic tests -
-        if: always()
-        run: |
-          python -m unittest  'tests/nlu_core_tests/component_tests/classifier_tests/toxic_tests.py'
-      - name: NLU Classifier YAKE tests -
-        if: always()
-        run: |
-          python -m unittest  'tests/nlu_core_tests/component_tests/classifier_tests/yake_tests.py'
-#      - name: NLU Embed Component tests
-#        if: always()
-#        run: |
-#          python -m unittest  discover   -s './tests/nlu_core_tests/component_tests/embed_tests' -p '*tests.py'
-      - name: NLU Text Cleaner Component tests -
-        if: always()
-        run: |
-          python -m unittest  discover   -s './tests/nlu_core_tests/component_tests/pre_processing_tests' -p '*tests.py'
-#      - name: NLU Matcher Component tests # not yet converted in Spark 3
-#        if: always()
-#        run: |
-#          python -m unittest  discover   -s 'tests/nlu_core_tests/component_tests/matcher_tests' -p '*tests.py'
-      - name: NLU Typed Dependency Component tests -
-        if: always()
-        run: |
-          python -m unittest  discover   -s './tests/nlu_core_tests/component_tests/typed_dependency_tests' -p '*tests.py'
-      - name: NLU Untyped Dependency Component tests -
-        if: always()
-        run: |
-          python -m unittest  discover   -s './tests/nlu_core_tests/component_tests/untyped_dependency_tests' -p '*tests.py'
-      - name: NLU Pipe tests
-        if: always()
+          pip install wheel dataclasses pandas numpy pytest modin[ray] pytest-xdist pytest-xdist pytest-forked nbformat librosa johnsnowlabs==5.3.4rc1
+          pip uninstall nlu -y
+      - name: Install Licensed Libs
+        if:
         run: |
-          python -m unittest  discover   -s './tests/nlu_core_tests/pipeline_tests' -p '*tests.py'
-      - name: NLU Training sentiment tests
+          python -c 'from johnsnowlabs import nlp;nlp.install(browser_login = False, force_browser=False,visual=True)'
+      - name: Run one test per lib
         if: always()
         run: |
-          python -m unittest  discover   -s './tests/nlu_core_tests/training_tests/classifiers' -p '*sentiment_dl*.py'
-      # Too hevy for Github actions
-      #      - name: NLU Training classifier tests
-      #        if: always()
-      #        run: |
-      #          python -m unittest  discover   -s './tests/nlu_core_tests/training_tests/classifiers' -p '*classifier_dl*.py'
-#      - name: NLU Training multi classifier tests
-#        if: always()
-#        run: |
-#          python -m unittest  discover   -s './tests/nlu_core_tests/training_tests/classifiers' -p '*multi*.py'
-#      - name: NLU Training NER tests
-#        if: always()
-#        run: |
-#          python -m unittest  discover   -s './tests/nlu_core_tests/training_tests/classifiers' -p '*ner*.py'
-      - name: NLU Training POS tests
+          python tests/run_tests.py one_per_lib
+      - name: Run all tests
         if: always()
         run: |
-          python -m unittest  discover   -s './tests/nlu_core_tests/training_tests/classifiers' -p '*pos*.py'
-#      - name: NLU Healthcare Verification tests
-#        if: always()
-#        run: |
-#          python -m unittest  discover  -s ./tests/nlu_hc_tests -p 'verification_tests.py'
-#      - name: NLU OCR tests
-#        if: always()
-#        run: |
-#          python -m unittest  discover  -s ./tests/nlu_ocr_tests -p '*tests.py'
-#      - name: NLU Healthcare Assertion DL tests
-#        if: always()
-#        run: |
-#          python -m unittest  discover  -s ./tests/nlu_hc_tests/component_tests/assertion_dl -p '*tests.py'
-##      - name: NLU Healthcare Contextual Parser tests
-##        if: always()
-##        run: |
-##          python -m unittest  discover  -s ./tests/nlu_hc_tests/component_tests/contextual_parser -p '*tests.py'
-#      - name: NLU Healthcare De Identification tests
-#        if: always()
-#        run: |
-#          python -m unittest  discover  -s ./tests/nlu_hc_tests/component_tests/de_identification -p '*tests.py'
-#      - name: NLU Healthcare Drug Normalizer tests
-#        if: always()
-#        run: |
-#          python -m unittest  'tests/nlu_hc_tests/component_tests/drug_normalizer/drug_normalizer_test.py'
-#      - name: NLU Healthcare Generic Classifier tests
-#        if: always()
-#        run: |
-#          python -m unittest  discover  -s ./tests/nlu_hc_tests/component_tests/generic_classifier -p '*tests.py'
-#      - name: NLU Healthcare Licensed Classifier tests
-#        if: always()
-#        run: |
-#          python -m unittest  discover  -s ./tests/nlu_hc_tests/component_tests/licensed_classifier -p '*tests.py'
-#      - name: NLU Healthcare Relation Extraction tests
-#        if: always()
-#        run: |
-#          python -m unittest  discover  -s ./tests/nlu_hc_tests/component_tests/relation_extraction -p '*tests.py'
-#      - name: NLU Healthcare Sentence Entity Resolver tests
-#        if: always()
-#        run: |
-#          python -m unittest  discover  -s ./tests/nlu_hc_tests/component_tests/sentence_entity_resolver -p '*tests.py'
-##      - name: NLU Healthcare Pipe tests
-##        if: always()
-##        run: |
-##          python -m unittest  discover  -s ./tests/nlu_hc_tests/pipe_tests -p '*tests.py'
-##      - name: NLU Healthcare Training Chunk Resolution tests
-##        if: always()
-##        run: |
-##          python -m unittest  discover  -s ./tests/nlu_hc_tests/training_tests/chunk_resolution -p '*tests.py'
-#      - name: NLU Healthcare Training Sentence Resolution tests
-#        if: always()
-#        run: |
-#          python -m unittest  discover  -s ./tests/nlu_hc_tests/training_tests/sentence_resolution -p '*tests.py'
-##      - name: NLU Saving and Loading tests
-##        if: always()
-##        run: |
-##          python -m unittest  discover   -s './tests/nlu_core_tests/training_tests/trained_pipe_tests' -p '*tests.py'
-##      - name: NLU Modin tests
-##        if: always()
-##        run: |
-##          python -m unittest  discover   -s './tests/modin' -p '*tests.py'
+          python tests/run_tests.py all

MANIFEST.in

@@ -0,0 +1 @@
+include VERSION

README.md

@@ -1,4 +1,4 @@
-
+ 
 # NLU: The Power of Spark NLP, the Simplicity of Python
 John Snow Labs' NLU is a Python library for applying state-of-the-art text mining, directly on any dataframe, with a single line of code.
 As a facade of the award-winning Spark NLP library, it comes with **1000+** of pretrained models in **100+**, all production-grade, scalable, and trainable, with **everything in 1 line of code.**

VERSION

@@ -0,0 +1 @@
+5.4.1

conda/meta.yaml

@@ -0,0 +1,34 @@
+package:
+  name: nlu
+  version: {{ environ.get('CODE_VERSION', '') }}
+
+app:
+  entry: nlu
+  summary: The power of 15000+ State-of-the-art pre-trained NLP models in 300 languages with 1 line of Python code.
+
+source:
+  path: ../conda_src
+
+build:
+  noarch: generic
+  number: 0
+  script: "python3 -m pip install . --no-deps -vv"
+
+requirements:
+  build:
+    - python
+  run:
+    - python
+    - pyspark==3.0.1
+    - spark-nlp >=5.2.0
+    - numpy
+    - pyarrow >=0.16.0
+    - pandas >=1.3.5
+    - dataclasses
+about:
+  home: https://nlu.johnsnowlabs.com/
+  license: Apache License 2.0
+  license_family: APACHE
+  license_url: https://github.com/JohnSnowLabs/nlu/blob/master/LICENSE
+  description: John Snow Labs' NLU is a Python library for applying state-of-the-art text mining, directly on any dataframe, with a single line of code. As a facade of the award-winning Spark NLP library, it comes with hundreds of pretrained models in tens of languages - all production-grade, scalable, and trainable.
+  summary: The The power of 15000+ State-of-the-art pre-trained NLP models in 300 languages with 1 line of Python code.