Refs: #446: DKB migration to python3

Migration DKB to python3. DKB run in Conda python environment independent on OS used. Initial migration performed by running 2to3 python helper program: find /opt/inMem/dkb/ -type f -name *.py -exec 2to3 -n -w -f all -f idioms -f buffer -f set_literal -f ws_comma {} \; DBK specific fixes for python3: * all strings are unicode * no more unbuffered io for strings * sorted dictionary in mem * no more encoded strings by default * no more decode 'string_escape' * changes in error output for tests * changes in Elasticsearch python3 library * add conda python environment yaml
PanDAWMS · Jun 2, 2023 · 99fbf22 · 99fbf22
1 parent d747cbe
commit 99fbf22
Show file tree

Hide file tree

Showing 115 changed files with 12,585 additions and 12,307 deletions.
diff --git a/Docs/source/conf.py b/Docs/source/conf.py
@@ -20,14 +20,14 @@
 
 # -- Project information -----------------------------------------------------
 
-project = u'Data Knowledge Base'
-copyright = u'2018, DKB team'
-author = u'DKB team'
+project = 'Data Knowledge Base'
+copyright = '2018, DKB team'
+author = 'DKB team'
 
 # The short X.Y version
-version = u''
+version = ''
 # The full version, including alpha/beta/rc tags
-release = u''
+release = ''
 
 
 # -- General configuration ---------------------------------------------------
@@ -134,8 +134,8 @@
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'DKB.tex', u'DKB Documentation',
-     u'DKB team', 'manual'),
+    (master_doc, 'DKB.tex', 'DKB Documentation',
+     'DKB team', 'manual'),
 ]
 
 
@@ -144,7 +144,7 @@
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    (master_doc, 'dkb', u'DKB Documentation',
+    (master_doc, 'dkb', 'DKB Documentation',
      [author], 1)
 ]
 
@@ -155,7 +155,7 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'DKB', u'DKB Documentation',
+    (master_doc, 'DKB', 'DKB Documentation',
      author, 'DKB', 'One line description of project.',
      'Miscellaneous'),
 ]

diff --git a/Installation/Conda/README b/Installation/Conda/README
@@ -0,0 +1,13 @@
+# Setup conda environment to use DKB
+
+## Get latest Miniconda and install it
+  ```
+  wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
+  sh Miniconda3-latest-Linux-x86_64.sh
+  #install into /opt/dkb3/miniconda3
+  #add to init
+  ```
+
+## Create environment
+  conda env create -n dkb_python_3_9 -f environment.yaml
+
diff --git a/Installation/Conda/environment.yaml b/Installation/Conda/environment.yaml
@@ -0,0 +1,253 @@
+name: dkb_python_3_9
+channels:
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - aiohttp=3.8.3=py39h5eee18b_0
+  - aiosignal=1.2.0=pyhd3eb1b0_0
+  - async-timeout=4.0.2=py39h06a4308_0
+  - brotlipy=0.7.0=py39h27cfd23_1003
+  - ca-certificates=2023.01.10=h06a4308_0
+  - cchardet=2.1.7=py39h6a678d5_0
+  - certifi=2022.12.7=py39h06a4308_0
+  - cffi=1.15.1=py39h5eee18b_3
+  - cryptography=39.0.1=py39h9ce1e76_0
+  - cx_oracle=8.3.0=py39h7f8727e_0
+  - elasticsearch=7.13.3=pyhd3eb1b0_0
+  - frozenlist=1.3.3=py39h5eee18b_0
+  - idna=3.4=py39h06a4308_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - libffi=3.4.2=h6a678d5_6
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - multidict=6.0.2=py39h5eee18b_0
+  - ncurses=6.4=h6a678d5_0
+  - openssl=1.1.1t=h7f8727e_0
+  - pycparser=2.21=pyhd3eb1b0_0
+  - pyopenssl=23.0.0=py39h06a4308_0
+  - pysocks=1.7.1=py39h06a4308_0
+  - python=3.9.16=h7a1cb2a_2
+  - pytz=2022.7=py39h06a4308_0
+  - readline=8.2=h5eee18b_0
+  - setuptools=65.6.3=py39h06a4308_0
+  - sqlite=3.41.1=h5eee18b_0
+  - tk=8.6.12=h1ccaba5_0
+  - tzdata=2023c=h04d1e81_0
+  - urllib3=1.26.15=py39h06a4308_0
+  - wheel=0.38.4=py39h06a4308_0
+  - xz=5.2.10=h5eee18b_1
+  - yarl=1.8.1=py39h5eee18b_0
+  - zlib=1.2.13=h5eee18b_0
+  - pip:
+      - alabaster==0.7.13
+      - alembic==1.10.4
+      - anyio==3.6.2
+      - argcomplete==2.1.2
+      - argparse==1.4.0
+      - argparseplus==3.0.0
+      - astatine==0.3.2
+      - astor==0.8.1
+      - astpretty==3.0.0
+      - astroid==2.15.2
+      - asttokens==2.2.1
+      - attrs==22.2.0
+      - autoflake==1.7.8
+      - babel==2.12.1
+      - bandit==1.7.5
+      - bcrypt==4.0.1
+      - black==23.3.0
+      - boto3==1.26.8
+      - botocore==1.29.127
+      - cachetools==5.3.0
+      - chardet==5.1.0
+      - charset-normalizer==3.1.0
+      - click==8.1.3
+      - cognitive-complexity==1.3.0
+      - colorama==0.4.6
+      - coverage==7.2.2
+      - darglint==1.8.1
+      - decorator==5.1.1
+      - defusedxml==0.7.1
+      - dill==0.3.6
+      - distlib==0.3.6
+      - dlint==0.14.0
+      - doc8==1.1.1
+      - docformatter==1.6.0
+      - docopt==0.6.2
+      - docutils==0.19
+      - dogpile-cache==1.1.8
+      - domdf-python-tools==3.6.0
+      - eradicate==2.2.0
+      - exceptiongroup==1.1.1
+      - fastapi==0.95.0
+      - filelock==3.10.7
+      - flake8==5.0.4
+      - flake8-2020==1.7.0
+      - flake8-aaa==0.14.0
+      - flake8-annotations==3.0.0
+      - flake8-annotations-complexity==0.0.7
+      - flake8-annotations-coverage==0.0.6
+      - flake8-bandit==4.1.1
+      - flake8-black==0.3.6
+      - flake8-blind-except==0.2.1
+      - flake8-breakpoint==1.1.0
+      - flake8-broken-line==0.6.0
+      - flake8-bugbear==23.3.12
+      - flake8-builtins==1.5.3
+      - flake8-class-attributes-order==0.1.3
+      - flake8-coding==1.3.2
+      - flake8-cognitive-complexity==0.1.0
+      - flake8-comments==0.1.2
+      - flake8-comprehensions==3.11.1
+      - flake8-debugger==4.1.2
+      - flake8-django==1.1.5
+      - flake8-docstrings==1.7.0
+      - flake8-encodings==0.5.0.post1
+      - flake8-eradicate==1.4.0
+      - flake8-executable==2.1.3
+      - flake8-expression-complexity==0.0.11
+      - flake8-fastapi==0.7.0
+      - flake8-fixme==1.1.1
+      - flake8-functions==0.0.7
+      - flake8-functions-names==0.4.0
+      - flake8-future-annotations==0.0.5
+      - flake8-helper==0.2.1
+      - flake8-isort==6.0.0
+      - flake8-literal==1.3.0
+      - flake8-logging-format==0.9.0
+      - flake8-markdown==0.4.0
+      - flake8-mutable==1.2.0
+      - flake8-no-pep420==2.3.0
+      - flake8-noqa==1.3.1
+      - flake8-pie==0.16.0
+      - flake8-plugin-utils==1.3.2
+      - flake8-pyi==22.11.0
+      - flake8-pylint==0.2.0
+      - flake8-pytest-style==1.7.2
+      - flake8-quotes==3.3.2
+      - flake8-rst-docstrings==0.3.0
+      - flake8-secure-coding-standard==1.4.0
+      - flake8-simplify==0.20.0
+      - flake8-string-format==0.3.0
+      - flake8-tidy-imports==4.8.0
+      - flake8-typing-imports==1.14.0
+      - flake8-use-fstring==1.4
+      - flake8-use-pathlib==0.3.0
+      - flake8-useless-assert==0.4.4
+      - flake8-variables-names==0.0.5
+      - flake8-warnings==0.4.0
+      - flask==2.2.3
+      - future==0.18.3
+      - geoip2==4.6.0
+      - gitdb==4.0.10
+      - gitpython==3.1.31
+      - google-auth==2.16.2
+      - greenlet==2.0.2
+      - hypothesis==6.70.2
+      - hypothesmith==0.1.9
+      - imagesize==1.4.1
+      - importlib-metadata==6.1.0
+      - iniconfig==2.0.0
+      - isort==5.12.0
+      - itsdangerous==2.1.2
+      - jinja2==3.1.2
+      - jmespath==1.0.1
+      - jsonschema==4.17.3
+      - jupyterlab-flake8==0.7.1
+      - lark-parser==0.12.0
+      - lazy-object-proxy==1.9.0
+      - libcst==0.4.9
+      - mako==1.2.4
+      - markdown-it-py==2.2.0
+      - markupsafe==2.1.2
+      - maxminddb==2.2.0
+      - mccabe==0.7.0
+      - mdurl==0.1.2
+      - mr-proper==0.0.7
+      - mypy-extensions==1.0.0
+      - natsort==8.3.1
+      - oic==1.5.0
+      - packaging==23.0
+      - pandas-vet==0.2.3
+      - paramiko==3.0.0
+      - pathspec==0.10.3
+      - pbr==5.11.1
+      - pep8-naming==0.13.3
+      - pip==23.1.2
+      - platformdirs==3.2.0
+      - pluggy==1.0.0
+      - prometheus-client==0.16.0
+      - pyami-atlas==5.1.0.1
+      - pyami-core==5.1.2
+      - pyasn1==0.5.0
+      - pyasn1-modules==0.3.0
+      - pybetter==0.4.1
+      - pycln==2.1.3
+      - pycodestyle==2.9.1
+      - pycryptodomex==3.17
+      - pydantic==1.10.7
+      - pydocstyle==6.3.0
+      - pyemojify==0.2.0
+      - pyflakes==2.5.0
+      - pygments==2.14.0
+      - pyjwkest==1.4.2
+      - pylint==2.17.2
+      - pymemcache==4.0.0
+      - pynacl==1.5.0
+      - pyproject-api==1.5.1
+      - pyrsistent==0.19.3
+      - pytest==7.2.2
+      - pytest-cov==4.0.0
+      - pytest-sugar==0.9.6
+      - python-dateutil==2.8.2
+      - python-dev-tools==2023.3.24
+      - python-magic==0.4.27
+      - pyupgrade==3.3.1
+      - pyyaml==6.0
+      - redis==4.5.1
+      - removestar==1.3.1
+      - requests==2.28.2
+      - restructuredtext-lint==1.4.0
+      - rich==13.3.3
+      - rsa==4.9
+      - rucio==1.31.2
+      - s3transfer==0.6.1
+      - six==1.16.0
+      - smmap==5.0.0
+      - sniffio==1.3.0
+      - snowballstemmer==2.2.0
+      - sortedcontainers==2.4.0
+      - sphinx==6.1.3
+      - sphinxcontrib-applehelp==1.0.4
+      - sphinxcontrib-devhelp==1.0.2
+      - sphinxcontrib-htmlhelp==2.0.1
+      - sphinxcontrib-jsmath==1.0.1
+      - sphinxcontrib-qthelp==1.0.3
+      - sphinxcontrib-serializinghtml==1.1.5
+      - sqlalchemy==2.0.5.post1
+      - ssort==0.11.6
+      - starlette==0.26.1
+      - statsd==4.0.1
+      - stdlib-list==0.8.0
+      - stevedore==5.0.0
+      - stomp-py==8.1.0
+      - tabulate==0.8.10
+      - termcolor==2.2.0
+      - tiny-xslt==2.0.1
+      - tokenize-rt==5.0.0
+      - tomli==2.0.1
+      - tomlkit==0.11.7
+      - tox==4.4.8
+      - tox-travis==0.12
+      - typer==0.7.0
+      - typing-extensions==4.5.0
+      - typing-inspect==0.8.0
+      - untokenize==0.1.1
+      - virtualenv==20.21.0
+      - websocket-client==1.5.1
+      - werkzeug==2.3.3
+      - wrapt==1.15.0
+      - zipp==3.15.0
+prefix: /opt/dkb3/miniconda3/envs/dkb_python_3_9
diff --git a/Utils/API/server/lib/dkb/api/__init__.py b/Utils/API/server/lib/dkb/api/__init__.py
@@ -4,8 +4,8 @@
 
 import logging
 
-from exceptions import DkbApiException
-import methods
+from .exceptions import DkbApiException
+from . import methods
 
 CONFIG_DIR = '%%CFG_DIR%%'
 
@@ -47,7 +47,7 @@ def configure():
     retval = True
     try:
         methods.configure()
-    except DkbApiException, err:
+    except DkbApiException as err:
         logging.fatal("Server configuration failed: %s." % err)
         trace = traceback.format_exception(*sys.exc_info())
         for line in trace:

diff --git a/Utils/API/server/lib/dkb/api/config.py b/Utils/API/server/lib/dkb/api/config.py
@@ -2,7 +2,7 @@
 Module for handling configuration files.
 """
 
-from exceptions import (DkbApiNotImplemented,
+from .exceptions import (DkbApiNotImplemented,
                         ConfigurationNotFound,
                         ConfigurationError)
 from . import CONFIG_DIR
@@ -49,6 +49,6 @@ def read_config(fname):
     with open(full_path) as f:
         try:
             parsed = yaml.load(f)
-        except yaml.YAMLError, e:
+        except yaml.YAMLError as e:
             raise ConfigurationError('Format error: %s' % e)
     return parsed
diff --git a/Utils/API/server/lib/dkb/api/exceptions.py b/Utils/API/server/lib/dkb/api/exceptions.py
@@ -99,7 +99,7 @@ class MethodAlreadyExists(MethodException):
     code = 472
 
     def __init__(self, method, category):
-        if type(category) == list:
+        if isinstance(category, list):
             category = "categories (%s)" % category
         else:
             category = "category ('%s')" % category