diff --git a/.circleci/config.yml b/.circleci/config.yml index 67cdd3a..a0b446e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,3 +1,10 @@ +####### +## 2023-11-07 update @kik: +## Changes to the repo were needed to address bug[1854406] +# However, we do not want to publish any artifacts anywhere +# to avoid this CI steps with potential to publish artifacts were commented out. +### + #################### # CircleCI configuration reference: # https://circleci.com/docs/2.0/configuration-reference @@ -25,26 +32,27 @@ test_settings: &test_settings command: | # python-snappy compression relies on C bindings to libsnappy-dev. # pyspark needs to run Java, so we install openjdk. - echo "deb [check-valid-until=no] http://cdn-fastly.deb.debian.org/debian jessie main" > /etc/apt/sources.list.d/jessie.list - echo "deb [check-valid-until=no] http://archive.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/jessie-backports.list - sed -i '/deb http:\/\/deb.debian.org\/debian jessie-updates main/d' /etc/apt/sources.list + sed -i 's/^/#/' /etc/apt/sources.list + echo "deb [check-valid-until=no] http://archive.debian.org/debian-archive/debian jessie main" >> /etc/apt/sources.list.d/jessie.list + echo "deb [check-valid-until=no] http://archive.debian.org/debian-archive/debian jessie-backports main" >> /etc/apt/sources.list.d/jessie-backports.list + # sed -i '/deb http:\/\/deb.debian.org\/debian jessie-updates main/d' /etc/apt/sources.list apt-get -o Acquire::Check-Valid-Until=false update - apt-get install -t jessie-backports -y libsnappy-dev openjdk-8-jre-headless + apt-get install --force-yes -t jessie-backports -y libsnappy-dev openjdk-8-jre-headless /usr/sbin/update-java-alternatives -s java-1.8.0-openjdk-amd64 - run: &run_tox_environment_matching_circleci_job_name name: Run tox job command: | - pip install tox + pip install tox==3.24.5 tox -e $CIRCLE_JOB - - run: - name: Submit code coverage data - command: | - # Skip this step if running via the CircleCI local CLI. - [ -z "$CIRCLE_BUILD_NUM" ] && exit 0 - # Activate the virtualenv so that codecov can find the 'coverage' executable. - source .tox/$CIRCLE_JOB/bin/activate - # Upload to codecov and flag (-F) with the name of the job. - bash <(curl -s https://codecov.io/bash) -F $CIRCLE_JOB + # - run: + # name: Submit code coverage data + # command: | + # # Skip this step if running via the CircleCI local CLI. + # [ -z "$CIRCLE_BUILD_NUM" ] && exit 0 + # # Activate the virtualenv so that codecov can find the 'coverage' executable. + # source .tox/$CIRCLE_JOB/bin/activate + # # Upload to codecov and flag (-F) with the name of the job. + # bash <(curl -s https://codecov.io/bash) -F $CIRCLE_JOB @@ -73,44 +81,44 @@ jobs: docker: - image: python:3.7-jessie - docs: &docs_settings - docker: - - image: python:3.6-jessie - steps: - - checkout - - run: - <<: *install_packages - - run: - <<: *run_tox_environment_matching_circleci_job_name - - persist_to_workspace: - root: docs/_build - paths: html - - docs-deploy: - docker: - - image: node:8.10.0 - steps: - - checkout - - attach_workspace: - at: docs/_build - - run: - name: Disable jekyll builds - command: touch docs/_build/html/.nojekyll - # Needed for write access to the GitHub repository; - # see https://circleci.com/docs/2.0/gh-bb-integration/#deployment-keys-and-user-keys - - add_ssh_keys: - fingerprints: - - "bb:e1:fa:08:e2:ff:a4:ed:f9:b0:64:c0:e6:07:0d:08" - # The gh-pages npm package looks to be the most widely used utility for - # pushing a directory to a git branch; - # see https://www.npmjs.com/package/gh-pages - - run: - name: Deploy docs to gh-pages branch - command: | - git config user.email "fx-data-platform@mozilla.com" - git config user.name "CircleCI docs-deploy job" - npm install -g --silent gh-pages@2.0.1 - gh-pages --dotfiles --message "[skip ci] Updates" --dist docs/_build/html + # docs: &docs_settings + # docker: + # - image: python:3.6-jessie + # steps: + # - checkout + # - run: + # <<: *install_packages + # - run: + # <<: *run_tox_environment_matching_circleci_job_name + # - persist_to_workspace: + # root: docs/_build + # paths: html + + # docs-deploy: + # docker: + # - image: node:8.10.0 + # steps: + # - checkout + # - attach_workspace: + # at: docs/_build + # - run: + # name: Disable jekyll builds + # command: touch docs/_build/html/.nojekyll + # # Needed for write access to the GitHub repository; + # # see https://circleci.com/docs/2.0/gh-bb-integration/#deployment-keys-and-user-keys + # - add_ssh_keys: + # fingerprints: + # - "bb:e1:fa:08:e2:ff:a4:ed:f9:b0:64:c0:e6:07:0d:08" + # # The gh-pages npm package looks to be the most widely used utility for + # # pushing a directory to a git branch; + # # see https://www.npmjs.com/package/gh-pages + # - run: + # name: Deploy docs to gh-pages branch + # command: | + # git config user.email "fx-data-platform@mozilla.com" + # git config user.name "CircleCI docs-deploy job" + # npm install -g --silent gh-pages@2.0.1 + # gh-pages --dotfiles --message "[skip ci] Updates" --dist docs/_build/html lint: docker: @@ -120,32 +128,32 @@ jobs: - run: name: Run tests command: | - pip install flake8 + pip install flake8==3.9.1 flake8 moztelemetry/ tests/ # Only runs when a tag starting with 'v' is place on the repository; # see the workflows section below for trigger logic. - deploy: - docker: - - image: python:3.6 - steps: - - checkout - - run: - name: Install deployment tools - command: | - pip install --upgrade setuptools wheel twine - - run: - name: Create the distribution files - command: | - python setup.py sdist bdist_wheel - - run: - name: Upload to PyPI - command: | - # Relies on the TWINE_USERNAME and TWINE_PASSWORD environment variables configured at: - # https://circleci.com/gh/mozilla/python_moztelemetry/edit#env-vars - # For more on twine, see: - # https://twine.readthedocs.io/en/latest/ - twine upload dist/* + # deploy: + # docker: + # - image: python:3.6 + # steps: + # - checkout + # - run: + # name: Install deployment tools + # command: | + # pip install --upgrade setuptools wheel twine + # - run: + # name: Create the distribution files + # command: | + # python setup.py sdist bdist_wheel + # - run: + # name: Upload to PyPI + # command: | + # # Relies on the TWINE_USERNAME and TWINE_PASSWORD environment variables configured at: + # # https://circleci.com/gh/mozilla/python_moztelemetry/edit#env-vars + # # For more on twine, see: + # # https://twine.readthedocs.io/en/latest/ + # twine upload dist/* @@ -159,21 +167,21 @@ workflows: jobs: - py35 - py36 - - docs + # - docs - lint - - docs-deploy: - requires: - - docs - filters: - branches: - only: master - - tagged-deploy: - jobs: - - deploy: - filters: - branches: - # Ignore all branches; this workflow should only run for tags. - ignore: /.*/ - tags: - only: /^v.*/ + # - docs-deploy: + # requires: + # - docs + # filters: + # branches: + # only: master + + # tagged-deploy: + # jobs: + # - deploy: + # filters: + # branches: + # # Ignore all branches; this workflow should only run for tags. + # ignore: /.*/ + # tags: + # only: /^v.*/ diff --git a/moztelemetry/parse_scalars.py b/moztelemetry/parse_scalars.py index 8ae7d75..bdbc9a7 100644 --- a/moztelemetry/parse_scalars.py +++ b/moztelemetry/parse_scalars.py @@ -319,8 +319,9 @@ def load_scalars(filename, strict_type_checks=True): # Make sure that the category has at least one probe in it. if not category or len(category) == 0: - raise ParserError('Category "{}" must have at least one probe in it' + - '.\nSee: {}'.format(category_name, BASE_DOC_URL)) + raise ParserError( + '''Category "{}" must have at least one probe in it.' \ + See: {}'''.format(category_name, BASE_DOC_URL)) for probe_name in category: # We found a scalar type. Go ahead and parse it. diff --git a/setup.py b/setup.py index 9712282..cbcab0a 100755 --- a/setup.py +++ b/setup.py @@ -6,24 +6,48 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from setuptools import setup, find_packages -tests_require = ['mock', 'pytest-timeout', 'moto', 'responses', - 'scipy', 'pyspark', 'pytest', 'pytest-cov'] +tests_require = [ + 'mock>=3.0.4,<=5.0.0', + 'pytest-timeout>=1.3.4,<=2.0.2', + 'moto>=2.2.15,<=2.2.20', + 'responses>=0.16.0,<=0.18.0', + 'scipy>=1.3.3,<=1.9.2', + 'pyspark>=2.2.3,<=3.2.4', + 'pytest>=5.3.1,<=6.2.4', + 'pytest-cov>2.6.0,<=2.8.0', + 'flake8>3.8.3,<=3.9.1', +] setup( name='python_moztelemetry', - use_scm_version=True, + # use_scm_version=True, author='Roberto Agostino Vitillo', author_email='rvitillo@mozilla.com', description='Spark bindings for Mozilla Telemetry', url='https://github.com/mozilla/python_moztelemetry', packages=find_packages(), package_dir={'moztelemetry': 'moztelemetry'}, - install_requires=['boto', 'boto3', 'ujson', 'requests', - 'protobuf>=3.6.0', 'six', - 'expiringdict', "functools32;python_version<'3'", 'py4j', - 'pandas>=0.19.2', 'numpy>=1.8.2', - 'PyYAML', 'python-snappy'], - setup_requires=['pytest-runner', 'setuptools_scm'], + install_requires=[ + 'boto<=2.49.0', + 'boto3<=1.28.57', + 'ujson>=3.2.0,<=5.5.0', + 'requests>2.24.0,<=2.31.0', + 'protobuf>=3.17.3,<=3.19.6', + 'six==1.12', + 'expiringdict==1.2.2', + "functools32;python_version<'3'", + 'py4j>=0.10.9.4,<=0.10.9.7', + 'pandas>=0.19.2', + 'numpy>=1.18.5,<=1.21.6', + 'PyYAML==5.1.2', + 'python-snappy>=0.5.4,<=0.6.0', + 'urllib3<1.27,>=1.25.4', + 'typed-ast<1.5,>=1.4.0' + ], + setup_requires=[ + 'pytest-runner>=5.2,<=5.3.2', + 'setuptools_scm>=4.1.2,<=7.0.5' + ], extras_require={ 'testing': tests_require, },