Skip to content
This repository has been archived by the owner on Nov 21, 2023. It is now read-only.

# bug(1854406): Addressing installation problems affecting consumers of this repo. #277

Merged
merged 5 commits into from
Nov 9, 2023
Merged

# bug(1854406): Addressing installation problems affecting consumers of this repo. #277

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3c11bcc
updated setup.py and explitly specifying dependencies versions + remo…
kik-kik Oct 2, 2023
a32712b
Tweaked CI configuration to comment out any potential publishing steps
kik-kik Nov 7, 2023
d424bd6
trying to fix CI build
kik-kik Nov 8, 2023
5fb55d7
fixing linting error
kik-kik Nov 8, 2023
6efd6f8
Tweaked CI configuration to comment out any potential publishing steps
kik-kik Nov 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
190 changes: 99 additions & 91 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,10 @@
#######
## 2023-11-07 update @kik:
## Changes to the repo were needed to address bug[1854406]
# However, we do not want to publish any artifacts anywhere
# to avoid this CI steps with potential to publish artifacts were commented out.
###

####################
# CircleCI configuration reference:
# https://circleci.com/docs/2.0/configuration-reference
Expand Down Expand Up @@ -25,26 +32,27 @@ test_settings: &test_settings
command: |
# python-snappy compression relies on C bindings to libsnappy-dev.
# pyspark needs to run Java, so we install openjdk.
echo "deb [check-valid-until=no] http://cdn-fastly.deb.debian.org/debian jessie main" > /etc/apt/sources.list.d/jessie.list
echo "deb [check-valid-until=no] http://archive.debian.org/debian jessie-backports main" > /etc/apt/sources.list.d/jessie-backports.list
sed -i '/deb http:\/\/deb.debian.org\/debian jessie-updates main/d' /etc/apt/sources.list
sed -i 's/^/#/' /etc/apt/sources.list
echo "deb [check-valid-until=no] http://archive.debian.org/debian-archive/debian jessie main" >> /etc/apt/sources.list.d/jessie.list
echo "deb [check-valid-until=no] http://archive.debian.org/debian-archive/debian jessie-backports main" >> /etc/apt/sources.list.d/jessie-backports.list
# sed -i '/deb http:\/\/deb.debian.org\/debian jessie-updates main/d' /etc/apt/sources.list
apt-get -o Acquire::Check-Valid-Until=false update
apt-get install -t jessie-backports -y libsnappy-dev openjdk-8-jre-headless
apt-get install --force-yes -t jessie-backports -y libsnappy-dev openjdk-8-jre-headless
/usr/sbin/update-java-alternatives -s java-1.8.0-openjdk-amd64
- run: &run_tox_environment_matching_circleci_job_name
name: Run tox job
command: |
pip install tox
pip install tox==3.24.5
tox -e $CIRCLE_JOB
- run:
name: Submit code coverage data
command: |
# Skip this step if running via the CircleCI local CLI.
[ -z "$CIRCLE_BUILD_NUM" ] && exit 0
# Activate the virtualenv so that codecov can find the 'coverage' executable.
source .tox/$CIRCLE_JOB/bin/activate
# Upload to codecov and flag (-F) with the name of the job.
bash <(curl -s https://codecov.io/bash) -F $CIRCLE_JOB
# - run:
# name: Submit code coverage data
# command: |
# # Skip this step if running via the CircleCI local CLI.
# [ -z "$CIRCLE_BUILD_NUM" ] && exit 0
# # Activate the virtualenv so that codecov can find the 'coverage' executable.
# source .tox/$CIRCLE_JOB/bin/activate
# # Upload to codecov and flag (-F) with the name of the job.
# bash <(curl -s https://codecov.io/bash) -F $CIRCLE_JOB



Expand Down Expand Up @@ -73,44 +81,44 @@ jobs:
docker:
- image: python:3.7-jessie

docs: &docs_settings
docker:
- image: python:3.6-jessie
steps:
- checkout
- run:
<<: *install_packages
- run:
<<: *run_tox_environment_matching_circleci_job_name
- persist_to_workspace:
root: docs/_build
paths: html

docs-deploy:
docker:
- image: node:8.10.0
steps:
- checkout
- attach_workspace:
at: docs/_build
- run:
name: Disable jekyll builds
command: touch docs/_build/html/.nojekyll
# Needed for write access to the GitHub repository;
# see https://circleci.com/docs/2.0/gh-bb-integration/#deployment-keys-and-user-keys
- add_ssh_keys:
fingerprints:
- "bb:e1:fa:08:e2:ff:a4:ed:f9:b0:64:c0:e6:07:0d:08"
# The gh-pages npm package looks to be the most widely used utility for
# pushing a directory to a git branch;
# see https://www.npmjs.com/package/gh-pages
- run:
name: Deploy docs to gh-pages branch
command: |
git config user.email "[email protected]"
git config user.name "CircleCI docs-deploy job"
npm install -g --silent [email protected]
gh-pages --dotfiles --message "[skip ci] Updates" --dist docs/_build/html
# docs: &docs_settings
# docker:
# - image: python:3.6-jessie
# steps:
# - checkout
# - run:
# <<: *install_packages
# - run:
# <<: *run_tox_environment_matching_circleci_job_name
# - persist_to_workspace:
# root: docs/_build
# paths: html

# docs-deploy:
# docker:
# - image: node:8.10.0
# steps:
# - checkout
# - attach_workspace:
# at: docs/_build
# - run:
# name: Disable jekyll builds
# command: touch docs/_build/html/.nojekyll
# # Needed for write access to the GitHub repository;
# # see https://circleci.com/docs/2.0/gh-bb-integration/#deployment-keys-and-user-keys
# - add_ssh_keys:
# fingerprints:
# - "bb:e1:fa:08:e2:ff:a4:ed:f9:b0:64:c0:e6:07:0d:08"
# # The gh-pages npm package looks to be the most widely used utility for
# # pushing a directory to a git branch;
# # see https://www.npmjs.com/package/gh-pages
# - run:
# name: Deploy docs to gh-pages branch
# command: |
# git config user.email "[email protected]"
# git config user.name "CircleCI docs-deploy job"
# npm install -g --silent [email protected]
# gh-pages --dotfiles --message "[skip ci] Updates" --dist docs/_build/html

lint:
docker:
Expand All @@ -120,32 +128,32 @@ jobs:
- run:
name: Run tests
command: |
pip install flake8
pip install flake8==3.9.1
flake8 moztelemetry/ tests/

# Only runs when a tag starting with 'v' is place on the repository;
# see the workflows section below for trigger logic.
deploy:
docker:
- image: python:3.6
steps:
- checkout
- run:
name: Install deployment tools
command: |
pip install --upgrade setuptools wheel twine
- run:
name: Create the distribution files
command: |
python setup.py sdist bdist_wheel
- run:
name: Upload to PyPI
command: |
# Relies on the TWINE_USERNAME and TWINE_PASSWORD environment variables configured at:
# https://circleci.com/gh/mozilla/python_moztelemetry/edit#env-vars
# For more on twine, see:
# https://twine.readthedocs.io/en/latest/
twine upload dist/*
# deploy:
# docker:
# - image: python:3.6
# steps:
# - checkout
# - run:
# name: Install deployment tools
# command: |
# pip install --upgrade setuptools wheel twine
# - run:
# name: Create the distribution files
# command: |
# python setup.py sdist bdist_wheel
# - run:
# name: Upload to PyPI
# command: |
# # Relies on the TWINE_USERNAME and TWINE_PASSWORD environment variables configured at:
# # https://circleci.com/gh/mozilla/python_moztelemetry/edit#env-vars
# # For more on twine, see:
# # https://twine.readthedocs.io/en/latest/
# twine upload dist/*



Expand All @@ -159,21 +167,21 @@ workflows:
jobs:
- py35
- py36
- docs
# - docs
- lint
- docs-deploy:
requires:
- docs
filters:
branches:
only: master

tagged-deploy:
jobs:
- deploy:
filters:
branches:
# Ignore all branches; this workflow should only run for tags.
ignore: /.*/
tags:
only: /^v.*/
# - docs-deploy:
# requires:
# - docs
# filters:
# branches:
# only: master

# tagged-deploy:
# jobs:
# - deploy:
# filters:
# branches:
# # Ignore all branches; this workflow should only run for tags.
# ignore: /.*/
# tags:
# only: /^v.*/
5 changes: 3 additions & 2 deletions moztelemetry/parse_scalars.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,8 +319,9 @@ def load_scalars(filename, strict_type_checks=True):

# Make sure that the category has at least one probe in it.
if not category or len(category) == 0:
raise ParserError('Category "{}" must have at least one probe in it' +
'.\nSee: {}'.format(category_name, BASE_DOC_URL))
raise ParserError(
'''Category "{}" must have at least one probe in it.' \
See: {}'''.format(category_name, BASE_DOC_URL))

for probe_name in category:
# We found a scalar type. Go ahead and parse it.
Expand Down
42 changes: 33 additions & 9 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,24 +6,48 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from setuptools import setup, find_packages

tests_require = ['mock', 'pytest-timeout', 'moto', 'responses',
'scipy', 'pyspark', 'pytest', 'pytest-cov']
tests_require = [
'mock>=3.0.4,<=5.0.0',
'pytest-timeout>=1.3.4,<=2.0.2',
'moto>=2.2.15,<=2.2.20',
'responses>=0.16.0,<=0.18.0',
'scipy>=1.3.3,<=1.9.2',
'pyspark>=2.2.3,<=3.2.4',
'pytest>=5.3.1,<=6.2.4',
'pytest-cov>2.6.0,<=2.8.0',
'flake8>3.8.3,<=3.9.1',
]

setup(
name='python_moztelemetry',
use_scm_version=True,
# use_scm_version=True,
author='Roberto Agostino Vitillo',
author_email='[email protected]',
description='Spark bindings for Mozilla Telemetry',
url='https://github.com/mozilla/python_moztelemetry',
packages=find_packages(),
package_dir={'moztelemetry': 'moztelemetry'},
install_requires=['boto', 'boto3', 'ujson', 'requests',
'protobuf>=3.6.0', 'six',
'expiringdict', "functools32;python_version<'3'", 'py4j',
'pandas>=0.19.2', 'numpy>=1.8.2',
'PyYAML', 'python-snappy'],
setup_requires=['pytest-runner', 'setuptools_scm'],
install_requires=[
'boto<=2.49.0',
'boto3<=1.28.57',
'ujson>=3.2.0,<=5.5.0',
'requests>2.24.0,<=2.31.0',
'protobuf>=3.17.3,<=3.19.6',
'six==1.12',
'expiringdict==1.2.2',
"functools32;python_version<'3'",
'py4j>=0.10.9.4,<=0.10.9.7',
'pandas>=0.19.2',
'numpy>=1.18.5,<=1.21.6',
'PyYAML==5.1.2',
'python-snappy>=0.5.4,<=0.6.0',
'urllib3<1.27,>=1.25.4',
'typed-ast<1.5,>=1.4.0'
],
setup_requires=[
'pytest-runner>=5.2,<=5.3.2',
'setuptools_scm>=4.1.2,<=7.0.5'
],
extras_require={
'testing': tests_require,
},
Expand Down