diff --git a/WORKSPACE b/WORKSPACE index 55657b71fdec..247ffb439550 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -66,12 +66,7 @@ load("@npm_bazel_typescript//:index.bzl", "ts_setup_workspace") ts_setup_workspace() -# TODO(clarketm): when `io_bazel_rules_python` supports `python3` interpreter, replace "//:pip.bzl" w/ "@io_bazel_rules_python//python:pip.bzl" -# https://github.com/bazelbuild/rules_python/issues/85 -# https://github.com/bazelbuild/rules_python/issues/158 -# https://github.com/bazelbuild/rules_python/issues/179 -# https://github.com/bazelbuild/rules_python/issues/220 -load("//:pip.bzl", "pip_import") +load("@io_bazel_rules_python//python:pip.bzl", "pip_import") pip_import( name = "py_deps", diff --git a/experiment/find_developers.py b/experiment/find_developers.py index 930d3c27236e..ca20b017b028 100755 --- a/experiment/find_developers.py +++ b/experiment/find_developers.py @@ -43,7 +43,7 @@ def load_content(data): @functools.total_ordering -class User: # pylint: disable=too-few-public-methods +class User(object): # pylint: disable=too-few-public-methods """Store .user and number of .total and .recent commits.""" def __init__(self, blob): self.user = blob['author']['login'] diff --git a/experiment/generate_tests.py b/experiment/generate_tests.py index cb59480202c0..bf97ed5d000d 100755 --- a/experiment/generate_tests.py +++ b/experiment/generate_tests.py @@ -107,7 +107,7 @@ def apply_job_overrides(envs_or_args, job_envs_or_args): envs_or_args.append(job_env_or_arg) -class E2ENodeTest: +class E2ENodeTest(object): def __init__(self, job_name, job, config): self.job_name = job_name @@ -214,10 +214,10 @@ def generate(self): return job_config, prow_config, None -class E2ETest: +class E2ETest(object): def __init__(self, output_dir, job_name, job, config): - self.env_filename = os.path.join(output_dir, '%s.env' % job_name) + self.env_filename = os.path.join(output_dir, '%s.env' % job_name), self.job_name = job_name self.job = job self.common = config['common'] diff --git a/experiment/graphql_issue_example.py b/experiment/graphql_issue_example.py index af72e4a67136..97d54062e2ff 100755 --- a/experiment/graphql_issue_example.py +++ b/experiment/graphql_issue_example.py @@ -16,6 +16,8 @@ # USAGE: find_issues.py +# Required for pylint: 1.9.4 to tokenize the python3 print function. +from __future__ import print_function import sys import json diff --git a/experiment/parse_build_log.py b/experiment/parse_build_log.py index f7f9d9a7f1ba..1d3cacf5c751 100755 --- a/experiment/parse_build_log.py +++ b/experiment/parse_build_log.py @@ -29,7 +29,7 @@ _CURRENT_YEAR = datetime.datetime.utcnow().year -class TestOutput: +class TestOutput(object): def __init__(self): self._lines = [] self._start = None diff --git a/experiment/prepare_release_branch.py b/experiment/prepare_release_branch.py index 4679646d301b..b1c7d5eb8c49 100644 --- a/experiment/prepare_release_branch.py +++ b/experiment/prepare_release_branch.py @@ -22,8 +22,8 @@ import glob import re -import ruamel.yaml as yaml import sh +import ruamel.yaml as yaml TEST_CONFIG_YAML = "experiment/test_config.yaml" diff --git a/hack/BUILD.bazel b/hack/BUILD.bazel index 3648e2711102..f81022ae5ed2 100644 --- a/hack/BUILD.bazel +++ b/hack/BUILD.bazel @@ -226,22 +226,27 @@ test_suite( py_binary( name = "pylint_bin", srcs = ["pylint_bin.py"], - python_version = "PY3", + python_version = "PY2", tags = ["lint"], # NOTE: this should only contain direct third party imports and pylint deps = [ requirement("astroid"), + requirement("backports.functools_lru_cache"), + requirement("configparser"), + requirement("enum34"), requirement("influxdb"), requirement("isort"), requirement("lazy-object-proxy"), - requirement("mccabe"), requirement("pylint"), + requirement("pytz"), + requirement("PyYAML"), requirement("requests"), - requirement("ruamel.yaml"), - requirement("typed-ast"), + requirement("setuptools"), requirement("sh"), + requirement("singledispatch"), requirement("six"), requirement("wrapt"), + requirement("ruamel.yaml"), ], ) diff --git a/hack/pylint_bin.py b/hack/pylint_bin.py index 859dba38f8ec..8a4cb2fb95ff 100755 --- a/hack/pylint_bin.py +++ b/hack/pylint_bin.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python2 # Copyright 2017 The Kubernetes Authors. # diff --git a/hack/verify-pylint.sh b/hack/verify-pylint.sh index 3537d3039edd..314b98a79d93 100755 --- a/hack/verify-pylint.sh +++ b/hack/verify-pylint.sh @@ -36,5 +36,6 @@ export PYLINTHOME=$TEST_TMPDIR shopt -s extglob globstar -# TODO(clarketm): remove `boskos` exclusion after upgrading to PY3. -"$DIR/pylint_bin" !(gubernator|external|vendor|jenkins|scenarios|triage|boskos|bazel-*)/**/*.py +# TODO(clarketm) there is no version of `pylint` that supports "both" PY2 and PY3 +# I am disabling pylint checks for python3 files until migration complete +"$DIR/pylint_bin" !(kettle|metrics|triage|velodrome|hack|gubernator|external|vendor|testgrid|bazel-*)/**/*.py diff --git a/kettle/make_db.py b/kettle/make_db.py index 9afac7cff9dc..fa2c2e0981d2 100755 --- a/kettle/make_db.py +++ b/kettle/make_db.py @@ -28,8 +28,8 @@ import multiprocessing import multiprocessing.pool -import ruamel.yaml as yaml import requests +import ruamel.yaml as yaml import model diff --git a/kettle/make_json.py b/kettle/make_json.py index c8b02fc490aa..ccd7a75f4ef3 100755 --- a/kettle/make_json.py +++ b/kettle/make_json.py @@ -88,10 +88,7 @@ def buckets_yaml(): # pypy compatibility hack def python_buckets_yaml(python='python3'): return json.loads(subprocess.check_output( - [python, - '-c', - 'import json, ruamel.yaml as yaml; print(json.dumps(yaml.safe_load(open("buckets.yaml"))))' - ], + [python, '-c', 'import json, ruamel.yaml as yaml; print(json.dumps(yaml.safe_load(open("buckets.yaml"))))'], cwd=os.path.dirname(os.path.abspath(__file__))).decode("utf-8")) for attempt in [python_buckets_yaml, buckets_yaml, lambda: python_buckets_yaml(python='python')]: diff --git a/kettle/make_json_test.py b/kettle/make_json_test.py index 1ecf5118b24b..31cc54a260e5 100755 --- a/kettle/make_json_test.py +++ b/kettle/make_json_test.py @@ -135,7 +135,7 @@ def expect(args, needles, negneedles, expected_ret=None): for needle in negneedles: # Only match negative needles in the middle of a word, to avoid # failures on timestamps that happen to contain a short number. - self.assertNotRegexpMatches(result, r'\b%s\b' % needle) # pylint: disable=deprecated-method + self.assertNotRegexpMatches(result, r'\b%s\b' % needle) add_build('some-job/123', last_month, last_month + 10, 'SUCCESS', junits) add_build('some-job/456', now - 10, now, 'FAILURE', junits) diff --git a/kettle/stream.py b/kettle/stream.py index 543463b8df23..60b02212f83e 100755 --- a/kettle/stream.py +++ b/kettle/stream.py @@ -69,16 +69,15 @@ def get_started_finished(gcs_client, db, todo): pool = multiprocessing.pool.ThreadPool(16) try: for ack_id, (build_dir, started, finished) in pool.imap_unordered( - lambda ack_id_job_build: (ack_id_job_build[0], gcs_client.get_started_finished( - ack_id_job_build[1], ack_id_job_build[2])), + lambda ack_id_job_build: (ack_id_job_build[0], gcs_client.get_started_finished(ack_id_job_build[1], ack_id_job_build[2])), todo): if finished: if not db.insert_build(build_dir, started, finished): print('already present??') start = time.localtime(started.get('timestamp', 0) if started else 0) print((build_dir, bool(started), bool(finished), - time.strftime('%F %T %Z', start), - finished and finished.get('result'))) + time.strftime('%F %T %Z', start), + finished and finished.get('result'))) build_dirs.append(build_dir) acks.append(ack_id) else: diff --git a/metrics/bigquery.py b/metrics/bigquery.py index 35fbd5746b51..378862f0209f 100755 --- a/metrics/bigquery.py +++ b/metrics/bigquery.py @@ -28,9 +28,9 @@ import time import traceback -import ruamel.yaml as yaml import influxdb import requests +import ruamel.yaml as yaml def check(cmd, **kwargs): @@ -65,7 +65,7 @@ def do_jq(jq_filter, data_filename, out_filename, jq_bin='jq'): check([jq_bin, jq_filter, data_filename], stdout=out_file) -class BigQuerier: +class BigQuerier(object): def __init__(self, project, bucket_path, backfill_days, influx_client): if not project: raise ValueError('project', project) diff --git a/pip.bzl b/pip.bzl deleted file mode 100644 index 638a916a243d..000000000000 --- a/pip.bzl +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright 2017 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Import pip requirements into Bazel.""" - -def _pip_import_impl(repository_ctx): - """Core implementation of pip_import.""" - - # Add an empty top-level BUILD file. - # This is because Bazel requires BUILD files along all paths accessed - # via //this/sort/of:path and we wouldn't be able to load our generated - # requirements.bzl without it. - repository_ctx.file("BUILD", "") - - # To see the output, pass: quiet=False - result = repository_ctx.execute([ - "python3", - repository_ctx.path(repository_ctx.attr._script), - "--name", - repository_ctx.attr.name, - "--input", - repository_ctx.path(repository_ctx.attr.requirements), - "--output", - repository_ctx.path("requirements.bzl"), - "--directory", - repository_ctx.path(""), - ]) - - if result.return_code: - fail("pip_import failed: %s (%s)" % (result.stdout, result.stderr)) - -pip_import = repository_rule( - attrs = { - "requirements": attr.label( - mandatory = True, - allow_single_file = True, - ), - "_script": attr.label( - executable = True, - default = Label("@io_bazel_rules_python//tools:piptool.par"), - cfg = "host", - ), - }, - implementation = _pip_import_impl, -) - -"""A rule for importing requirements.txt dependencies into Bazel. - -This rule imports a requirements.txt file Λ™and generates a new -requirements.bzl file. This is used via the WORKSPACE -pattern: -
pip_import(
-    name = "foo",
-    requirements = ":requirements.txt",
-)
-load("@foo//:requirements.bzl", "pip_install")
-pip_install()
-
- -You can then reference imported dependencies from your BUILD -file with: -
load("@foo//:requirements.bzl", "requirement")
-py_library(
-    name = "bar",
-    ...
-    deps = [
-       "//my/other:dep",
-       requirement("futures"),
-       requirement("mock"),
-    ],
-)
-
- -Or alternatively: -
load("@foo//:requirements.bzl", "all_requirements")
-py_binary(
-    name = "baz",
-    ...
-    deps = [
-       ":foo",
-    ] + all_requirements,
-)
-
- -Args: - requirements: The label of a requirements.txt file. -""" - -def pip_repositories(): - """Pull in dependencies needed to use the packaging rules.""" - - # At the moment this is a placeholder, in that it does not actually pull in - # any dependencies. However, it does do some validation checking. - # - # As a side effect of migrating our canonical workspace name from - # "@io_bazel_rules_python" to "@rules_python" (#203), users who still - # imported us by the old name would get a confusing error about a - # repository dependency cycle in their workspace. (The cycle is likely - # related to the fact that our repo name is hardcoded into the template - # in piptool.py.) - # - # To produce a more informative error message in this situation, we - # fail-fast here if we detect that we're not being imported by the new - # name. (I believe we have always had the requirement that we're imported - # by the canonical name, because of the aforementioned hardcoding.) - # - # Users who, against best practice, do not call pip_repositories() in their - # workspace will not benefit from this check. - if "rules_python" not in native.existing_rules(): - message = "=" * 79 + """\n\ -It appears that you are trying to import rules_python without using its -canonical name, "@rules_python". This does not work. Please change your -WORKSPACE file to import this repo with `name = "rules_python"` instead. -""" - if "io_bazel_rules_python" in native.existing_rules(): - message += """\n\ -Note that the previous name of "@io_bazel_rules_python" is no longer used. -See https://github.com/bazelbuild/rules_python/issues/203 for context. -""" - message += "=" * 79 - fail(message) diff --git a/requirements.txt b/requirements.txt index b9f21f28a66a..534e6a3a0895 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,19 @@ -astroid==2.2.5 +astroid==1.6.6 +backports.functools_lru_cache==1.5 certifi==2019.6.16 chardet==3.0.4 +configparser==3.8.1 +enum34==1.1.6 google-cloud-bigquery==1.18.0 google-cloud-pubsub==0.45.0 idna==2.8 influxdb==5.2.2 isort==4.3.21 jinja2==2.10.1 -lazy-object-proxy==1.4.2 +lazy-object-proxy==1.4.1 MarkupSafe==1.1.1 mccabe==0.6.1 -pylint==2.3.1 +pylint==1.9.5 python-dateutil==2.8.0 pytz==2019.2 PyYAML==5.1.1 @@ -18,8 +21,8 @@ requests==2.22.0 ruamel.yaml==0.15.97 setuptools==41.1.0 sh==1.12.14 +singledispatch==3.4.0.3 six==1.12.0 -typed-ast==1.4.0 urllib3==1.25.3 webapp2==2.5.2 wrapt==1.11.2 diff --git a/testgrid/conformance/upload_e2e.py b/testgrid/conformance/upload_e2e.py index ca858d1ac92e..cb2920d54af2 100755 --- a/testgrid/conformance/upload_e2e.py +++ b/testgrid/conformance/upload_e2e.py @@ -32,6 +32,8 @@ # # Usage: see README.md +# Required for pylint: 1.9.4 to tokenize the python3 print function. +from __future__ import print_function import re import sys diff --git a/triage/summarize.py b/triage/summarize.py index d6a4cd4ad69f..6e14f193a590 100755 --- a/triage/summarize.py +++ b/triage/summarize.py @@ -484,8 +484,7 @@ def render(builds, clustered): SIG_LABEL_RE = re.compile(r'\[sig-([^]]*)\]') - -def annotate_owners(data, builds, owners): # pylint: disable=too-many-branches +def annotate_owners(data, builds, owners): """ Assign ownership to a cluster based on the share of hits in the last day. """ diff --git a/triage/summarize_test.py b/triage/summarize_test.py index 265d02ab7994..fc0e2f50ed7f 100755 --- a/triage/summarize_test.py +++ b/triage/summarize_test.py @@ -119,8 +119,7 @@ def expect(test, owner, owners=None): 'cols': {'started': [now]} }, 'clustered': [ - {'tests': [{'name': test, 'jobs': [ - {'name': 'somejob', 'builds': ['123', '125']}]}]} + {'tests': [{'name': test, 'jobs': [{'name': 'somejob', 'builds': ['123', '125']}]}]} ], } summarize.annotate_owners(