+
+ As of January 1, 2020 this library no longer supports Python 2 on the latest released version.
+ Library versions released prior to that date will continue to be available. For more information please
+ visit
Python 2 support on Google Cloud.
+
{% block body %} {% endblock %}
diff --git a/docs/conf.py b/docs/conf.py
index 3ab5be5103..af8c5efda8 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-# Copyright 2021 Google LLC
+# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -25,8 +25,8 @@
# serve to show the default.
import os
+import shlex
import sys
-from typing import Any, Dict
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
@@ -80,8 +80,8 @@
root_doc = "index"
# General information about the project.
-project = "BigQuery DataFrames"
-copyright = "2022-2023 Google LLC"
+project = "bigframes"
+copyright = "2019, Google"
author = "Google APIs"
# The version info for the project you're documenting, acts as replacement for
@@ -98,7 +98,7 @@
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
-language = "en"
+language = None
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
@@ -154,10 +154,10 @@
# further. For a list of options available for each theme, see the
# documentation.
html_theme_options = {
- "description": "BigQuery DataFrames provides DataFrame APIs on the BigQuery engine.",
- # "github_user": "googleapis",
- # "github_repo": "python-bigquery-storage",
- # "github_banner": True,
+ "description": "BigQuery DataFrames provides DataFrame APIs on the BigQuery engine",
+ "github_user": "googleapis",
+ "github_repo": "python-bigquery-dataframes",
+ "github_banner": True,
"font_family": "'Roboto', Georgia, sans",
"head_font_family": "'Roboto', Georgia, serif",
"code_font_family": "'Roboto Mono', 'Consolas', monospace",
@@ -185,7 +185,7 @@
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
-# html_static_path = ["_static"]
+html_static_path = ["_static"]
# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
@@ -264,13 +264,15 @@
# -- Options for LaTeX output ---------------------------------------------
-latex_elements: Dict[str, Any] = {
- # Avoid "too deeply nested" error by using enumitem package.
- # See: https://stackoverflow.com/a/28454426/101923
- "preamble": r"""
-\usepackage{enumitem}
-\setlistdepth{99}
-"""
+latex_elements = {
+ # The paper size ('letterpaper' or 'a4paper').
+ #'papersize': 'letterpaper',
+ # The font size ('10pt', '11pt' or '12pt').
+ #'pointsize': '10pt',
+ # Additional stuff for the LaTeX preamble.
+ #'preamble': '',
+ # Latex figure (float) alignment
+ #'figure_align': 'htbp',
}
# Grouping the document tree into LaTeX files. List of tuples
@@ -366,6 +368,11 @@
"grpc": ("https://grpc.github.io/grpc/python/", None),
"proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None),
"protobuf": ("https://googleapis.dev/python/protobuf/latest/", None),
+ "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None),
+ "pydata-google-auth": (
+ "https://pydata-google-auth.readthedocs.io/en/latest/",
+ None,
+ ),
}
diff --git a/noxfile.py b/noxfile.py
index 1ceca6831b..2355a9b27b 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -31,7 +31,7 @@
BLACK_VERSION = "black==22.3.0"
ISORT_VERSION = "isort==5.12.0"
SPHINX_VERSION = "sphinx==4.5.0"
-LINT_PATHS = ["docs", "bigframes", "tests", "noxfile.py", "setup.py"]
+LINT_PATHS = ["docs", "bigframes", "tests", "third_party", "noxfile.py", "setup.py"]
DEFAULT_PYTHON_VERSION = "3.10"
@@ -108,6 +108,7 @@ def lint(session):
"--check",
*LINT_PATHS,
)
+ # TODO(tswast): lint all LINT_PATHS
session.run("flake8", "bigframes", "tests")
diff --git a/owlbot.py b/owlbot.py
index 4ba7d14eb5..be30eea5c2 100644
--- a/owlbot.py
+++ b/owlbot.py
@@ -15,6 +15,7 @@
"""This script is used to synthesize generated parts of this library."""
import pathlib
+import re
from synthtool import gcp
import synthtool as s
@@ -27,11 +28,10 @@
# ----------------------------------------------------------------------------
# Add templated files
# ----------------------------------------------------------------------------
-
templated_files = common.py_library(
unit_test_python_versions=["3.9", "3.10", "3.11"],
system_test_python_versions=["3.9", "3.11"],
- cov_level=40,
+ cov_level=35,
intersphinx_dependencies={
"pandas": "https://pandas.pydata.org/pandas-docs/stable/",
"pydata-google-auth": "https://pydata-google-auth.readthedocs.io/en/latest/",
@@ -40,11 +40,17 @@
s.move(
templated_files,
excludes=[
- # Multi-processing note isn't relevant, as pandas_gbq is responsible for
+ # Multi-processing note isn't relevant, as bigframes is responsible for
# creating clients, not the end user.
"docs/multiprocessing.rst",
"noxfile.py",
+ ".pre-commit-config.yaml",
"README.rst",
+ ".github/release-trigger.yml",
+ # BigQuery DataFrames manages its own Kokoro cluster for presubmit & continuous tests.
+ ".kokoro/build.sh",
+ ".kokoro/continuous/common.cfg",
+ ".kokoro/presubmit/common.cfg",
],
)
@@ -52,6 +58,46 @@
# Fixup files
# ----------------------------------------------------------------------------
+# Make sure build includes all necessary files.
+s.replace(
+ ["MANIFEST.in"],
+ re.escape("recursive-include google"),
+ "recursive-include third_party *\nrecursive-include bigframes",
+)
+
+# Even though BigQuery DataFrames isn't technically a client library, we are
+# opting into Cloud RAD for docs hosting.
+s.replace(
+ [".kokoro/docs/common.cfg"],
+ re.escape('value: "docs-staging-v2-staging"'),
+ 'value: "docs-staging-v2"',
+)
+
+# Use a custom table of contents since the default one isn't organized well
+# enough for the number of classes we have.
+s.replace(
+ [".kokoro/publish-docs.sh"],
+ (
+ re.escape("# upload docs")
+ + "\n"
+ + re.escape(
+ 'python3 -m docuploader upload docs/_build/html/docfx_yaml --metadata-file docs.metadata --destination-prefix docfx --staging-bucket "${V2_STAGING_BUCKET}"'
+ )
+ ),
+ (
+ "# Replace toc.yml template file\n"
+ + "mv docs/templates/toc.yml docs/_build/html/docfx_yaml/toc.yml\n\n"
+ + "# upload docs\n"
+ + 'python3 -m docuploader upload docs/_build/html/docfx_yaml --metadata-file docs.metadata --destination-prefix docfx --staging-bucket "${V2_STAGING_BUCKET}"'
+ ),
+)
+
+# Fixup the documentation.
+s.replace(
+ ["docs/conf.py"],
+ re.escape("Google Cloud Client Libraries for bigframes"),
+ "BigQuery DataFrames provides DataFrame APIs on the BigQuery engine",
+)
# ----------------------------------------------------------------------------
# Samples templates
@@ -63,6 +109,6 @@
# Final cleanup
# ----------------------------------------------------------------------------
-s.shell.run(["nox", "-s", "blacken"], hide_output=False)
+s.shell.run(["nox", "-s", "format"], hide_output=False)
for noxfile in REPO_ROOT.glob("samples/**/noxfile.py"):
s.shell.run(["nox", "-s", "blacken"], cwd=noxfile.parent, hide_output=False)
diff --git a/renovate.json b/renovate.json
new file mode 100644
index 0000000000..39b2a0ec92
--- /dev/null
+++ b/renovate.json
@@ -0,0 +1,12 @@
+{
+ "extends": [
+ "config:base",
+ "group:all",
+ ":preserveSemverRanges",
+ ":disableDependencyDashboard"
+ ],
+ "ignorePaths": [".pre-commit-config.yaml", ".kokoro/requirements.txt", "setup.py"],
+ "pip_requirements": {
+ "fileMatch": ["requirements-test.txt", "samples/[\\S/]*constraints.txt", "samples/[\\S/]*constraints-test.txt"]
+ }
+}
diff --git a/scripts/decrypt-secrets.sh b/scripts/decrypt-secrets.sh
new file mode 100755
index 0000000000..0018b421dd
--- /dev/null
+++ b/scripts/decrypt-secrets.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Copyright 2023 Google LLC All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+ROOT=$( dirname "$DIR" )
+
+# Work from the project root.
+cd $ROOT
+
+# Prevent it from overriding files.
+# We recommend that sample authors use their own service account files and cloud project.
+# In that case, they are supposed to prepare these files by themselves.
+if [[ -f "testing/test-env.sh" ]] || \
+ [[ -f "testing/service-account.json" ]] || \
+ [[ -f "testing/client-secrets.json" ]]; then
+ echo "One or more target files exist, aborting."
+ exit 1
+fi
+
+# Use SECRET_MANAGER_PROJECT if set, fallback to cloud-devrel-kokoro-resources.
+PROJECT_ID="${SECRET_MANAGER_PROJECT:-cloud-devrel-kokoro-resources}"
+
+gcloud secrets versions access latest --secret="python-docs-samples-test-env" \
+ --project="${PROJECT_ID}" \
+ > testing/test-env.sh
+gcloud secrets versions access latest \
+ --secret="python-docs-samples-service-account" \
+ --project="${PROJECT_ID}" \
+ > testing/service-account.json
+gcloud secrets versions access latest \
+ --secret="python-docs-samples-client-secrets" \
+ --project="${PROJECT_ID}" \
+ > testing/client-secrets.json
diff --git a/scripts/readme-gen/readme_gen.py b/scripts/readme-gen/readme_gen.py
new file mode 100644
index 0000000000..1acc119835
--- /dev/null
+++ b/scripts/readme-gen/readme_gen.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Generates READMEs using configuration defined in yaml."""
+
+import argparse
+import io
+import os
+import subprocess
+
+import jinja2
+import yaml
+
+
+jinja_env = jinja2.Environment(
+ trim_blocks=True,
+ loader=jinja2.FileSystemLoader(
+ os.path.abspath(os.path.join(os.path.dirname(__file__), "templates"))
+ ),
+ autoescape=True,
+)
+
+README_TMPL = jinja_env.get_template("README.tmpl.rst")
+
+
+def get_help(file):
+ return subprocess.check_output(["python", file, "--help"]).decode()
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("source")
+ parser.add_argument("--destination", default="README.rst")
+
+ args = parser.parse_args()
+
+ source = os.path.abspath(args.source)
+ root = os.path.dirname(source)
+ destination = os.path.join(root, args.destination)
+
+ jinja_env.globals["get_help"] = get_help
+
+ with io.open(source, "r") as f:
+ config = yaml.load(f)
+
+ # This allows get_help to execute in the right directory.
+ os.chdir(root)
+
+ output = README_TMPL.render(config)
+
+ with io.open(destination, "w") as f:
+ f.write(output)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/readme-gen/templates/README.tmpl.rst b/scripts/readme-gen/templates/README.tmpl.rst
new file mode 100644
index 0000000000..4fd239765b
--- /dev/null
+++ b/scripts/readme-gen/templates/README.tmpl.rst
@@ -0,0 +1,87 @@
+{# The following line is a lie. BUT! Once jinja2 is done with it, it will
+ become truth! #}
+.. This file is automatically generated. Do not edit this file directly.
+
+{{product.name}} Python Samples
+===============================================================================
+
+.. image:: https://gstatic.com/cloudssh/images/open-btn.png
+ :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor={{folder}}/README.rst
+
+
+This directory contains samples for {{product.name}}. {{product.description}}
+
+{{description}}
+
+.. _{{product.name}}: {{product.url}}
+
+{% if required_api_url %}
+To run the sample, you need to enable the API at: {{required_api_url}}
+{% endif %}
+
+{% if required_role %}
+To run the sample, you need to have `{{required_role}}` role.
+{% endif %}
+
+{{other_required_steps}}
+
+{% if setup %}
+Setup
+-------------------------------------------------------------------------------
+
+{% for section in setup %}
+
+{% include section + '.tmpl.rst' %}
+
+{% endfor %}
+{% endif %}
+
+{% if samples %}
+Samples
+-------------------------------------------------------------------------------
+
+{% for sample in samples %}
+{{sample.name}}
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+{% if not sample.hide_cloudshell_button %}
+.. image:: https://gstatic.com/cloudssh/images/open-btn.png
+ :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor={{folder}}/{{sample.file}},{{folder}}/README.rst
+{% endif %}
+
+
+{{sample.description}}
+
+To run this sample:
+
+.. code-block:: bash
+
+ $ python {{sample.file}}
+{% if sample.show_help %}
+
+ {{get_help(sample.file)|indent}}
+{% endif %}
+
+
+{% endfor %}
+{% endif %}
+
+{% if cloud_client_library %}
+
+The client library
+-------------------------------------------------------------------------------
+
+This sample uses the `Google Cloud Client Library for Python`_.
+You can read the documentation for more details on API usage and use GitHub
+to `browse the source`_ and `report issues`_.
+
+.. _Google Cloud Client Library for Python:
+ https://googlecloudplatform.github.io/google-cloud-python/
+.. _browse the source:
+ https://github.com/GoogleCloudPlatform/google-cloud-python
+.. _report issues:
+ https://github.com/GoogleCloudPlatform/google-cloud-python/issues
+
+{% endif %}
+
+.. _Google Cloud SDK: https://cloud.google.com/sdk/
\ No newline at end of file
diff --git a/scripts/readme-gen/templates/auth.tmpl.rst b/scripts/readme-gen/templates/auth.tmpl.rst
new file mode 100644
index 0000000000..1446b94a5e
--- /dev/null
+++ b/scripts/readme-gen/templates/auth.tmpl.rst
@@ -0,0 +1,9 @@
+Authentication
+++++++++++++++
+
+This sample requires you to have authentication setup. Refer to the
+`Authentication Getting Started Guide`_ for instructions on setting up
+credentials for applications.
+
+.. _Authentication Getting Started Guide:
+ https://cloud.google.com/docs/authentication/getting-started
diff --git a/scripts/readme-gen/templates/auth_api_key.tmpl.rst b/scripts/readme-gen/templates/auth_api_key.tmpl.rst
new file mode 100644
index 0000000000..11957ce271
--- /dev/null
+++ b/scripts/readme-gen/templates/auth_api_key.tmpl.rst
@@ -0,0 +1,14 @@
+Authentication
+++++++++++++++
+
+Authentication for this service is done via an `API Key`_. To obtain an API
+Key:
+
+1. Open the `Cloud Platform Console`_
+2. Make sure that billing is enabled for your project.
+3. From the **Credentials** page, create a new **API Key** or use an existing
+ one for your project.
+
+.. _API Key:
+ https://developers.google.com/api-client-library/python/guide/aaa_apikeys
+.. _Cloud Console: https://console.cloud.google.com/project?_
diff --git a/scripts/readme-gen/templates/install_deps.tmpl.rst b/scripts/readme-gen/templates/install_deps.tmpl.rst
new file mode 100644
index 0000000000..6f069c6c87
--- /dev/null
+++ b/scripts/readme-gen/templates/install_deps.tmpl.rst
@@ -0,0 +1,29 @@
+Install Dependencies
+++++++++++++++++++++
+
+#. Clone python-docs-samples and change directory to the sample directory you want to use.
+
+ .. code-block:: bash
+
+ $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git
+
+#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions.
+
+ .. _Python Development Environment Setup Guide:
+ https://cloud.google.com/python/setup
+
+#. Create a virtualenv. Samples are compatible with Python 3.7+.
+
+ .. code-block:: bash
+
+ $ virtualenv env
+ $ source env/bin/activate
+
+#. Install the dependencies needed to run the samples.
+
+ .. code-block:: bash
+
+ $ pip install -r requirements.txt
+
+.. _pip: https://pip.pypa.io/
+.. _virtualenv: https://virtualenv.pypa.io/
diff --git a/scripts/readme-gen/templates/install_portaudio.tmpl.rst b/scripts/readme-gen/templates/install_portaudio.tmpl.rst
new file mode 100644
index 0000000000..5ea33d18c0
--- /dev/null
+++ b/scripts/readme-gen/templates/install_portaudio.tmpl.rst
@@ -0,0 +1,35 @@
+Install PortAudio
++++++++++++++++++
+
+Install `PortAudio`_. This is required by the `PyAudio`_ library to stream
+audio from your computer's microphone. PyAudio depends on PortAudio for cross-platform compatibility, and is installed differently depending on the
+platform.
+
+* For Mac OS X, you can use `Homebrew`_::
+
+ brew install portaudio
+
+ **Note**: if you encounter an error when running `pip install` that indicates
+ it can't find `portaudio.h`, try running `pip install` with the following
+ flags::
+
+ pip install --global-option='build_ext' \
+ --global-option='-I/usr/local/include' \
+ --global-option='-L/usr/local/lib' \
+ pyaudio
+
+* For Debian / Ubuntu Linux::
+
+ apt-get install portaudio19-dev python-all-dev
+
+* Windows may work without having to install PortAudio explicitly (it will get
+ installed with PyAudio).
+
+For more details, see the `PyAudio installation`_ page.
+
+
+.. _PyAudio: https://people.csail.mit.edu/hubert/pyaudio/
+.. _PortAudio: http://www.portaudio.com/
+.. _PyAudio installation:
+ https://people.csail.mit.edu/hubert/pyaudio/#downloads
+.. _Homebrew: http://brew.sh
diff --git a/scripts/upload_to_google_drive.py b/scripts/upload_to_google_drive.py
index e579151359..dcdc9168ba 100644
--- a/scripts/upload_to_google_drive.py
+++ b/scripts/upload_to_google_drive.py
@@ -41,12 +41,9 @@
wheel_id = "15fZ1DkrFDk4ibMNTzms4akpxmf2pzeAR"
wheel_path = next(iter((repo_root / "dist").glob("bigframes-*.whl")))
-pdf_id = "1agYjxmPLrxelsaHI-lc41QHcgnQYemcX"
-pdf_path = repo_root / "docs" / "_build" / "latex" / "bigframes-latest.pdf"
-
uploads = (
(wheel_id, wheel_path, "application/octet-stream"),
- (pdf_id, pdf_path, "application/pdf"),
+ # (pdf_id, pdf_path, "application/pdf"),
)
upload_template = (
diff --git a/setup.cfg b/setup.cfg
index 8bd749387e..0523500895 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
#
-# Copyright 2020 Google LLC
+# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -17,17 +17,3 @@
# Generated by synthtool. DO NOT EDIT!
[bdist_wheel]
universal = 1
-
-[pytype]
-python_version = 3.9
-inputs =
- google/cloud/
-exclude =
- tests/
- google/cloud/bigquery_v2/ # Legacy proto-based types.
-output = .pytype/
-disable =
- # There's some issue with finding some pyi files, thus disabling.
- # The issue https://github.com/google/pytype/issues/150 is closed, but the
- # error still occurs for some reason.
- pyi-error
diff --git a/testing/.gitignore b/testing/.gitignore
new file mode 100644
index 0000000000..b05fbd6308
--- /dev/null
+++ b/testing/.gitignore
@@ -0,0 +1,3 @@
+test-env.sh
+service-account.json
+client-secrets.json
\ No newline at end of file