From 93d176329959a2a5704b0c753666a9413d81959d Mon Sep 17 00:00:00 2001 From: robnagler <5495179+robnagler@users.noreply.github.com> Date: Thu, 20 Jun 2024 00:02:40 +0000 Subject: [PATCH 1/4] Fix #466 projex generates pyproject.toml - pksetup: removed tox, readthedocs support Fix #429 added readthedocs.yml Fix #386 projex generates proper sphinx and readthedocs.yml Fix #393 added pkcli.sphinx.prepare for readthedocs - Added sphinx test which also tests projex - Fixed various RST errors in modules Fix #449 use GitHub workflow actions/checkout@v4 (also projex) --- .github/workflows/python-ci.yml | 2 +- .gitignore | 1 - .readthedocs.yml | 12 + docs/index.rst | 4 + pykern/fconf.py | 4 +- .../projex-licenses/proprietary.jinja | 3 +- .../.github/workflows/python-ci.yml.jinja | 2 +- .../package_data/projex/docs/index.rst.jinja | 4 + .../package_data/projex/dot-gitignore.jinja | 3 +- .../projex/dot-readthedocs.yml.jinja | 12 + .../package_data/projex/pyproject.toml.jinja | 36 +++ pykern/package_data/projex/setup.py.jinja | 25 -- .../projex/tests/import_test.py.jinja | 7 +- .../{docs-conf.py.format => sphinx-conf.py} | 196 ++++++------- pykern/pkcli/projex.py | 22 +- pykern/pkcli/sphinx.py | 106 +++++++ pykern/pkcollections.py | 3 +- pykern/pkconfig.py | 2 +- pykern/pkio.py | 2 +- pykern/pksetup.py | 270 ------------------ pykern/pkunit.py | 8 +- pyproject.toml | 1 + tests/pkcli/sphinx_test.py | 49 ++++ 23 files changed, 335 insertions(+), 439 deletions(-) create mode 100644 .readthedocs.yml create mode 100644 pykern/package_data/projex/dot-readthedocs.yml.jinja create mode 100644 pykern/package_data/projex/pyproject.toml.jinja delete mode 100644 pykern/package_data/projex/setup.py.jinja rename pykern/package_data/{docs-conf.py.format => sphinx-conf.py} (69%) create mode 100644 pykern/pkcli/sphinx.py create mode 100644 tests/pkcli/sphinx_test.py diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 54748fd4..aa976489 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -13,7 +13,7 @@ jobs: python-ci: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: curl -LsS https://radia.run | bash -s python-ci diff --git a/.gitignore b/.gitignore index f488a579..b19f8bb0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -.eggs/ MANIFEST.in pytest.ini tox.ini diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..a4206602 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,12 @@ +# Read The Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details +version: 2 +build: + os: ubuntu-22.04 + tools: + python: "3.9" + jobs: + pre_build: + - curl https://radia.run | bash -s readthedocs +sphinx: + configuration: docs/conf.py diff --git a/docs/index.rst b/docs/index.rst index b1e65e54..b78593a6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,6 +6,10 @@ Python programs and servers. PyKern defines policies, which make it easy to eliminate boiler-plate. +.. autosummary:: + :toctree: _autosummary + :recursive: + .. toctree:: :maxdepth: 2 diff --git a/pykern/fconf.py b/pykern/fconf.py index 38aaa090..493c7053 100644 --- a/pykern/fconf.py +++ b/pykern/fconf.py @@ -4,7 +4,7 @@ PKDict. FConf is not a replacement for `pykern.pkconfig`. Rather it is for complex configuration input files to programs that often require programmatic generation. FConf was written for -RSConf . +`RSConf `_. The Basic YAML configuration look like this:: @@ -306,7 +306,7 @@ def parse_all(path, base_vars=None): YAML files next. YAML file evaluation happens in that same order. Args: - path (py.path): directory that *.py and *.yml files + path (py.path): directory that ``*.py`` and ``*.yml`` files base_vars (PKDict): initial variable state. May be hierarchical. [None] Returns: PKDict: evaluated and merged files plus base_vars diff --git a/pykern/package_data/projex-licenses/proprietary.jinja b/pykern/package_data/projex-licenses/proprietary.jinja index 91f36882..d3fa8849 100644 --- a/pykern/package_data/projex-licenses/proprietary.jinja +++ b/pykern/package_data/projex-licenses/proprietary.jinja @@ -2,5 +2,4 @@ PROPRIETARY AND CONFIDENTIAL Copyright (C) {{ author }}. All Rights Reserved -Unauthorized copying of the contents of the contents of -this repository via any medium is strictly prohibited. +Unauthorized copying of the contents of this repository via any medium is strictly prohibited. diff --git a/pykern/package_data/projex/.github/workflows/python-ci.yml.jinja b/pykern/package_data/projex/.github/workflows/python-ci.yml.jinja index 42954ad7..cd332144 100644 --- a/pykern/package_data/projex/.github/workflows/python-ci.yml.jinja +++ b/pykern/package_data/projex/.github/workflows/python-ci.yml.jinja @@ -13,5 +13,5 @@ jobs: python-ci: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: curl -LsS https://radia.run | bash -s python-ci diff --git a/pykern/package_data/projex/docs/index.rst.jinja b/pykern/package_data/projex/docs/index.rst.jinja index 7280a71e..c04998c9 100644 --- a/pykern/package_data/projex/docs/index.rst.jinja +++ b/pykern/package_data/projex/docs/index.rst.jinja @@ -3,6 +3,10 @@ Welcome to {{ name }} {{ description }} +.. autosummary:: + :toctree: _autosummary + :recursive: + .. toctree:: :maxdepth: 2 diff --git a/pykern/package_data/projex/dot-gitignore.jinja b/pykern/package_data/projex/dot-gitignore.jinja index 3990c8b6..41fc8034 100644 --- a/pykern/package_data/projex/dot-gitignore.jinja +++ b/pykern/package_data/projex/dot-gitignore.jinja @@ -1,7 +1,6 @@ MANIFEST.in -tox.ini pytest.ini -pykern_setup.yaml +tox.ini .python-version .#* \#* diff --git a/pykern/package_data/projex/dot-readthedocs.yml.jinja b/pykern/package_data/projex/dot-readthedocs.yml.jinja new file mode 100644 index 00000000..a4206602 --- /dev/null +++ b/pykern/package_data/projex/dot-readthedocs.yml.jinja @@ -0,0 +1,12 @@ +# Read The Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details +version: 2 +build: + os: ubuntu-22.04 + tools: + python: "3.9" + jobs: + pre_build: + - curl https://radia.run | bash -s readthedocs +sphinx: + configuration: docs/conf.py diff --git a/pykern/package_data/projex/pyproject.toml.jinja b/pykern/package_data/projex/pyproject.toml.jinja new file mode 100644 index 00000000..c5975e5a --- /dev/null +++ b/pykern/package_data/projex/pyproject.toml.jinja @@ -0,0 +1,36 @@ +[build-system] +requires = ["chronver", "setuptools>=66"] +build-backend = "setuptools.build_meta" + +[project] +authors = [ + { name = "{{ author }}", email = "{{ author_email }}" }, +] +classifiers = [ + "Development Status :: 2 - Pre-Alpha", + "Environment :: Console", + "Intended Audience :: Developers", + "{{ classifier_license }}", + "Natural Language :: English", + "Programming Language :: Python", + "Topic :: Utilities", +] +dependencies = [ + "pykern", +] +description = "{{ description }}" +dynamic = ["version"] +name = "{{ name }}" +readme = "README.md" + +[project.scripts] +{{ name }} = "{{ name }}.{{ name }}_console:main" + +[project.urls] +Homepage = "http://git.radiasoft.org/{{ name }}" + +[tool.setuptools.package-data] +{{ name }} = ["package_data/**"] + +[tool.setuptools.packages.find] +include = ["{{ name }}*"] diff --git a/pykern/package_data/projex/setup.py.jinja b/pykern/package_data/projex/setup.py.jinja deleted file mode 100644 index 5b82a09d..00000000 --- a/pykern/package_data/projex/setup.py.jinja +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -"""{{ name }} setup script - -{{ copyright_license_rst }} -""" -import pykern.pksetup - -pykern.pksetup.setup( - name="{{ name }}", - author="{{ author }}", - author_email="{{ author_email }}", - description="{{ description }}", - install_requires=[ - "pykern", - ], - license="{{ license }}", - url="{{ url }}", - classifiers=[ - "Development Status :: 2 - Pre-Alpha", - "Intended Audience :: Developers", - "{{ classifier_license }}", - "Programming Language :: Python", - "Topic :: Utilities", - ], -) diff --git a/pykern/package_data/projex/tests/import_test.py.jinja b/pykern/package_data/projex/tests/import_test.py.jinja index a694e192..8367aa34 100644 --- a/pykern/package_data/projex/tests/import_test.py.jinja +++ b/pykern/package_data/projex/tests/import_test.py.jinja @@ -1,12 +1,9 @@ -# -*- coding: utf-8 -*- """Test that module imports. You should delete the test once you have real tests. Only necessary if you have no other tests so that -tox will work. +pykern.pkcli.test passes. """ -import pytest - -def test_1(): +def test_import(): import {{ name }} diff --git a/pykern/package_data/docs-conf.py.format b/pykern/package_data/sphinx-conf.py similarity index 69% rename from pykern/package_data/docs-conf.py.format rename to pykern/package_data/sphinx-conf.py index 601c1c59..eb55a614 100644 --- a/pykern/package_data/docs-conf.py.format +++ b/pykern/package_data/sphinx-conf.py @@ -1,15 +1,20 @@ -# -*- coding: utf-8 -*- -# -# OVERWRITTEN by pykern.pksetup every "python setup.py" build (egg, bdist) -# -# NOTE: If you add variables, make sure they are in triple-quoted strings so -# they escape any single or double quotes in description and author. -# -# {name} documentation build configuration file. +# Sphinx documentation build configuration file. # # This file is execfile()d with the current directory set to its # containing dir. -# + +# Dynamically filled variables +author = """$author""" +project = """$name""" +version = """$version""" +year = """$year""" + +####################################### +# The rest uses the python values above +####################################### + +project_documentation = f"{project} Documentation" + # Note that not all possible configuration values are present in this # autogenerated file. # @@ -28,90 +33,85 @@ # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', - 'sphinx.ext.mathjax', - 'sphinx.ext.ifconfig', - 'sphinx.ext.viewcode', - 'sphinx.ext.napoleon', + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "sphinx.ext.mathjax", + "sphinx.ext.ifconfig", + "sphinx.ext.viewcode", + "sphinx.ext.napoleon", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" -# General information about the project. -project = '''{name}''' - -copyright = '''{year}, {author}''' -author = '''{author}''' +copyright = f"{year}, {author}" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # -# The short X.Y version. -version = '''{version}''' # The full version, including alpha/beta/rc tags. -release = '''{version}''' +release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +exclude_patterns = ["_build"] # The reST default role (used for this markup: `text`) to use for all # documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = True @@ -121,181 +121,163 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'alabaster' +html_theme = "alabaster" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {{}} +html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {{}} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {{}} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Language to be used for generating the HTML full-text search index. # Sphinx supports the following languages: # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' -#html_search_language = 'en' +# html_search_language = 'en' # A dictionary with options for the search language support, empty by default. # Now only 'ja' uses this config value -#html_search_options = {{'type': 'default'}} +# html_search_options = {'type': 'default'} # The name of a javascript file (relative to the configuration directory) that # implements a search results scorer. If empty, the default will be used. -#html_search_scorer = 'scorer.js' +# html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. -htmlhelp_basename = '''{name}doc''' +htmlhelp_basename = f"{project}doc" # -- Options for LaTeX output --------------------------------------------- -latex_elements = {{ -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', - -# Latex figure (float) alignment -#'figure_align': 'htbp', -}} +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + #'preamble': '', + # Latex figure (float) alignment + #'figure_align': 'htbp', +} # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, '''{name}.tex''', '''{name} Documentation''', - '''{author}''', 'manual'), + (master_doc, f"{project}.tex", project_documentation, author, "manual"), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, '''{name}''', '''{name} Documentation''', - [author], 1) -] +man_pages = [(master_doc, project, project_documentation, [author], 1)] # If true, show URL addresses after external links. -#man_show_urls = False - +# man_show_urls = False -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, '''{name}''', '''{name} Documentation''', - author, '''{name}''', '''{description}''', - 'Miscellaneous'), -] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {{'https://docs.python.org/': None}} +intersphinx_mapping = {"python": ("https://docs.python.org/", None)} diff --git a/pykern/pkcli/projex.py b/pykern/pkcli/projex.py index a439f79a..18e4b31c 100644 --- a/pykern/pkcli/projex.py +++ b/pykern/pkcli/projex.py @@ -1,25 +1,21 @@ -# -*- coding: utf-8 -*- -"""Manage Python development projects +"""Initialize Python project directories :copyright: Copyright (c) 2015 Bivio Software, Inc. All Rights Reserved. :license: http://www.apache.org/licenses/LICENSE-2.0.html """ -from __future__ import absolute_import, division, print_function -from pykern.pkdebug import pkdc, pkdp +from pykern import pkcli +from pykern import pkio +from pykern import pkjinja +from pykern import pkresource +from pykern.pkdebug import pkdc, pkdp import copy import datetime import os +import py.path import re import subprocess -import py.path - -from pykern import pkcli -from pykern import pkio -from pykern import pkjinja -from pykern import pkresource - #: Default values DEFAULTS = { "year": datetime.datetime.now().year, @@ -90,7 +86,7 @@ def init_rs_tree(description): def init_tree(name, author, author_email, description, license, url): """Setup a project tree with: docs, tests, etc., and checkin to git. - Creates: setup.py, index.rst, project dir, _console.py, etc. + Creates: pyproject.toml, index.rst, project dir, _console.py, etc. Overwrites files if they exist without checking. Args: @@ -98,7 +94,7 @@ def init_tree(name, author, author_email, description, license, url): author (str): copyright holder, e.g. ``RadiaSoft LLC`` author_email (str): how to reach author, e.g. ``pip@pykern.org`` description (str): one-line summary of project - license (str): url of license + license (str): name of license, e.g. apache2, mit, and proprietary. url (str): website for project, e.g. http://pykern.org """ assert os.path.isdir(".git"), "Must be run from the root directory of the repo" diff --git a/pykern/pkcli/sphinx.py b/pykern/pkcli/sphinx.py new file mode 100644 index 00000000..804205e8 --- /dev/null +++ b/pykern/pkcli/sphinx.py @@ -0,0 +1,106 @@ +"""Generate documentation with Sphinx + +:copyright: Copyright (c) 2024 RadiaSoft LLC. All Rights Reserved. +:license: http://www.apache.org/licenses/LICENSE-2.0.html +""" + +from pykern.pkcollections import PKDict +from pykern.pkdebug import pkdc, pkdlog, pkdp +import email.utils +import importlib.metadata +import pykern.pkcli +import pykern.pkcollections +import pykern.pkio +import pykern.pkresource +import pykern.pksubprocess +import re +import toml + +_DOCS_SUBDIR = "docs" + + +def prepare(package): + """Create conf.py and run ``sphinx-apidoc`` + + conf.py is generated from `package` metadata so `package` must be + installed. + + Args: + package (str): package to build + + """ + + def _conf(substitutions): + rv = pykern.pkio.read_text(pykern.pkresource.file_path("sphinx-conf.py")) + for k, v in substitutions.items(): + rv = rv.replace("$" + k, v) + return rv + + def _parse_metadata(): + try: + return _vars(importlib.metadata.metadata(package)) + except importlib.metadata.PackageNotFoundError as e: + pykern.pkcli.command_error( + f"unable to import package={package}; run pip install ." + ) + + def _vars(metadata): + return _year( + PKDict( + author=email.utils.parseaddr(metadata["Author-email"])[0], + name=metadata["Name"], + version=metadata["Version"], + ), + ) + + def _year(rv): + m = re.search(r"^(\d{4})\d{4}\.\d+$", rv.version) + if m: + return rv.pkupdate(year=m.group(1)) + pykern.pkcli.command_error( + f"unable to parse version={rv.version} package={rv.name}" + ) + + d = pykern.pkio.py_path(_DOCS_SUBDIR) + if not d.check(dir=True): + pykern.pkcli.command_error( + f"missing directory={d.basename}; run from Python root directory" + ) + v = _parse_metadata() + pykern.pkio.write_text(d.join("conf.py"), _conf(v)) + pykern.pksubprocess.check_call_with_signals( + ("sphinx-apidoc", "--force", "-o", d.basename, v.name), + ) + + +def build(builder="html"): + """`prepare` and run ``sphinx-build`` for html + + Reads ``pyproject.toml`` to get project name. Creates ``_build_`` subdirectory + with the output. + + Args: + builder (str): types of builders. See `Sphinx builder list `_ [html] + + """ + + try: + t = toml.load("pyproject.toml") + except Exception as e: + if pykern.pkio.exception_is_not_found(e): + pykern.pkcli.command_error( + "pyproject.toml not found; convert setup.py (if exists) or run from Python root directory" + ) + raise + prepare(t["project"]["name"]) + pykern.pksubprocess.check_call_with_signals( + ( + "sphinx-build", + "-M", + builder, + _DOCS_SUBDIR, + f"_build_{builder}", + # --fail-on-warning not yet in this version + "-W", + ), + ) diff --git a/pykern/pkcollections.py b/pykern/pkcollections.py index 0b82e753..8c72d65f 100644 --- a/pykern/pkcollections.py +++ b/pykern/pkcollections.py @@ -216,10 +216,9 @@ def pksetdefault(self, *args, **kwargs): `args` must be an even number and are interpreted in (key, value) order. `kwargs` are accepted as key=value. - If self does not have `key`, then it will be set to `value`. If `key` is already in self, its value is not changed. - If `value` is a callable, it will be called, and + If `value` is a callable, it will be called, and `key` will be set to the returned value. Args: diff --git a/pykern/pkconfig.py b/pykern/pkconfig.py index 827288e1..12686f84 100644 --- a/pykern/pkconfig.py +++ b/pykern/pkconfig.py @@ -302,7 +302,7 @@ def flatten_values(base, new): base (object): dict-like that is already flattened new (object): dict-like that will be flattened and overriden Returns: - dict: modified `base`a + dict: modified `base` """ new_values = {} _flatten_keys([], new, new_values) diff --git a/pykern/pkio.py b/pykern/pkio.py index 11e53cf3..d8720ea1 100644 --- a/pykern/pkio.py +++ b/pykern/pkio.py @@ -80,7 +80,7 @@ def compare_files(path1, path2, force=False): def exception_is_not_found(exc): """True if exception is one various file not found errors - Checks file `FileNotFoundError`, `IO + Checks `FileNotFoundError` and `IOError` with `errno.ENOENT`. Args: exc (BaseException): to check diff --git a/pykern/pksetup.py b/pykern/pksetup.py index 0bfd9b16..17bf4f8c 100644 --- a/pykern/pksetup.py +++ b/pykern/pksetup.py @@ -69,140 +69,6 @@ class is provided by this module. _cfg = None -class NullCommand(distutils.cmd.Command, object): - """Use to eliminate a ``cmdclass``. - - Does nothing but complies with :class:`distutils.cmd.Command` protocol. - """ - - user_options = [] - - def initialize_options(*args, **kwargs): - pass - - def finalize_options(*args, **kwargs): - pass - - def run(*args, **kwargs): - pass - - -class PKDeploy(NullCommand): - """Run tests, build sdist or wheel, upload. Only use this on a clean git repo. - - The command will build the distro, then run tests on it with tox, which sets - up a virtual environment. - - You must have the following environment variables: - - $PKSETUP_PYPI_USER - Name of the user to login as on pypi - - $PKSETUP_PYPI_PASSWORD - Name of the password - - This optional variable is useful for testing out your distro: - - $PKSETUP_PYPI_IS_TEST - If set, will use testpypi, otherwise uses pypi.python.org - - All values provided by environment variables. - """ - - description = "Runs git clean and tox; if successful, uploads to (test)pypi" - - def run(self): - if self.distribution.dry_run: - raise ValueError("--dry-run not supported") - self.__env = {} - # We assert these values before git clean, which would be a nasty - # surprise if executed in an ordinary development environ - is_test = self.__assert_env("PKSETUP_PYPI_IS_TEST", False) - password = self.__assert_env("PKSETUP_PYPI_PASSWORD") - user = self.__assert_env("PKSETUP_PYPI_USER") - if not self.__assert_env("PKSETUP_PKDEPLOY_IS_DEV", False): - subprocess.check_call(["git", "clean", "-dfx"]) - self.__run_cmd("tox") - sdist = glob.glob(".tox/dist/*-*.*") - self.distribution.dist_files.append(("sdist", "", sdist[0])) - if len(sdist) != 1: - raise ValueError("{}: should be exactly one sdist".format(sdist)) - repo = ( - "https://test.pypi.org/pypi/" if is_test else "https://pypi.python.org/pypi" - ) - if self.__is_unique_version(sdist[0], repo): - self.__run_twine( - sdist=sdist[0], - user=user, - password=password, - is_test=is_test, - ) - - def __assert_env(self, key, default=None): - v = os.getenv(key, default) - if v is None: - raise ValueError("${}: environment variable must be set".format(key)) - return v - - def __is_unique_version(self, fn, repo): - """If a rebuild occurs, we can't upload. PyPI doesn't allow overwrites. - - Generate https://testpypi.python.org/pypi/pksetupunit1/20170221.41054 - from sdist pksetupunit1-20170221.140313.zip, and test to see if it - exists. - """ - import requests - - m = re.search(r"([^/]+)-{}\.zip$".format(_VERSION_RE), fn) - repo += "/{}/{}".format(m.group(1), m.group(2)) - # Sometimes fails because of 404 caching - s = requests.head(repo).status_code - return s != 200 - - def __run_cmd(self, cmd_name, **kwargs): - self.announce("running {}".format(cmd_name), level=distutils.log.INFO) - klass = self.distribution.get_command_class(cmd_name) - cmd = klass(self.distribution) - cmd.initialize_options() - for k in kwargs: - assert hasattr(cmd, k), '{}: "{}" command has no such option'.format( - k, cmd_name - ) - setattr(cmd, k, kwargs[k]) - cmd.finalize_options() - cmd.run() - - def __run_twine(self, **kwargs): - kwargs["repo"] = ( - "repository = https://test.pypi.org/legacy/" if kwargs["is_test"] else "" - ) - cf = ".tox/.pypirc" - _write( - cf, - """ -[distutils] -index-servers=pypi -[pypi] -{repo} -username = {user} -password = {password} -""".format( - **kwargs - ), - ) - try: - out = _check_output( - ["twine", "upload", "--config-file", cf, kwargs["sdist"]], - stderr=subprocess.STDOUT, - ) - sys.stdout.write(out) - finally: - try: - os.remove(cf) - except Exception: - pass - - class SDist(setuptools.command.sdist.sdist, object): """Fix up a few things before running sdist""" @@ -215,101 +81,6 @@ def check_readme(self, *args, **kwargs): pass -class Test(setuptools.command.test.test, object): - """Run tests with `pykern.pkcli.test` for ``python setup.py test`` - - See also `:mod:pykern.pytest_plugin`. - """ - - def finalize_options(self): - """Initialize test_args and set test_suite to True""" - super(Test, self).finalize_options() - self.test_args = [] - self.test_suite = True - - def run_tests(self): - if os.getenv("PKSETUP_PKDEPLOY_IS_DEV", False): - distutils.log.info( - "*** PKSETUP_PKDEPLOY_IS_DEV=True: not running tests ***" - ) - sys.exit(0) - import pykern.pkcli.test - - sys.stdout.write(pykern.pkcli.test.default_command(TESTS_DIR) + "\n") - - -class Tox(setuptools.Command, object): - """Create tox.ini file""" - - description = "create tox.ini and run tox" - - user_options = [] - - def initialize_options(self, *args, **kwargs): - pass - - def finalize_options(self, *args, **kwargs): - pass - - def run(self, *args, **kwargs): - params = self._distribution_to_dict() - _sphinx_apidoc(params) - tox_ini = """# OVERWRITTEN by pykern.pksetup every "python setup.py tox" run -[tox] -envlist={pyenv} -sitepackages=True -[testenv] -passenv=PKSETUP_PKDEPLOY_IS_DEV CFLAGS CPPFLAGS LDFLAGS TRAVIS -deps={deps} -commands=python setup.py build test -[testenv:docs] -basepython=python -changedir=docs -commands=sphinx-build -b html -d {{envtmpdir}}/doctrees . {{envtmpdir}}/html -""" - try: - deps = "pykern" - d = os.path.dirname(os.path.dirname(__file__)) - if os.path.exists(os.path.join(d, "setup.py")): - # use local copy of pykern - deps = "-e" + d - if os.path.exists("requirements.txt"): - deps += " -rrequirements.txt " - _write( - TOX_INI_FILE, - tox_ini.format( - deps=deps, - pyenv=self._pyenv(params), - ), - ) - subprocess.check_call(["tox"]) - finally: - _remove(TOX_INI_FILE) - - def _distribution_to_dict(self): - d = self.distribution.metadata - res = {} - for k in d._METHOD_BASENAMES: - m = getattr(d, "get_" + k) - res[k] = m() - res["packages"] = self.distribution.packages - return res - - def _pyenv(self, params): - pyenv = [] - for c in params["classifiers"]: - m = re.search( - r"Programming Language :: Python :: (\d+).(\d+)", - c, - flags=re.IGNORECASE, - ) - if m: - pyenv.append("py{}{}".format(m.group(1), m.group(2))) - if not pyenv: - pyenv.append("py37") - return ",".join(pyenv) - - def install_requires(): """Parse requirements.txt. @@ -390,9 +161,6 @@ def _assert_package_versions(): base = _state(base, kwargs) _merge_kwargs(base, kwargs) _extras_require(base) - if os.getenv("READTHEDOCS"): - _readthedocs_fixup() - _sphinx_apidoc(base) op = setuptools.setup if base["pksetup"].get("numpy_distutils", False): import numpy.distutils.core @@ -592,18 +360,6 @@ def _readme(): raise ValueError("You need to create a README.rst") -def _readthedocs_fixup(): - """Fixups when readthedocs has conflicts""" - # https://github.com/radiasoft/sirepo/issues/1463 - subprocess.call( - [ - "pip", - "install", - "python-dateutil>=2.6.0", - ] - ) - - def _remove(path): """Remove path without throwing an exception""" try: @@ -612,32 +368,6 @@ def _remove(path): pass -def _sphinx_apidoc(base): - """Call `sphinx-apidoc` with appropriately configured ``conf.py``. - - Args: - base (dict): values to be passed to ``conf.py.in`` template - """ - # Deferred import so initial setup.py works - values = copy.deepcopy(base) - values["year"] = datetime.datetime.now().year - values["empty_braces"] = "{}" - from pykern import pkresource - - data = _read(pkresource.filename("docs-conf.py.format")) - _write("docs/conf.py", data.format(**values)) - subprocess.check_call( - [ - "sphinx-apidoc", - "-f", - "-o", - "docs", - ] - + base["packages"], - ) - return base - - def _state(base, kwargs): """Gets version and package_data. Writes MANIFEST.in. diff --git a/pykern/pkunit.py b/pykern/pkunit.py index ad3a297d..3957c6cb 100644 --- a/pykern/pkunit.py +++ b/pykern/pkunit.py @@ -157,9 +157,7 @@ def case_dirs(group_prefix="", **kwargs): Args: group_prefix (string): target subdir [''] j2_ctx (dict): passed to `pykern.pkjinja.render_file` - ignore_lines (iterable): `POSIX standard regular expressions - `_ - to be passed to `diff` + ignore_lines (iterable): `POSIX standard regular expressions `_ to be passed to `diff` is_bytes (bool): do a binary comparison [False] Returns: @@ -274,9 +272,7 @@ def file_eq(expect_path, *args, **kwargs): actual (object): string or json data structure; if missing, read `actual_path` (may be positional) actual_path (py.path or str): where to write results; if str, then joined with `work_dir`; if None, ``work_dir().join(expect_path.relto(data_dir()))`` j2_ctx (dict): passed to `pykern.pkjinja.render_file` - ignore_lines (iterable): `POSIX standard regular expressions - `_ - to be passed to `diff` + ignore_lines (iterable): `POSIX standard regular expressions `_ to be passed to `diff` is_bytes (bool): do a binary comparison [False] """ _FileEq(expect_path, *args, **kwargs) diff --git a/pyproject.toml b/pyproject.toml index 8e796493..01351496 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ dependencies = [ "setuptools>=66", "six>=1.9", "Sphinx>=1.3.5", + "toml>=0.10", "tornado", "tox>=1.9", "twine>=1.9", diff --git a/tests/pkcli/sphinx_test.py b/tests/pkcli/sphinx_test.py new file mode 100644 index 00000000..bd5cb359 --- /dev/null +++ b/tests/pkcli/sphinx_test.py @@ -0,0 +1,49 @@ +"""test pkcli.sphinx + +:copyright: Copyright (c) 2024 RadiaSoft LLC. All Rights Reserved. +:license: http://www.apache.org/licenses/LICENSE-2.0.html +""" + +_PROJECT = "xyzzy" +_VERSION = "20200202.123456" +_AUTHOR = "Arthur Coder" + + +def test_build(monkeypatch): + from pykern import pkunit, pkio + + _mock_metadata(monkeypatch) + with pkio.save_chdir(pkunit.empty_work_dir().join(_PROJECT).ensure(dir=True)): + _projex() + from pykern.pkcli import sphinx + + sphinx.build() + pkunit.pkre( + f"{_PROJECT} {_VERSION}", pkio.read_text("_build_html/html/index.html") + ) + + +def _mock_metadata(monkeypatch): + from importlib import metadata + + def _mock(*args, **kwargs): + nonlocal _prev + if not args or args[0] != _PROJECT: + return _prev(*args, **kwargs) + # Do not use PKDict because that's not what metadata returns + return { + "Author-email": f"{_AUTHOR} ", + "Name": _PROJECT, + "Version": _VERSION, + } + + _prev = metadata.metadata + monkeypatch.setattr(metadata, "metadata", _mock) + + +def _projex(): + from pykern.pkcli import projex + import subprocess + + subprocess.check_call(("git", "init", "."), stdout=subprocess.DEVNULL) + projex.init_rs_tree(f"{_PROJECT} for sphinx_test") From 427dc33913fcf7ed5f58557fbb98ceda6d885883 Mon Sep 17 00:00:00 2001 From: robnagler <5495179+robnagler@users.noreply.github.com> Date: Thu, 20 Jun 2024 00:31:40 +0000 Subject: [PATCH 2/4] fixed tests --- .../projex/projex/__init__.py.jinja | 1 - .../projex/projex/projex_console.py.jinja | 1 - tests/pkcli/projex_test.py | 6 +- tests/pksetup_test.py | 159 ------------------ 4 files changed, 3 insertions(+), 164 deletions(-) delete mode 100644 tests/pksetup_test.py diff --git a/pykern/package_data/projex/projex/__init__.py.jinja b/pykern/package_data/projex/projex/__init__.py.jinja index 2f07d248..4f99044e 100644 --- a/pykern/package_data/projex/projex/__init__.py.jinja +++ b/pykern/package_data/projex/projex/__init__.py.jinja @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """:mod:`{{ name }}` package {{ copyright_license_rst }} diff --git a/pykern/package_data/projex/projex/projex_console.py.jinja b/pykern/package_data/projex/projex/projex_console.py.jinja index 26db125a..0739540a 100644 --- a/pykern/package_data/projex/projex/projex_console.py.jinja +++ b/pykern/package_data/projex/projex/projex_console.py.jinja @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Front-end command line for :mod:`{{ name }}`. See :mod:`pykern.pkcli` for how this module is used. diff --git a/tests/pkcli/projex_test.py b/tests/pkcli/projex_test.py index f14bda08..3b022c27 100644 --- a/tests/pkcli/projex_test.py +++ b/tests/pkcli/projex_test.py @@ -28,7 +28,7 @@ def test_init_rs_tree(): ) for expect_fn, expect_re in ( ("LICENSE", "Apache License"), - ("setup.py", 'author="RadiaSoft LLC"'), + ("pyproject.toml", 'name = "rs_proj1"'), ): assert re.search( expect_re, pkio.read_text(expect_fn) @@ -64,8 +64,8 @@ def test_init_tree(): ("docs/_static/.gitignore", ""), ("docs/_templates/.gitignore", ""), ("docs/index.rst", name), - ("setup.py", 'author="zauthor"'), - ("setup.py", r":copyright:.*zauthor\."), + ("pyproject.toml", 'name = "zauthor"'), + ("proj1/proj1_console.py", r":copyright:.*zauthor\."), ("tests/.gitignore", "_work"), (name + "/__init__.py", ""), (name + "/package_data/.gitignore", ""), diff --git a/tests/pksetup_test.py b/tests/pksetup_test.py deleted file mode 100644 index 7e5339c2..00000000 --- a/tests/pksetup_test.py +++ /dev/null @@ -1,159 +0,0 @@ -# -*- coding: utf-8 -*- -"""pytest for `pykern.pksetup` - -:copyright: Copyright (c) 2015 Bivio Software, Inc. All Rights Reserved. -:license: http://www.apache.org/licenses/LICENSE-2.0.html -""" -from __future__ import absolute_import, division, print_function -import subprocess -import contextlib -import glob -import os -import os.path -import py -import pytest -import re -import sys -import tarfile -import zipfile - -_TEST_PYPI = "testpypi" - - -def test_build_clean(): - """Create a normal distribution""" - from pykern import pkio - from pykern import pksetup - from pykern import pkunit - - with _project_dir("pksetupunit1") as d: - subprocess.check_call(["python", "setup.py", "sdist", "--formats=zip"]) - archive = _assert_members( - ["pksetupunit1", "package_data", "data1"], - ["scripts", "script1"], - ["examples", "example1.txt"], - ["tests", "mod2_test.py"], - ) - subprocess.check_call(["python", "setup.py", "build"]) - dat = os.path.join("build", "lib", "pksetupunit1", "package_data", "data1") - assert os.path.exists( - dat - ), "When building, package_data should be installed in lib" - bin_dir = "scripts-{}.{}".format(*(sys.version_info[0:2])) - subprocess.check_call(["python", "setup.py", "test"]) - assert os.path.exists("tests/mod2_test.py") - subprocess.check_call(["git", "clean", "-dfx"]) - assert not os.path.exists( - "build" - ), "When git clean runs, build directory should not exist" - subprocess.check_call(["python", "setup.py", "sdist"]) - pkio.unchecked_remove(archive) - _assert_members( - ["!", "tests", "mod2_work", "do_not_include_in_sdist.py"], - ["tests", "mod2_test.py"], - ) - # TODO(robnagler) need another sentinel here - if os.environ.get("PKSETUP_PKDEPLOY_IS_DEV", False): - subprocess.check_call(["python", "setup.py", "pkdeploy"]) - - -def test_optional_args(): - """Create a normal distribution - - Installs into the global environment, which messes up pykern's install. - Due to incorrect editing of easy-install.pth. - """ - from pykern import pkio - from pykern import pksetup - from pykern import pkunit - from pykern import pkdebug - - def _results(pwd, expect): - x = d.dirpath().listdir(fil=lambda x: x.basename != d.basename) - pkunit.pkeq(1, len(x)) - a = list(pkio.walk_tree(x[0], "/(civicjson.py|adhan.py|pksetup.*METADATA)$")) - pkunit.pkeq(expect, len(a)) - for f in a: - if f.basename == "METADATA": - return x[0], f - pkunit.pkfail("METADATA not found in files={}", a) - - with _project_dir("pksetupunit2") as d: - # clean the work dir then run afind - subprocess.check_call( - [ - "pip", - "install", - "--root", - pkunit.work_dir(), - # No -e or it will modify global environment - ".[all]", - ], - ) - x, m1 = _results(d, 3) - pkio.unchecked_remove(x, ".git") - e = os.environ.copy() - e["PYKERN_PKSETUP_NO_VERSION"] = "1" - subprocess.check_call( - [ - "pip", - "install", - "--root", - pkunit.work_dir(), - ".", - ], - env=e, - ) - _, m2 = _results(d, 1) - pkunit.pkne(m1, m2) - - -@contextlib.contextmanager -def _project_dir(project): - """Copy "data_dir/project" to "work_dir/project" - - Initializes as a git repo. - - Args: - project (str): subdirectory name - - Returns: - py.path.local: working directory""" - from pykern import pkio - from pykern import pksetup - from pykern import pkunit - - d = pkunit.empty_work_dir().join(project) - pkunit.data_dir().join(d.basename).copy(d) - with pkio.save_chdir(d): - subprocess.check_call(["git", "init", "."]) - subprocess.check_call(["git", "config", "user.email", "pip@pykern.org"]) - subprocess.check_call(["git", "config", "user.name", "pykern"]) - subprocess.check_call(["git", "add", "."]) - # Need a commit - subprocess.check_call(["git", "commit", "-m", "n/a"]) - yield d - - -def _assert_members(*expect): - arc = glob.glob(os.path.join("dist", "pksetupunit1*")) - assert 1 == len(arc), "Verify setup.py sdist creates an archive file" - arc = arc[0] - m = re.search(r"(.+)\.(zip|tar.gz)$", os.path.basename(arc)) - base = m.group(1) - if m.group(2) == "zip": - with zipfile.ZipFile(arc) as z: - members = z.namelist() - else: - with tarfile.open(arc) as t: - members = t.getnames() - for member in expect: - exists = member[0] != "!" - if not exists: - member = member[1:] - m = os.path.join(base, *member) - assert bool(m in members) == exists, "When sdist, {} is {} from archive".format( - m, - "included" if exists else "excluded", - ) - return arc From 11e648afc487cf328ed8504d6291cef4ceff949b Mon Sep 17 00:00:00 2001 From: robnagler <5495179+robnagler@users.noreply.github.com> Date: Fri, 28 Jun 2024 15:15:45 +0000 Subject: [PATCH 3/4] remove tox and twine --- pykern/pksetup.py | 15 +--- pykern/xlsx.py | 214 ++++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 - 3 files changed, 216 insertions(+), 15 deletions(-) diff --git a/pykern/pksetup.py b/pykern/pksetup.py index 17bf4f8c..cf95e467 100644 --- a/pykern/pksetup.py +++ b/pykern/pksetup.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Wrapper for setuptools.setup to simplify creating of `setup.py` files. Python `setup.py` files should be short for well-structured projects. @@ -20,7 +19,7 @@ class is provided by this module. Assumptions: - - the use of ``pytest`` for tests. GUI and console scripts are + - GUI and console scripts are found automatically by special suffixes ``_gui.py`` and ``_console.py``. See ``setup`` documentation for an example. @@ -31,6 +30,7 @@ class is provided by this module. :copyright: Copyright (c) 2015 Radiasoft LLC. All Rights Reserved. :license: http://www.apache.org/licenses/LICENSE-2.0.html """ + # DO NOT import __future__. setuptools breaks with unicode in PY2: # http://bugs.python.org/setuptools/issue152 # Get errors about package_data not containing wildcards, name not found, etc. @@ -55,15 +55,9 @@ class is provided by this module. #: The subdirectory in the top-level Python where to put resources PACKAGE_DATA = "package_data" -#: Created only during Tox run -TOX_INI_FILE = "tox.ini" - #: Where scripts live, you probably don't want this SCRIPTS_DIR = "scripts" -#: Where the tests live -TESTS_DIR = "tests" - _VERSION_RE = r"(\d{8}\.\d+)" _cfg = None @@ -145,18 +139,13 @@ def _assert_package_versions(): base = { "classifiers": [], "cmdclass": { - "pkdeploy": PKDeploy, "sdist": SDist, - "test": Test, - "tox": Tox, }, "entry_points": _entry_points(name), # These both need to be set "name": name, "packages": _packages(name), "pksetup": flags, - "tests_require": ["pytest"], - "test_suite": TESTS_DIR, } base = _state(base, kwargs) _merge_kwargs(base, kwargs) diff --git a/pykern/xlsx.py b/pykern/xlsx.py index d6b8a14a..fe42ac39 100644 --- a/pykern/xlsx.py +++ b/pykern/xlsx.py @@ -47,6 +47,8 @@ def _error(self, fmt, *args, **kwargs): # TODO: print stack raise AssertionError(pkdformat(fmt + "; {self}", *args, self=self, **kwargs)) +_OPS = set(_OP_UNARY.keys()).union(_OP_BINARY.keys()).union(_OP_MULTI.keys()) + class _Base(_SimpleBase): def __init__(self, cfg): @@ -225,10 +227,54 @@ def _compile_pass2(self): "round_digits", _DEFAULT_ROUND_DIGITS ), ) +<<<<<<< Updated upstream if (v := self.defaults.get("fmt")) is not None: self.pksetdefault(fmt=v) self.is_decimal = self.expr.is_decimal() self.is_compiled = True +======= + self.value = _rnd(value, self.round_digits) + self.content = str(self.value) + self.is_decimal = True + + def _compile_expr(self, expr): + if isinstance(expr, (float, int, decimal.Decimal)): + v = decimal.Decimal(expr) + return _Operand( + content=str(v), + count=1, + fmt=None, + round_digits=None, + value=v, + is_decimal=True, + ) + if not isinstance(expr, (tuple, list)): + self._error("invalid type={} expr={}", type(expr), expr) + if len(expr) == 0: + self._error("empty expr") + # TODO(robnagler) document that links must begin with alnum + if expr[0][0].isalnum() and len(expr) == 1: + return self._compile_ref(expr[0], expect_count=None) + if expr[0] in _OPS: + return self._compile_op(expr) + self._error("invalid op or link expr={}", expr) + + def _compile_expr_root(self): + self._is_expr = True + r = self._compile_expr(self.content) + # expression's value overrides defaults + for x in "fmt", "round_digits": + if r.get(x) is not None: + self.pksetdefault(x, r[x]) + if isinstance(r.value, decimal.Decimal): + self._compile_decimal(r.value) + self.content = f"=ROUND({r.content}, {self.round_digits})" + elif isinstance(r.value, str): + self._compile_str(r.value) + self.content = f"={r.content}" + else: + raise AssertionError(f"_compile_expr invalid r={r}") +>>>>>>> Stashed changes def _compile_link1(self): if "link" in self: @@ -237,6 +283,174 @@ def _compile_link1(self): self._error("link={} must begin with alphanumeric", l) self.workbook.links.setdefault(l, []).append(self) +<<<<<<< Updated upstream +======= + def _compile_op(self, expr): + o = expr[0] + e = expr[1:] + if o == "+": + return self._compile_operands(o, e, expect_count=None) + if o == "-": + if len(expr) > 2: + return self._compile_operands(o, e, expect_count=2) + return self._compile_operands(o, e, expect_count=1) + if o == "*": + return self._compile_operands(o, e, expect_count=None) + if o == "/": + return self._compile_operands(o, e, expect_count=2) + self._error("operator={} not supported", o) + + def _compile_op_binary(self, op, operands): + if len(operands) == 1: + self._error("op={} requires two distinct operands={}", op, operands) + l = operands[0] + r = operands[1] + o = _Operand( + content=f"({l.content}{op}{r.content})", + count=1, + value=_OP_BINARY[op](l.value, r.value), + is_decimal=True, + ) + # TODO: if the fmt is not the same, that may be ok, because no fmt (decimal) + # for divide, do you have a format, e.g. $/$ has no format by default. + self._compile_op_options(o, operands) + return o + + def _compile_op_multi(self, op, operands): + r = _Operand(count=1, is_decimal=True) + c = "" + m = _OP_MULTI[op] + v = decimal.Decimal(m.init) + for o in operands: + if len(c): + c += "," + c += o.content + if "cells" in o: + for p in o.cells: + v = m.func(v, p.value) + else: + v = m.func(v, o.value) + self._compile_op_options(r, operands) + return r.pkupdate( + content=f"{m.xl}({c})", + value=v, + ) + + def _compile_op_options(self, res, operands): + for o in operands: + for x in "fmt", "round_digits": + if o.get(x) is None: + continue + if x not in res: + res[x] = o[x] + # TODO doc that grabs the leftmost fmt or round + elif res[x] is not None and res[x] != o[x]: + res[x] = None + return res + + def _compile_op_unary(self, op, operands): + o = operands[0] + return _Operand( + content=f"({op}{o.content})", + count=1, + fmt=o.get("fmt"), + is_decimal=True, + round_digits=o.round_digits, + value=_OP_UNARY[op](o.value), + ) + + def _compile_operands(self, op, operands, expect_count): + n = 0 + z = [] + for o in operands: + e = self._compile_expr(o) + if not e.is_decimal: + self._error("operand={} must numeric for op={}", e.value, op) + z.append(e) + n += e.count + if expect_count is None: + return self._compile_op_multi(op, z) + if expect_count != n: + x = ( + "; You might need to be more specific link names to avoid automatic link operand grouping." + if max(expect_count, 2) < n + else "" + ) + self._error( + "operator={} expect_count={} operands, not actual={} operands={}{x}", + op, + expect_count, + n, + operands, + ) + if expect_count == 1: + return self._compile_op_unary(op, z) + if expect_count == 2: + return self._compile_op_binary(op, z) + raise AssertionError(f"incorrect expect_count={expect_count}") + + def _compile_ref(self, link, expect_count): + def _xl_id(other): + r = "" + if other.sheet != self.sheet: + r = f"'{other.sheet.title}'!" + return r + other.xl_id + + l = self.workbook.links + if link not in l: + self._error("link={} not found", link) + p = None + n = 0 + z = [] + for c in sorted(l[link], key=lambda x: x.sort_index): + c._compile_pass2() + n += 1 + # _ROW_MODULUS ensures a gap so columns are separated by more than "+1" + # and this will never link the wrong column + if ( + p is not None + and p.sheet == c.sheet + and p.sort_index + 1 == c.sort_index + ): + p = z[-1][1] = c + continue + z.append([c, None]) + p = c + r = "" + for x in z: + if len(r): + r += "," + r += _xl_id(x[0]) + if x[1] is not None: + r += ":" + _xl_id(x[1]) + if expect_count is not None and expect_count != n: + self._error( + "incorrect operands={} expect={} count={}", + l[link], + expect_count, + n, + ) + # _assert_link_pair validates that the cells are the same type + return _Operand( + cells=l[link], + content=r, + count=n, + fmt=p.get("fmt"), + # TODO(robnagler) + # default round_digits: if p has round_digits explicit, does not + # matter, because target round digits overrides, always. format + # is different, though. + round_digits=p.round_digits, + value=p.value if n == 1 else None, + is_decimal=p.is_decimal, + ) + + def _compile_str(self, value): + self.round_digits = None + self.value = self.content = value + self.is_decimal = False + +>>>>>>> Stashed changes def _save(self): f = _Fmt(self) self.sheet.width(_XL_COLS[self.col_num], f.width(self)) diff --git a/pyproject.toml b/pyproject.toml index 01351496..b799ce48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,8 +42,6 @@ dependencies = [ "Sphinx>=1.3.5", "toml>=0.10", "tornado", - "tox>=1.9", - "twine>=1.9", "urllib3", "XlsxWriter>=3.0.3", ] From dda0721324f948c5bb0e0c5328ebd41ea118ba93 Mon Sep 17 00:00:00 2001 From: robnagler <5495179+robnagler@users.noreply.github.com> Date: Fri, 28 Jun 2024 16:12:04 +0000 Subject: [PATCH 4/4] undo accidental stash pop --- pykern/xlsx.py | 214 ------------------------------------------------- 1 file changed, 214 deletions(-) diff --git a/pykern/xlsx.py b/pykern/xlsx.py index fe42ac39..d6b8a14a 100644 --- a/pykern/xlsx.py +++ b/pykern/xlsx.py @@ -47,8 +47,6 @@ def _error(self, fmt, *args, **kwargs): # TODO: print stack raise AssertionError(pkdformat(fmt + "; {self}", *args, self=self, **kwargs)) -_OPS = set(_OP_UNARY.keys()).union(_OP_BINARY.keys()).union(_OP_MULTI.keys()) - class _Base(_SimpleBase): def __init__(self, cfg): @@ -227,54 +225,10 @@ def _compile_pass2(self): "round_digits", _DEFAULT_ROUND_DIGITS ), ) -<<<<<<< Updated upstream if (v := self.defaults.get("fmt")) is not None: self.pksetdefault(fmt=v) self.is_decimal = self.expr.is_decimal() self.is_compiled = True -======= - self.value = _rnd(value, self.round_digits) - self.content = str(self.value) - self.is_decimal = True - - def _compile_expr(self, expr): - if isinstance(expr, (float, int, decimal.Decimal)): - v = decimal.Decimal(expr) - return _Operand( - content=str(v), - count=1, - fmt=None, - round_digits=None, - value=v, - is_decimal=True, - ) - if not isinstance(expr, (tuple, list)): - self._error("invalid type={} expr={}", type(expr), expr) - if len(expr) == 0: - self._error("empty expr") - # TODO(robnagler) document that links must begin with alnum - if expr[0][0].isalnum() and len(expr) == 1: - return self._compile_ref(expr[0], expect_count=None) - if expr[0] in _OPS: - return self._compile_op(expr) - self._error("invalid op or link expr={}", expr) - - def _compile_expr_root(self): - self._is_expr = True - r = self._compile_expr(self.content) - # expression's value overrides defaults - for x in "fmt", "round_digits": - if r.get(x) is not None: - self.pksetdefault(x, r[x]) - if isinstance(r.value, decimal.Decimal): - self._compile_decimal(r.value) - self.content = f"=ROUND({r.content}, {self.round_digits})" - elif isinstance(r.value, str): - self._compile_str(r.value) - self.content = f"={r.content}" - else: - raise AssertionError(f"_compile_expr invalid r={r}") ->>>>>>> Stashed changes def _compile_link1(self): if "link" in self: @@ -283,174 +237,6 @@ def _compile_link1(self): self._error("link={} must begin with alphanumeric", l) self.workbook.links.setdefault(l, []).append(self) -<<<<<<< Updated upstream -======= - def _compile_op(self, expr): - o = expr[0] - e = expr[1:] - if o == "+": - return self._compile_operands(o, e, expect_count=None) - if o == "-": - if len(expr) > 2: - return self._compile_operands(o, e, expect_count=2) - return self._compile_operands(o, e, expect_count=1) - if o == "*": - return self._compile_operands(o, e, expect_count=None) - if o == "/": - return self._compile_operands(o, e, expect_count=2) - self._error("operator={} not supported", o) - - def _compile_op_binary(self, op, operands): - if len(operands) == 1: - self._error("op={} requires two distinct operands={}", op, operands) - l = operands[0] - r = operands[1] - o = _Operand( - content=f"({l.content}{op}{r.content})", - count=1, - value=_OP_BINARY[op](l.value, r.value), - is_decimal=True, - ) - # TODO: if the fmt is not the same, that may be ok, because no fmt (decimal) - # for divide, do you have a format, e.g. $/$ has no format by default. - self._compile_op_options(o, operands) - return o - - def _compile_op_multi(self, op, operands): - r = _Operand(count=1, is_decimal=True) - c = "" - m = _OP_MULTI[op] - v = decimal.Decimal(m.init) - for o in operands: - if len(c): - c += "," - c += o.content - if "cells" in o: - for p in o.cells: - v = m.func(v, p.value) - else: - v = m.func(v, o.value) - self._compile_op_options(r, operands) - return r.pkupdate( - content=f"{m.xl}({c})", - value=v, - ) - - def _compile_op_options(self, res, operands): - for o in operands: - for x in "fmt", "round_digits": - if o.get(x) is None: - continue - if x not in res: - res[x] = o[x] - # TODO doc that grabs the leftmost fmt or round - elif res[x] is not None and res[x] != o[x]: - res[x] = None - return res - - def _compile_op_unary(self, op, operands): - o = operands[0] - return _Operand( - content=f"({op}{o.content})", - count=1, - fmt=o.get("fmt"), - is_decimal=True, - round_digits=o.round_digits, - value=_OP_UNARY[op](o.value), - ) - - def _compile_operands(self, op, operands, expect_count): - n = 0 - z = [] - for o in operands: - e = self._compile_expr(o) - if not e.is_decimal: - self._error("operand={} must numeric for op={}", e.value, op) - z.append(e) - n += e.count - if expect_count is None: - return self._compile_op_multi(op, z) - if expect_count != n: - x = ( - "; You might need to be more specific link names to avoid automatic link operand grouping." - if max(expect_count, 2) < n - else "" - ) - self._error( - "operator={} expect_count={} operands, not actual={} operands={}{x}", - op, - expect_count, - n, - operands, - ) - if expect_count == 1: - return self._compile_op_unary(op, z) - if expect_count == 2: - return self._compile_op_binary(op, z) - raise AssertionError(f"incorrect expect_count={expect_count}") - - def _compile_ref(self, link, expect_count): - def _xl_id(other): - r = "" - if other.sheet != self.sheet: - r = f"'{other.sheet.title}'!" - return r + other.xl_id - - l = self.workbook.links - if link not in l: - self._error("link={} not found", link) - p = None - n = 0 - z = [] - for c in sorted(l[link], key=lambda x: x.sort_index): - c._compile_pass2() - n += 1 - # _ROW_MODULUS ensures a gap so columns are separated by more than "+1" - # and this will never link the wrong column - if ( - p is not None - and p.sheet == c.sheet - and p.sort_index + 1 == c.sort_index - ): - p = z[-1][1] = c - continue - z.append([c, None]) - p = c - r = "" - for x in z: - if len(r): - r += "," - r += _xl_id(x[0]) - if x[1] is not None: - r += ":" + _xl_id(x[1]) - if expect_count is not None and expect_count != n: - self._error( - "incorrect operands={} expect={} count={}", - l[link], - expect_count, - n, - ) - # _assert_link_pair validates that the cells are the same type - return _Operand( - cells=l[link], - content=r, - count=n, - fmt=p.get("fmt"), - # TODO(robnagler) - # default round_digits: if p has round_digits explicit, does not - # matter, because target round digits overrides, always. format - # is different, though. - round_digits=p.round_digits, - value=p.value if n == 1 else None, - is_decimal=p.is_decimal, - ) - - def _compile_str(self, value): - self.round_digits = None - self.value = self.content = value - self.is_decimal = False - ->>>>>>> Stashed changes def _save(self): f = _Fmt(self) self.sheet.width(_XL_COLS[self.col_num], f.width(self))