Skip to content

Commit 3ab4528

Browse files
committed
Document and streamline procedure to generate process listings from openeo-processes repo
related to #390
1 parent 858c06d commit 3ab4528

File tree

6 files changed

+133
-8
lines changed

6 files changed

+133
-8
lines changed

docs/development.rst

+21
Original file line numberDiff line numberDiff line change
@@ -366,3 +366,24 @@ For development, what you need to do is:
366366
367367
# Now install the openeo code in editable mode.
368368
pip install -e .[dev]
369+
370+
371+
372+
Update of generated files
373+
==========================
374+
375+
Some parts of the openEO Python Client Library source code are
376+
generated/compiled from upstream sources (e.g. official openEO specifications).
377+
Because updates are not often required,
378+
it's just a semi-manual procedure (to run from the project root):
379+
380+
.. code-block:: console
381+
382+
# Update the sub-repositories (like git submodules, but optional)
383+
python specs/update-subrepos.py
384+
385+
# Update `openeo/processes.py` from specifications in openeo-processes repository
386+
python openeo/internal/processes/generator.py specs/openeo-processes specs/openeo-processes/proposals --output openeo/processes.py
387+
388+
# Update the openEO process mapping documentation page
389+
python docs/process_mapping.py > docs/process_mapping.rst

docs/process_mapping.rst

+4-4
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
..
33
!Warning! This is an auto-generated file.
44
Do not edit directly.
5-
Generated from: ['process_mapping.py']
5+
Generated from: ['docs/process_mapping.py']
66
77
.. _openeo_process_mapping:
88

@@ -63,11 +63,11 @@ method or function in the openEO Python Client Library.
6363
* - `filter_temporal <https://processes.openeo.org/#filter_temporal>`_
6464
- :py:meth:`DataCube.filter_temporal() <openeo.rest.datacube.DataCube.filter_temporal>`
6565
* - `fit_class_random_forest <https://processes.openeo.org/#fit_class_random_forest>`_
66-
- :py:meth:`DataCube.fit_class_random_forest() <openeo.rest.datacube.DataCube.fit_class_random_forest>`
66+
- :py:meth:`VectorCube.fit_class_random_forest() <openeo.rest.vectorcube.VectorCube.fit_class_random_forest>`
6767
* - `fit_curve <https://processes.openeo.org/#fit_curve>`_
6868
- :py:meth:`DataCube.fit_curve() <openeo.rest.datacube.DataCube.fit_curve>`
6969
* - `fit_regr_random_forest <https://processes.openeo.org/#fit_regr_random_forest>`_
70-
- :py:meth:`DataCube.fit_regr_random_forest() <openeo.rest.datacube.DataCube.fit_regr_random_forest>`
70+
- :py:meth:`VectorCube.fit_regr_random_forest() <openeo.rest.vectorcube.VectorCube.fit_regr_random_forest>`
7171
* - `flatten_dimensions <https://processes.openeo.org/#flatten_dimensions>`_
7272
- :py:meth:`DataCube.flatten_dimensions() <openeo.rest.datacube.DataCube.flatten_dimensions>`
7373
* - `ge <https://processes.openeo.org/#ge>`_
@@ -143,4 +143,4 @@ method or function in the openEO Python Client Library.
143143
* - `unflatten_dimension <https://processes.openeo.org/#unflatten_dimension>`_
144144
- :py:meth:`DataCube.unflatten_dimension() <openeo.rest.datacube.DataCube.unflatten_dimension>`
145145

146-
:subscript:`(Table autogenerated on 2022-11-14)`
146+
:subscript:`(Table autogenerated on 2023-03-15)`

openeo/internal/processes/generator.py

+8-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import argparse
2+
import datetime
23
import keyword
34
import sys
45
import textwrap
@@ -207,13 +208,17 @@ def __gt__(self, other) -> 'ProcessBuilder':
207208
# Used command line arguments:
208209
# {cli}
209210
""".format(cli=" ".join(argv))))
211+
output.write(f"# Generated on {datetime.date.today().isoformat()}\n")
212+
210213
output.write(oo_src)
211-
output.write(fun_src)
214+
output.write(fun_src.rstrip() + "\n")
212215

213216

214217
def main():
215-
# Usage example (from project root, assuming the `openeo-process` repo is checked out as well):
216-
# python openeo/internal/processes/generator.py ../openeo-processes --output openeo/processes.py
218+
# Usage example (from project root):
219+
# # Update subrepos (with process specs)
220+
# python specs/update-subrepos.py
221+
# python openeo/internal/processes/generator.py specs/openeo-processes specs/openeo-processes/proposals --output openeo/processes.py
217222

218223
argv = sys.argv
219224
arg_parser = argparse.ArgumentParser()

openeo/processes.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22
# Do not edit this file directly.
33
# It is automatically generated.
44
# Used command line arguments:
5-
# openeo/internal/processes/generator.py ../openeo-processes/ ../openeo-processes/proposals/ --output openeo/processes.py
5+
# openeo/internal/processes/generator.py specs/openeo-processes specs/openeo-processes/proposals --output openeo/processes.py
6+
# Generated on 2023-03-15
67

78
import builtins
89
from openeo.internal.processes.builder import ProcessBuilderBase, UNSET

specs/.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
openeo-processes/

specs/update-subrepos.py

+97
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#!/usr/bin/env python
2+
"""
3+
Script to manage/update specification sub-repositories,
4+
in the style of git submodules, but without requiring
5+
them for all development situations
6+
because they are only necessary for a small subset of
7+
maintenance tasks.
8+
9+
This script should be runnable without non-stdlib dependencies.
10+
"""
11+
12+
# TODO: windows support?
13+
14+
import collections
15+
import contextlib
16+
import logging
17+
import logging.config
18+
import os
19+
import subprocess
20+
import sys
21+
from pathlib import Path
22+
from typing import List
23+
24+
25+
_log = logging.getLogger(__name__)
26+
_ROOT_DIR = Path(__file__).parent
27+
28+
SubRepo = collections.namedtuple("SubRepo", ["url", "rev", "path"])
29+
30+
31+
def main():
32+
logging.basicConfig(
33+
level=logging.INFO, format=">>> {message}", style="{", stream=sys.stdout
34+
)
35+
subrepos = [
36+
SubRepo(
37+
url="https://github.com/Open-EO/openeo-processes.git",
38+
rev="ca9e31094b863233d88459b6cf2a37416bc90d4e",
39+
path="openeo-processes",
40+
)
41+
]
42+
43+
for subrepo in subrepos:
44+
ensure_subrepo(subrepo)
45+
46+
run_command(["ls", "-al", _ROOT_DIR])
47+
48+
49+
def run_command(cmd: List[str], *, return_stdout: bool = False, **kwargs):
50+
_log.info(f"Running {cmd} ({kwargs=})")
51+
if not return_stdout:
52+
return subprocess.check_call(args=cmd, **kwargs)
53+
else:
54+
return subprocess.check_output(args=cmd, **kwargs)
55+
56+
57+
@contextlib.contextmanager
58+
def in_cwd(path: Path):
59+
orig_cwd = Path.cwd()
60+
try:
61+
_log.info(f"Changing working directory to {path}")
62+
os.chdir(path)
63+
yield
64+
finally:
65+
_log.info(f"Resetting working directory to {path}")
66+
os.chdir(orig_cwd)
67+
68+
69+
def ensure_subrepo(subrepo: SubRepo):
70+
path = _ROOT_DIR / subrepo.path
71+
_log.info(f"Ensuring subrepo {subrepo}")
72+
if not path.exists():
73+
run_command(["git", "clone", subrepo.url, str(path)])
74+
run_command(["git", "config", "advice.detachedHead", "false"], cwd=path)
75+
76+
if path.is_dir() and (path / ".git").is_dir():
77+
_log.info(f"{path} already looks like a git repo")
78+
else:
79+
raise RuntimeError(f"{path} exists but does not look like a git repo")
80+
81+
# Checkout to desired revision
82+
run_command(["git", "checkout", subrepo.rev], cwd=path)
83+
run_command(["git", "log", "-1"], cwd=path)
84+
run_command(["git", "submodule", "update", "--init", "--recursive"], cwd=path)
85+
86+
# Check that repo is not dirty
87+
dirty = run_command(
88+
["git", "status", "--short", "--untracked-files=all", "--ignored"],
89+
cwd=path,
90+
return_stdout=True,
91+
)
92+
if dirty.strip():
93+
raise RuntimeError(f"Repo {path} seems dirty: {dirty}")
94+
95+
96+
if __name__ == "__main__":
97+
main()

0 commit comments

Comments
 (0)