Auto-generate fully specified requirements (#1868)

* Auto-generate fully specified requirements

* Generate hashes

* license year

.gitignore

@@ -3,6 +3,9 @@
 **/dependency-reduced-pom.xml
 **/target/**/*
 
+# Python requirements build files
+**/__build__/**/*
+
 # IntelliJ files
 **/.idea/**/*
 **/*.iml

python/generate_all_dependencies.sh

@@ -0,0 +1,19 @@
+# Copyright 2024 Google Inc. All Rights Reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SCRIPTPATH=$(dirname "$0")
+
+sh $SCRIPTPATH/generate_dependencies.sh $SCRIPTPATH/../v1/src/main/python/base_requirements.txt $SCRIPTPATH/../v1/src/main/python/requirements.txt
+sh $SCRIPTPATH/generate_dependencies.sh $SCRIPTPATH/../python/src/main/python/word-count-python/base_requirements.txt $SCRIPTPATH/../python/src/main/python/word-count-python/requirements.txt
+sh $SCRIPTPATH/generate_dependencies.sh $SCRIPTPATH/../python/src/main/python/streaming-llm/base_requirements.txt $SCRIPTPATH/../python/src/main/python/streaming-llm/requirements.txt

python/generate_dependencies.sh

@@ -0,0 +1,75 @@
+# Copyright 2024 Google Inc. All Rights Reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+BASE_REQUIREMENTS_PATH=$1
+TARGET_REQUIREMENTS_PATH=$2
+
+if ! python3 --version > /dev/null 2>&1 ; then
+  echo "Please install a python3 interpreter. See s.apache.org/beam-python-dev-wiki for Python installation tips."
+  exit 1
+fi
+
+if ! python3 -m venv --help > /dev/null 2>&1 ; then
+  echo "Your python3 installation does not have a required venv module. See s.apache.org/beam-python-dev-wiki for Python installation tips."
+  exit 1
+fi
+
+set -ex
+
+ENV_PATH="$PWD/__build__/python${PY_VERSION/./}_requirements_gen"
+rm -rf "$ENV_PATH" 2>/dev/null || true
+python3 -m venv "$ENV_PATH"
+source "$ENV_PATH"/bin/activate
+pip install --upgrade pip setuptools wheel
+pip install pip-tools
+
+# Install requirements from base file
+pip install ${PIP_EXTRA_OPTIONS:+"$PIP_EXTRA_OPTIONS"}  --no-cache-dir -r $BASE_REQUIREMENTS_PATH
+
+echo "Checking for broken dependencies:"
+pip check
+echo "Installed dependencies:"
+pip freeze
+
+# Generate hashes for new requirements file
+echo "Running pip-compile to generate hashes"
+pip-compile $BASE_REQUIREMENTS_PATH -o $TARGET_REQUIREMENTS_PATH --generate-hashes
+
+PY_IMAGE="py${PY_VERSION//.}"
+cat <<EOT > "$TARGET_REQUIREMENTS_PATH"
+# Copyright 2024 Google Inc. All Rights Reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Autogenerated requirements file for Apache Beam container image.
+# From the templates base directory to update,
+# run: sh python/generate_dependencies.sh $BASE_REQUIREMENTS_PATH $TARGET_REQUIREMENTS_PATH
+# or: sh python/generate_all_dependencies.sh
+# Do not edit manually, adjust the base requirements file, and regenerate the list.
+
+$(cat $TARGET_REQUIREMENTS_PATH)
+
+EOT
+
+rm -rf "$ENV_PATH"

python/src/main/python/streaming-llm/base_requirements.txt

@@ -0,0 +1,4 @@
+apache-beam[gcp]
+torch
+transformers
+torchvision

python/src/main/python/word-count-python/base_requirements.txt

@@ -0,0 +1 @@
+apache-beam[gcp]

v1/src/main/python/base_requirements.txt

@@ -0,0 +1,16 @@
+# Copyright 2020 Google Inc. All Rights Reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+google-cloud
+google-cloud-pubsub