From 2214ef6cec5958f8de1f63de09811df05f4b380d Mon Sep 17 00:00:00 2001 From: Erick Friis Date: Wed, 25 Sep 2024 18:06:55 -0700 Subject: [PATCH] Initial commit --- .github/actions/poetry_setup/action.yml | 93 ++++++ .github/scripts/check_diff.py | 48 ++++ .github/scripts/get_min_versions.py | 65 +++++ .github/workflows/_codespell.yml | 39 +++ .../workflows/_compile_integration_test.yml | 57 ++++ .github/workflows/_lint.yml | 102 +++++++ .github/workflows/_release.yml | 265 ++++++++++++++++++ .github/workflows/_test.yml | 58 ++++ .github/workflows/_test_release.yml | 95 +++++++ .github/workflows/check_diffs.yml | 89 ++++++ .../workflows/extract_ignored_words_list.py | 10 + LICENSE | 21 ++ README.md | 59 ++++ 13 files changed, 1001 insertions(+) create mode 100644 .github/actions/poetry_setup/action.yml create mode 100644 .github/scripts/check_diff.py create mode 100644 .github/scripts/get_min_versions.py create mode 100644 .github/workflows/_codespell.yml create mode 100644 .github/workflows/_compile_integration_test.yml create mode 100644 .github/workflows/_lint.yml create mode 100644 .github/workflows/_release.yml create mode 100644 .github/workflows/_test.yml create mode 100644 .github/workflows/_test_release.yml create mode 100644 .github/workflows/check_diffs.yml create mode 100644 .github/workflows/extract_ignored_words_list.py create mode 100644 LICENSE create mode 100644 README.md diff --git a/.github/actions/poetry_setup/action.yml b/.github/actions/poetry_setup/action.yml new file mode 100644 index 0000000..68b099f --- /dev/null +++ b/.github/actions/poetry_setup/action.yml @@ -0,0 +1,93 @@ +# An action for setting up poetry install with caching. +# Using a custom action since the default action does not +# take poetry install groups into account. +# Action code from: +# https://github.com/actions/setup-python/issues/505#issuecomment-1273013236 +name: poetry-install-with-caching +description: Poetry install with support for caching of dependency groups. + +inputs: + python-version: + description: Python version, supporting MAJOR.MINOR only + required: true + + poetry-version: + description: Poetry version + required: true + + cache-key: + description: Cache key to use for manual handling of caching + required: true + + working-directory: + description: Directory whose poetry.lock file should be cached + required: true + +runs: + using: composite + steps: + - uses: actions/setup-python@v5 + name: Setup python ${{ inputs.python-version }} + id: setup-python + with: + python-version: ${{ inputs.python-version }} + + - uses: actions/cache@v4 + id: cache-bin-poetry + name: Cache Poetry binary - Python ${{ inputs.python-version }} + env: + SEGMENT_DOWNLOAD_TIMEOUT_MIN: "1" + with: + path: | + /opt/pipx/venvs/poetry + # This step caches the poetry installation, so make sure it's keyed on the poetry version as well. + key: bin-poetry-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-${{ inputs.poetry-version }} + + - name: Refresh shell hashtable and fixup softlinks + if: steps.cache-bin-poetry.outputs.cache-hit == 'true' + shell: bash + env: + POETRY_VERSION: ${{ inputs.poetry-version }} + PYTHON_VERSION: ${{ inputs.python-version }} + run: | + set -eux + + # Refresh the shell hashtable, to ensure correct `which` output. + hash -r + + # `actions/cache@v3` doesn't always seem able to correctly unpack softlinks. + # Delete and recreate the softlinks pipx expects to have. + rm /opt/pipx/venvs/poetry/bin/python + cd /opt/pipx/venvs/poetry/bin + ln -s "$(which "python$PYTHON_VERSION")" python + chmod +x python + cd /opt/pipx_bin/ + ln -s /opt/pipx/venvs/poetry/bin/poetry poetry + chmod +x poetry + + # Ensure everything got set up correctly. + /opt/pipx/venvs/poetry/bin/python --version + /opt/pipx_bin/poetry --version + + - name: Install poetry + if: steps.cache-bin-poetry.outputs.cache-hit != 'true' + shell: bash + env: + POETRY_VERSION: ${{ inputs.poetry-version }} + PYTHON_VERSION: ${{ inputs.python-version }} + # Install poetry using the python version installed by setup-python step. + run: pipx install "poetry==$POETRY_VERSION" --python '${{ steps.setup-python.outputs.python-path }}' --verbose + + - name: Restore pip and poetry cached dependencies + uses: actions/cache@v4 + env: + SEGMENT_DOWNLOAD_TIMEOUT_MIN: "4" + WORKDIR: ${{ inputs.working-directory == '' && '.' || inputs.working-directory }} + with: + path: | + ~/.cache/pip + ~/.cache/pypoetry/virtualenvs + ~/.cache/pypoetry/cache + ~/.cache/pypoetry/artifacts + ${{ env.WORKDIR }}/.venv + key: py-deps-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-poetry-${{ inputs.poetry-version }}-${{ inputs.cache-key }}-${{ hashFiles(format('{0}/**/poetry.lock', env.WORKDIR)) }} diff --git a/.github/scripts/check_diff.py b/.github/scripts/check_diff.py new file mode 100644 index 0000000..0fcb163 --- /dev/null +++ b/.github/scripts/check_diff.py @@ -0,0 +1,48 @@ +import json +import sys +from typing import Dict + +LIB_DIRS = ["libs/{lib}"] + +if __name__ == "__main__": + files = sys.argv[1:] + + dirs_to_run: Dict[str, set] = { + "lint": set(), + "test": set(), + } + + if len(files) == 300: + # max diff length is 300 files - there are likely files missing + raise ValueError("Max diff reached. Please manually run CI on changed libs.") + + for file in files: + if any( + file.startswith(dir_) + for dir_ in ( + ".github/workflows", + ".github/tools", + ".github/actions", + ".github/scripts/check_diff.py", + ) + ): + # add all LANGCHAIN_DIRS for infra changes + dirs_to_run["test"].update(LIB_DIRS) + + if any(file.startswith(dir_) for dir_ in LIB_DIRS): + for dir_ in LIB_DIRS: + if file.startswith(dir_): + dirs_to_run["test"].add(dir_) + elif file.startswith("libs/"): + raise ValueError( + f"Unknown lib: {file}. check_diff.py likely needs " + "an update for this new library!" + ) + + outputs = { + "dirs-to-lint": list(dirs_to_run["lint"] | dirs_to_run["test"]), + "dirs-to-test": list(dirs_to_run["test"]), + } + for key, value in outputs.items(): + json_output = json.dumps(value) + print(f"{key}={json_output}") # noqa: T201 diff --git a/.github/scripts/get_min_versions.py b/.github/scripts/get_min_versions.py new file mode 100644 index 0000000..46e74c9 --- /dev/null +++ b/.github/scripts/get_min_versions.py @@ -0,0 +1,65 @@ +import sys + +import tomllib +from packaging.version import parse as parse_version +import re + +MIN_VERSION_LIBS = ["langchain-core"] + + +def get_min_version(version: str) -> str: + # case ^x.x.x + _match = re.match(r"^\^(\d+(?:\.\d+){0,2})$", version) + if _match: + return _match.group(1) + + # case >=x.x.x,=(\d+(?:\.\d+){0,2}),<(\d+(?:\.\d+){0,2})$", version) + if _match: + _min = _match.group(1) + _max = _match.group(2) + assert parse_version(_min) < parse_version(_max) + return _min + + # case x.x.x + _match = re.match(r"^(\d+(?:\.\d+){0,2})$", version) + if _match: + return _match.group(1) + + raise ValueError(f"Unrecognized version format: {version}") + + +def get_min_version_from_toml(toml_path: str): + # Parse the TOML file + with open(toml_path, "rb") as file: + toml_data = tomllib.load(file) + + # Get the dependencies from tool.poetry.dependencies + dependencies = toml_data["tool"]["poetry"]["dependencies"] + + # Initialize a dictionary to store the minimum versions + min_versions = {} + + # Iterate over the libs in MIN_VERSION_LIBS + for lib in MIN_VERSION_LIBS: + # Check if the lib is present in the dependencies + if lib in dependencies: + # Get the version string + version_string = dependencies[lib] + + # Use parse_version to get the minimum supported version from version_string + min_version = get_min_version(version_string) + + # Store the minimum version in the min_versions dictionary + min_versions[lib] = min_version + + return min_versions + + +# Get the TOML file path from the command line argument +toml_file = sys.argv[1] + +# Call the function to get the minimum versions +min_versions = get_min_version_from_toml(toml_file) + +print(" ".join([f"{lib}=={version}" for lib, version in min_versions.items()])) diff --git a/.github/workflows/_codespell.yml b/.github/workflows/_codespell.yml new file mode 100644 index 0000000..fc81ef7 --- /dev/null +++ b/.github/workflows/_codespell.yml @@ -0,0 +1,39 @@ +--- +name: make spell_check + +on: + workflow_call: + inputs: + working-directory: + required: true + type: string + description: "From which folder this pipeline executes" + +permissions: + contents: read + +jobs: + codespell: + name: (Check for spelling errors) + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Dependencies + run: | + pip install toml + + - name: Extract Ignore Words List + working-directory: ${{ inputs.working-directory }} + run: | + # Use a Python script to extract the ignore words list from pyproject.toml + python ../../.github/workflows/extract_ignored_words_list.py + id: extract_ignore_words + + - name: Codespell + uses: codespell-project/actions-codespell@v2 + with: + skip: guide_imports.json + ignore_words_list: ${{ steps.extract_ignore_words.outputs.ignore_words_list }} diff --git a/.github/workflows/_compile_integration_test.yml b/.github/workflows/_compile_integration_test.yml new file mode 100644 index 0000000..d8b5b9b --- /dev/null +++ b/.github/workflows/_compile_integration_test.yml @@ -0,0 +1,57 @@ +name: compile-integration-test + +on: + workflow_call: + inputs: + working-directory: + required: true + type: string + description: "From which folder this pipeline executes" + +env: + POETRY_VERSION: "1.7.1" + +jobs: + build: + defaults: + run: + working-directory: ${{ inputs.working-directory }} + runs-on: ubuntu-latest + strategy: + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + name: "poetry run pytest -m compile tests/integration_tests #${{ matrix.python-version }}" + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }} + uses: "./.github/actions/poetry_setup" + with: + python-version: ${{ matrix.python-version }} + poetry-version: ${{ env.POETRY_VERSION }} + working-directory: ${{ inputs.working-directory }} + cache-key: compile-integration + + - name: Install integration dependencies + shell: bash + run: poetry install --with=test_integration,test + + - name: Check integration tests compile + shell: bash + run: poetry run pytest -m compile tests/integration_tests + + - name: Ensure the tests did not create any additional files + shell: bash + run: | + set -eu + + STATUS="$(git status)" + echo "$STATUS" + + # grep will exit non-zero if the target message isn't found, + # and `set -e` above will cause the step to fail. + echo "$STATUS" | grep 'nothing to commit, working tree clean' diff --git a/.github/workflows/_lint.yml b/.github/workflows/_lint.yml new file mode 100644 index 0000000..78baea7 --- /dev/null +++ b/.github/workflows/_lint.yml @@ -0,0 +1,102 @@ +name: lint + +on: + workflow_call: + inputs: + working-directory: + required: true + type: string + description: "From which folder this pipeline executes" + +env: + POETRY_VERSION: "1.7.1" + WORKDIR: ${{ inputs.working-directory == '' && '.' || inputs.working-directory }} + + # This env var allows us to get inline annotations when ruff has complaints. + RUFF_OUTPUT_FORMAT: github + +jobs: + build: + name: "make lint #${{ matrix.python-version }}" + runs-on: ubuntu-latest + strategy: + matrix: + # Only lint on the min and max supported Python versions. + # It's extremely unlikely that there's a lint issue on any version in between + # that doesn't show up on the min or max versions. + # + # GitHub rate-limits how many jobs can be running at any one time. + # Starting new jobs is also relatively slow, + # so linting on fewer versions makes CI faster. + python-version: + - "3.8" + - "3.11" + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }} + uses: "./.github/actions/poetry_setup" + with: + python-version: ${{ matrix.python-version }} + poetry-version: ${{ env.POETRY_VERSION }} + working-directory: ${{ inputs.working-directory }} + cache-key: lint-with-extras + + - name: Check Poetry File + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + poetry check + + - name: Check lock file + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + poetry lock --check + + - name: Install dependencies + # Also installs dev/lint/test/typing dependencies, to ensure we have + # type hints for as many of our libraries as possible. + # This helps catch errors that require dependencies to be spotted, for example: + # https://github.com/langchain-ai/langchain/pull/10249/files#diff-935185cd488d015f026dcd9e19616ff62863e8cde8c0bee70318d3ccbca98341 + # + # If you change this configuration, make sure to change the `cache-key` + # in the `poetry_setup` action above to stop using the old cache. + # It doesn't matter how you change it, any change will cause a cache-bust. + working-directory: ${{ inputs.working-directory }} + run: | + poetry install --with lint,typing + + - name: Get .mypy_cache to speed up mypy + uses: actions/cache@v4 + env: + SEGMENT_DOWNLOAD_TIMEOUT_MIN: "2" + with: + path: | + ${{ env.WORKDIR }}/.mypy_cache + key: mypy-lint-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ inputs.working-directory }}-${{ hashFiles(format('{0}/poetry.lock', inputs.working-directory)) }} + + + - name: Analysing the code with our lint + working-directory: ${{ inputs.working-directory }} + run: | + make lint_package + + - name: Install unit+integration test dependencies + working-directory: ${{ inputs.working-directory }} + run: | + poetry install --with test,test_integration + + - name: Get .mypy_cache_test to speed up mypy + uses: actions/cache@v4 + env: + SEGMENT_DOWNLOAD_TIMEOUT_MIN: "2" + with: + path: | + ${{ env.WORKDIR }}/.mypy_cache_test + key: mypy-test-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ inputs.working-directory }}-${{ hashFiles(format('{0}/poetry.lock', inputs.working-directory)) }} + + - name: Analysing the code with our lint + working-directory: ${{ inputs.working-directory }} + run: | + make lint_tests diff --git a/.github/workflows/_release.yml b/.github/workflows/_release.yml new file mode 100644 index 0000000..8a7a6c6 --- /dev/null +++ b/.github/workflows/_release.yml @@ -0,0 +1,265 @@ +name: release +run-name: Release ${{ inputs.working-directory }} by @${{ github.actor }} +on: + workflow_call: + inputs: + working-directory: + required: true + type: string + description: "From which folder this pipeline executes" + workflow_dispatch: + inputs: + working-directory: + required: true + type: string + default: 'libs/{lib}' + +env: + PYTHON_VERSION: "3.11" + POETRY_VERSION: "1.7.1" + +jobs: + build: + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + + outputs: + pkg-name: ${{ steps.check-version.outputs.pkg-name }} + version: ${{ steps.check-version.outputs.version }} + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + Poetry ${{ env.POETRY_VERSION }} + uses: "./.github/actions/poetry_setup" + with: + python-version: ${{ env.PYTHON_VERSION }} + poetry-version: ${{ env.POETRY_VERSION }} + working-directory: ${{ inputs.working-directory }} + cache-key: release + + # We want to keep this build stage *separate* from the release stage, + # so that there's no sharing of permissions between them. + # The release stage has trusted publishing and GitHub repo contents write access, + # and we want to keep the scope of that access limited just to the release job. + # Otherwise, a malicious `build` step (e.g. via a compromised dependency) + # could get access to our GitHub or PyPI credentials. + # + # Per the trusted publishing GitHub Action: + # > It is strongly advised to separate jobs for building [...] + # > from the publish job. + # https://github.com/pypa/gh-action-pypi-publish#non-goals + - name: Build project for distribution + run: poetry build + working-directory: ${{ inputs.working-directory }} + + - name: Upload build + uses: actions/upload-artifact@v3 + with: + name: dist + path: ${{ inputs.working-directory }}/dist/ + + - name: Check Version + id: check-version + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + echo pkg-name="$(poetry version | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT + echo version="$(poetry version --short)" >> $GITHUB_OUTPUT + + test-pypi-publish: + needs: + - build + uses: + ./.github/workflows/_test_release.yml + permissions: write-all + with: + working-directory: ${{ inputs.working-directory }} + secrets: inherit + + pre-release-checks: + needs: + - build + - test-pypi-publish + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # We explicitly *don't* set up caching here. This ensures our tests are + # maximally sensitive to catching breakage. + # + # For example, here's a way that caching can cause a falsely-passing test: + # - Make the langchain package manifest no longer list a dependency package + # as a requirement. This means it won't be installed by `pip install`, + # and attempting to use it would cause a crash. + # - That dependency used to be required, so it may have been cached. + # When restoring the venv packages from cache, that dependency gets included. + # - Tests pass, because the dependency is present even though it wasn't specified. + # - The package is published, and it breaks on the missing dependency when + # used in the real world. + + - name: Set up Python + Poetry ${{ env.POETRY_VERSION }} + uses: "./.github/actions/poetry_setup" + with: + python-version: ${{ env.PYTHON_VERSION }} + poetry-version: ${{ env.POETRY_VERSION }} + working-directory: ${{ inputs.working-directory }} + + - name: Import published package + shell: bash + working-directory: ${{ inputs.working-directory }} + env: + PKG_NAME: ${{ needs.build.outputs.pkg-name }} + VERSION: ${{ needs.build.outputs.version }} + # Here we use: + # - The default regular PyPI index as the *primary* index, meaning + # that it takes priority (https://pypi.org/simple) + # - The test PyPI index as an extra index, so that any dependencies that + # are not found on test PyPI can be resolved and installed anyway. + # (https://test.pypi.org/simple). This will include the PKG_NAME==VERSION + # package because VERSION will not have been uploaded to regular PyPI yet. + # - attempt install again after 5 seconds if it fails because there is + # sometimes a delay in availability on test pypi + run: | + poetry run pip install \ + --extra-index-url https://test.pypi.org/simple/ \ + "$PKG_NAME==$VERSION" || \ + ( \ + sleep 5 && \ + poetry run pip install \ + --extra-index-url https://test.pypi.org/simple/ \ + "$PKG_NAME==$VERSION" \ + ) + + # Replace all dashes in the package name with underscores, + # since that's how Python imports packages with dashes in the name. + IMPORT_NAME="$(echo "$PKG_NAME" | sed s/-/_/g)" + + poetry run python -c "import $IMPORT_NAME; print(dir($IMPORT_NAME))" + + - name: Import test dependencies + run: poetry install --with test,test_integration + working-directory: ${{ inputs.working-directory }} + + # Overwrite the local version of the package with the test PyPI version. + - name: Import published package (again) + working-directory: ${{ inputs.working-directory }} + shell: bash + env: + PKG_NAME: ${{ needs.build.outputs.pkg-name }} + VERSION: ${{ needs.build.outputs.version }} + run: | + poetry run pip install \ + --extra-index-url https://test.pypi.org/simple/ \ + "$PKG_NAME==$VERSION" + + - name: Run unit tests + run: make tests + working-directory: ${{ inputs.working-directory }} + + - name: Run integration tests + env: + PARTNER_API_KEY: ${{ secrets.PARTNER_API_KEY }} + run: make integration_tests + working-directory: ${{ inputs.working-directory }} + + - name: Get minimum versions + working-directory: ${{ inputs.working-directory }} + id: min-version + run: | + poetry run pip install packaging + min_versions="$(poetry run python $GITHUB_WORKSPACE/.github/scripts/get_min_versions.py pyproject.toml)" + echo "min-versions=$min_versions" >> "$GITHUB_OUTPUT" + echo "min-versions=$min_versions" + + - name: Run unit tests with minimum dependency versions + if: ${{ steps.min-version.outputs.min-versions != '' }} + env: + MIN_VERSIONS: ${{ steps.min-version.outputs.min-versions }} + run: | + poetry run pip install $MIN_VERSIONS + make tests + working-directory: ${{ inputs.working-directory }} + + publish: + needs: + - build + - test-pypi-publish + - pre-release-checks + runs-on: ubuntu-latest + permissions: + # This permission is used for trusted publishing: + # https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/ + # + # Trusted publishing has to also be configured on PyPI for each package: + # https://docs.pypi.org/trusted-publishers/adding-a-publisher/ + id-token: write + + defaults: + run: + working-directory: ${{ inputs.working-directory }} + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + Poetry ${{ env.POETRY_VERSION }} + uses: "./.github/actions/poetry_setup" + with: + python-version: ${{ env.PYTHON_VERSION }} + poetry-version: ${{ env.POETRY_VERSION }} + working-directory: ${{ inputs.working-directory }} + cache-key: release + + - uses: actions/download-artifact@v3 + with: + name: dist + path: ${{ inputs.working-directory }}/dist/ + + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: ${{ inputs.working-directory }}/dist/ + verbose: true + print-hash: true + + mark-release: + needs: + - build + - test-pypi-publish + - pre-release-checks + - publish + runs-on: ubuntu-latest + permissions: + # This permission is needed by `ncipollo/release-action` to + # create the GitHub release. + contents: write + + defaults: + run: + working-directory: ${{ inputs.working-directory }} + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + Poetry ${{ env.POETRY_VERSION }} + uses: "./.github/actions/poetry_setup" + with: + python-version: ${{ env.PYTHON_VERSION }} + poetry-version: ${{ env.POETRY_VERSION }} + working-directory: ${{ inputs.working-directory }} + cache-key: release + + - uses: actions/download-artifact@v3 + with: + name: dist + path: ${{ inputs.working-directory }}/dist/ + + - name: Create Release + uses: ncipollo/release-action@v1 + with: + artifacts: "dist/*" + token: ${{ secrets.GITHUB_TOKEN }} + draft: false + generateReleaseNotes: true + tag: ${{ inputs.working-directory }}/v${{ needs.build.outputs.version }} + commit: main diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml new file mode 100644 index 0000000..8f2ace4 --- /dev/null +++ b/.github/workflows/_test.yml @@ -0,0 +1,58 @@ +name: test + +on: + workflow_call: + inputs: + working-directory: + required: true + type: string + description: "From which folder this pipeline executes" + +env: + POETRY_VERSION: "1.7.1" + +jobs: + build: + defaults: + run: + working-directory: ${{ inputs.working-directory }} + runs-on: ubuntu-latest + strategy: + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.11" + name: "make test #${{ matrix.python-version }}" + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }} + uses: "./.github/actions/poetry_setup" + with: + python-version: ${{ matrix.python-version }} + poetry-version: ${{ env.POETRY_VERSION }} + working-directory: ${{ inputs.working-directory }} + cache-key: core + + - name: Install dependencies + shell: bash + run: poetry install --with test + + - name: Run core tests + shell: bash + run: | + make test + + - name: Ensure the tests did not create any additional files + shell: bash + run: | + set -eu + + STATUS="$(git status)" + echo "$STATUS" + + # grep will exit non-zero if the target message isn't found, + # and `set -e` above will cause the step to fail. + echo "$STATUS" | grep 'nothing to commit, working tree clean' diff --git a/.github/workflows/_test_release.yml b/.github/workflows/_test_release.yml new file mode 100644 index 0000000..f49e505 --- /dev/null +++ b/.github/workflows/_test_release.yml @@ -0,0 +1,95 @@ +name: test-release + +on: + workflow_call: + inputs: + working-directory: + required: true + type: string + description: "From which folder this pipeline executes" + +env: + POETRY_VERSION: "1.7.1" + PYTHON_VERSION: "3.10" + +jobs: + build: + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + + outputs: + pkg-name: ${{ steps.check-version.outputs.pkg-name }} + version: ${{ steps.check-version.outputs.version }} + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + Poetry ${{ env.POETRY_VERSION }} + uses: "./.github/actions/poetry_setup" + with: + python-version: ${{ env.PYTHON_VERSION }} + poetry-version: ${{ env.POETRY_VERSION }} + working-directory: ${{ inputs.working-directory }} + cache-key: release + + # We want to keep this build stage *separate* from the release stage, + # so that there's no sharing of permissions between them. + # The release stage has trusted publishing and GitHub repo contents write access, + # and we want to keep the scope of that access limited just to the release job. + # Otherwise, a malicious `build` step (e.g. via a compromised dependency) + # could get access to our GitHub or PyPI credentials. + # + # Per the trusted publishing GitHub Action: + # > It is strongly advised to separate jobs for building [...] + # > from the publish job. + # https://github.com/pypa/gh-action-pypi-publish#non-goals + - name: Build project for distribution + run: poetry build + working-directory: ${{ inputs.working-directory }} + + - name: Upload build + uses: actions/upload-artifact@v3 + with: + name: test-dist + path: ${{ inputs.working-directory }}/dist/ + + - name: Check Version + id: check-version + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + echo pkg-name="$(poetry version | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT + echo version="$(poetry version --short)" >> $GITHUB_OUTPUT + + publish: + needs: + - build + runs-on: ubuntu-latest + permissions: + # This permission is used for trusted publishing: + # https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/ + # + # Trusted publishing has to also be configured on PyPI for each package: + # https://docs.pypi.org/trusted-publishers/adding-a-publisher/ + id-token: write + + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v3 + with: + name: test-dist + path: ${{ inputs.working-directory }}/dist/ + + - name: Publish to test PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: ${{ inputs.working-directory }}/dist/ + verbose: true + print-hash: true + repository-url: https://test.pypi.org/legacy/ + + # We overwrite any existing distributions with the same name and version. + # This is *only for CI use* and is *extremely dangerous* otherwise! + # https://github.com/pypa/gh-action-pypi-publish#tolerating-release-package-file-duplicates + skip-existing: true diff --git a/.github/workflows/check_diffs.yml b/.github/workflows/check_diffs.yml new file mode 100644 index 0000000..b6f4891 --- /dev/null +++ b/.github/workflows/check_diffs.yml @@ -0,0 +1,89 @@ +--- +name: CI + +on: + push: + branches: [main] + pull_request: + +# If another push to the same PR or branch happens while this workflow is still running, +# cancel the earlier run in favor of the next run. +# +# There's no point in testing an outdated version of the code. GitHub only allows +# a limited number of job runners to be active at the same time, so it's better to cancel +# pointless jobs early so that more useful jobs can run sooner. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + POETRY_VERSION: "1.7.1" + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - id: files + uses: Ana06/get-changed-files@v2.2.0 + - id: set-matrix + run: | + python .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUT + outputs: + dirs-to-lint: ${{ steps.set-matrix.outputs.dirs-to-lint }} + dirs-to-test: ${{ steps.set-matrix.outputs.dirs-to-test }} + lint: + name: cd ${{ matrix.working-directory }} + needs: [ build ] + if: ${{ needs.build.outputs.dirs-to-lint != '[]' }} + strategy: + matrix: + working-directory: ${{ fromJson(needs.build.outputs.dirs-to-lint) }} + uses: ./.github/workflows/_lint.yml + with: + working-directory: ${{ matrix.working-directory }} + secrets: inherit + + test: + name: cd ${{ matrix.working-directory }} + needs: [ build ] + if: ${{ needs.build.outputs.dirs-to-test != '[]' }} + strategy: + matrix: + working-directory: ${{ fromJson(needs.build.outputs.dirs-to-test) }} + uses: ./.github/workflows/_test.yml + with: + working-directory: ${{ matrix.working-directory }} + secrets: inherit + + compile-integration-tests: + name: cd ${{ matrix.working-directory }} + needs: [ build ] + if: ${{ needs.build.outputs.dirs-to-test != '[]' }} + strategy: + matrix: + working-directory: ${{ fromJson(needs.build.outputs.dirs-to-test) }} + uses: ./.github/workflows/_compile_integration_test.yml + with: + working-directory: ${{ matrix.working-directory }} + secrets: inherit + ci_success: + name: "CI Success" + needs: [build, lint, test, compile-integration-tests] + if: | + always() + runs-on: ubuntu-latest + env: + JOBS_JSON: ${{ toJSON(needs) }} + RESULTS_JSON: ${{ toJSON(needs.*.result) }} + EXIT_CODE: ${{!contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') && '0' || '1'}} + steps: + - name: "CI Success" + run: | + echo $JOBS_JSON + echo $RESULTS_JSON + echo "Exiting with $EXIT_CODE" + exit $EXIT_CODE diff --git a/.github/workflows/extract_ignored_words_list.py b/.github/workflows/extract_ignored_words_list.py new file mode 100644 index 0000000..7c800e0 --- /dev/null +++ b/.github/workflows/extract_ignored_words_list.py @@ -0,0 +1,10 @@ +import toml + +pyproject_toml = toml.load("pyproject.toml") + +# Extract the ignore words list (adjust the key as per your TOML structure) +ignore_words_list = ( + pyproject_toml.get("tool", {}).get("codespell", {}).get("ignore-words-list") +) + +print(f"::set-output name=ignore_words_list::{ignore_words_list}") diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..57d0481 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 LangChain + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..3b485db --- /dev/null +++ b/README.md @@ -0,0 +1,59 @@ +# 🦜️🔗 LangChain {partner} + +This repository contains 1 package with {partner} integrations with LangChain: + +- [langchain-{package_lower}](https://pypi.org/project/langchain-{package_lower}/) + +## Initial Repo Checklist (Remove this section after completing) + +This setup assumes that the partner package is already split. For those instructions, +see [these docs](https://python.langchain.com/docs/contributing/integrations#partner-packages). + +Code (auto ecli) + +- [ ] Fill out the readme above (for folks that follow pypi link) +- [ ] Copy package into /libs folder +- [ ] Update these fields in /libs/*/pyproject.toml + + - `tool.poetry.repository` + - `tool.poetry.urls["Source Code"]` + +Workflow code (auto ecli) + +- [ ] Populate .github/workflows/_release.yml with `on.workflow_dispatch.inputs.working-directory.default` +- [ ] Configure `LIB_DIRS` in .github/scripts/check_diff.py + +Workflow code (manual) + +- [ ] Add secrets as env vars in .github/workflows/_release.yml + +In github (manual) + +- [ ] Add integration testing secrets in Github (ask Erick for help) +- [ ] Add partner collaborators in Github (ask Erick for help) +- [ ] "Allow auto-merge" in General Settings +- [ ] Only "Allow squash merging" in General Settings +- [ ] Set up ruleset matching CI build (ask Erick for help) + - name: ci build + - enforcement: active + - bypass: write + - target: default branch + - rules: restrict deletions, require status checks ("CI Success"), block force pushes +- [ ] Set up ruleset + - name: require prs + - enforcement: active + - bypass: none + - target: default branch + - rules: restrict deletions, require a pull request before merging (0 approvals, no boxes), block force pushes + +Pypi (manual) + +- [ ] Add new repo to test-pypi and pypi trusted publishing (ask Erick for help) + +Slack + +- [ ] Set up release alerting in Slack (ask Erick for help) + +release: +/github subscribe langchain-ai/langchain-{partner_lower} releases workflows:{name:"release"} +/github unsubscribe langchain-ai/langchain-{partner_lower} issues pulls commits deployments