job_tokenizers.yml

name: OpenVINO tokenizers extension

on:
  workflow_call:
    inputs:
      runner:
        description: 'Machine on which the tests would run'
        type: string
        required: true
      shell:
        description: "shell to override the default shell settings in the runner's operating system."
        type: string
        required: true
      container:
        description: 'JSON to be converted to the value of the "container" configuration for the job'
        type: string
        required: false
        default: '{"image": null}'
      affected-components:
        description: 'Components that are affected by changes in the commit defined by the Smart CI Action'
        type: string
        required: true

permissions: read-all

env:
  PIP_CACHE_PATH: /mount/caches/pip/linux
  PYTHON_VERSION: '3.11'
  TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }}

jobs:
  Openvino_tokenizers:
    name: OpenVINO tokenizers extension
    timeout-minutes: 25
    defaults:
      run:
        shell: ${{ inputs.shell }}
    runs-on: ${{ inputs.runner }}
    container: ${{ fromJSON(inputs.container) }}
    env:
      INSTALL_DIR: ${{ github.workspace }}/install
      OPENVINO_TOKENIZERS_REPO: ${{ github.workspace }}/openvino_tokenizers
      EXTENSION_BUILD_DIR: ${{ github.workspace }}/build

    steps:
      # Needed as ${{ github.workspace }} is not working correctly when using Docker
      - name: Setup Variables
        run: |
          echo "INSTALL_DIR=$GITHUB_WORKSPACE/install" >> "$GITHUB_ENV"
          echo "OPENVINO_TOKENIZERS_REPO=$GITHUB_WORKSPACE/openvino_tokenizers" >> "$GITHUB_ENV"
          echo "EXTENSION_BUILD_DIR=$GITHUB_WORKSPACE/build" >> "$GITHUB_ENV"

      - name: checkout action
        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
        with:
          sparse-checkout: |
            .github/actions/setup_python
            .github/actions/cache
            install_build_dependencies.sh

      - name: Setup Python ${{ env.PYTHON_VERSION }}
        uses: ./.github/actions/setup_python
        with:
          version: ${{ env.PYTHON_VERSION }}
          pip-cache-path: ${{ runner.os == 'Linux' && env.PIP_CACHE_PATH || '' }}
          should-setup-pip-paths: ${{ runner.os == 'Linux' }}
          self-hosted-runner: ${{ runner.os == 'Linux' }}

      - name: Clone OpenVINO Tokenizers
        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
        with:
          repository: 'openvinotoolkit/openvino_tokenizers'
          path: ${{ env.OPENVINO_TOKENIZERS_REPO }}
          ref: ${{ env.TARGET_BRANCH }}

      - name: Download OpenVINO package
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          name: openvino_package
          path: ${{ env.INSTALL_DIR }}

      - name: Extract OpenVINO packages (Linux and macOS)
        if: runner.os != 'Windows'
        run: |
          pushd ${INSTALL_DIR}
            tar -I pigz -xf openvino_package.tar.gz -C ${INSTALL_DIR}
          popd

      - name: Extract OpenVINO packages (Windows)
        if: runner.os == 'Windows'
        run: |
          pushd ${{ env.INSTALL_DIR }}
            Expand-Archive openvino_package.zip -DestinationPath "${{ env.INSTALL_DIR }}"
          popd

      #
      # Dependencies
      #

      - name: Install OpenVINO Python wheel (Linux and macOS)
        if: runner.os != 'Windows'
        run: |
          # Find and install wheel
          pushd ${INSTALL_DIR}/tools
            wheel_name=$(find . -name 'openvino-*.whl')
            python3 -m pip install $wheel_name
          popd

      - name: Install OpenVINO Python wheel (Windows)
        if: runner.os == 'Windows'
        run: |
          # Find and install wheel
          $ovCoreWheelPath=Get-ChildItem -Path "${{ env.INSTALL_DIR }}\\tools" -Filter openvino-*.whl | % { $_.FullName }
          python3 -m pip install "$ovCoreWheelPath"

      #
      # Build
      #

      - name: Build tokenizers wheel (Linux and macOS)
        if: runner.os != 'Windows'
        run: |
          # use OpenVINO wheel package only to build the extension
          python -m pip wheel -v --no-deps --wheel-dir ${EXTENSION_BUILD_DIR} --find-links ${INSTALL_DIR}/tools ${OPENVINO_TOKENIZERS_REPO}
        env:
          CMAKE_BUILD_PARALLEL_LEVEL: '4'

      - name: Build tokenizers wheel (Windows)
        if: runner.os == 'Windows'
        run: |
          python3 -m pip wheel -v --no-deps --wheel-dir ${env:EXTENSION_BUILD_DIR} --find-links ${env:INSTALL_DIR}/tools ${env:OPENVINO_TOKENIZERS_REPO}
        env:
          CMAKE_BUILD_PARALLEL_LEVEL: '4'

      #
      # Upload build artifacts
      #

      - name: Upload openvino tokenizers wheel
        if: ${{ always() }}
        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
        with:
          name: openvino_tokenizers_wheel
          path: ${{ env.EXTENSION_BUILD_DIR }}/*.whl
          if-no-files-found: 'error'