diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..837bdb7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,38 @@ +--- +name: Bug report +about: Create a report to help us improve +title: 'bug: [DESCRIPTION]' +labels: 'type: bug' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (please complete the following information):** + - OS: [e.g. iOS] + - Browser [e.g. chrome, safari] + - Version [e.g. 22] + +**Smartphone (please complete the following information):** + - Device: [e.g. iPhone6] + - OS: [e.g. iOS8.1] + - Browser [e.g. stock browser, safari] + - Version [e.g. 22] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/discussion-thread.md b/.github/ISSUE_TEMPLATE/discussion-thread.md new file mode 100644 index 0000000..09e52ae --- /dev/null +++ b/.github/ISSUE_TEMPLATE/discussion-thread.md @@ -0,0 +1,14 @@ +--- +name: Discussion thread +about: Start an open ended discussion +title: 'Discussion: [TOPIC HERE]' +labels: '' +assignees: '' + +--- + +**Motivation** + +**Discussion** + +**Resources** diff --git a/.github/ISSUE_TEMPLATE/epic-request.md b/.github/ISSUE_TEMPLATE/epic-request.md new file mode 100644 index 0000000..bfad8e5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/epic-request.md @@ -0,0 +1,20 @@ +--- +name: Epic request +about: Suggest an idea for this project +title: 'epic: [DESCRIPTION]' +labels: 'type: epic' +assignees: '' + +--- + +**Problem** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Success Criteria** +A clear and concise description of what you want to happen. + +**Sub Issues** +- + +**Additional context** +Add any other context or screenshots about the epic request here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..26f586b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,17 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: 'feat: [DESCRIPTION]' +labels: 'type: feature request' +assignees: '' + +--- + +**Problem** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Success Criteria** +A clear and concise description of what you want to happen. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..320545b --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,14 @@ +## Describe Your Changes + +- + +## Fixes Issues + +- Closes # +- Closes # + +## Self Checklist + +- [ ] Added relevant comments, esp in complex areas +- [ ] Updated docs (for bug fixes / features) +- [ ] Created issues for follow-up changes or refactoring needed \ No newline at end of file diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml new file mode 100644 index 0000000..dd7239d --- /dev/null +++ b/.github/release-drafter.yml @@ -0,0 +1,26 @@ +categories: + - title: '🚀 Features' + labels: + - 'type: enhancement' + - 'type: epic' + - 'type: feature request' + - title: '🐛 Bug Fixes' + labels: + - 'type: bug' + - title: '🧰 Maintenance' + labels: + - 'type: chore' + - 'type: ci' + - title: '📖 Documentaion' + labels: + - 'type: documentation' +change-template: '- $TITLE @$AUTHOR (#$NUMBER)' +change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks. +template: | + ## Changes + + $CHANGES + + ## Contributor + + $CONTRIBUTORS \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..27e49e9 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,146 @@ +name: CI Cortex Release + +on: + push: + tags: ["v[0-9]+.[0-9]+.[0-9]+"] + paths: ["/**"] + workflow_dispatch: + +jobs: + create-draft-release: + runs-on: ubuntu-latest + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + outputs: + upload_url: ${{ steps.create_release.outputs.upload_url }} + version: ${{ steps.get_version.outputs.version }} + permissions: + contents: write + steps: + - name: Extract tag name without v prefix + id: get_version + run: | + echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV && echo "::set-output name=version::${GITHUB_REF#refs/tags/v}" + env: + GITHUB_REF: ${{ github.ref }} + - name: Create Draft Release + id: create_release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ github.ref_name }} + token: ${{ secrets.GITHUB_TOKEN }} + name: "${{ env.VERSION }}" + draft: true + prerelease: false + + build-and-test: + runs-on: ${{ matrix.runs-on }} + needs: [create-draft-release] + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + include: + - os: "linux" + name: "amd64" + runs-on: "ubuntu-20-04-cuda-12-0" + cmake-flags: "-DTOKENIZER_VERSION=${{github.event.pull_request.head.sha}}" + build-deps-cmake-flags: "" + ccache-dir: '' + - os: "mac" + name: "amd64" + runs-on: "macos-13" + cmake-flags: "-DTOKENIZER_VERSION=${{github.event.pull_request.head.sha}}" + build-deps-cmake-flags: "" + ccache-dir: '' + - os: "mac" + name: "arm64" + runs-on: "macos-latest" + cmake-flags: "-DTOKENIZER_VERSION=${{github.event.pull_request.head.sha}}" + build-deps-cmake-flags: "" + ccache-dir: '' + - os: "windows" + name: "amd64" + runs-on: "windows-cuda-12-0" + cmake-flags: "-DTOKENIZER_VERSION=${{github.event.pull_request.head.sha}}" + build-deps-cmake-flags: "" + ccache-dir: '' + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v3 + with: + submodules: recursive + + - uses: actions/setup-dotnet@v3 + if: runner.os == 'Windows' + with: + dotnet-version: "8.0.x" + + - uses: actions/setup-node@v3 + with: + node-version: "20.x" + registry-url: "https://registry.npmjs.org" + + - name: Install choco on Windows + if: runner.os == 'Windows' + run: | + choco install make pkgconfiglite ccache awscli 7zip ninja -y + + - name: Get Cer for code signing + if: runner.os == 'macOS' + run: base64 -d <<< "$CODE_SIGN_P12_BASE64" > /tmp/codesign.p12 + shell: bash + env: + CODE_SIGN_P12_BASE64: ${{ secrets.CODE_SIGN_P12_BASE64 }} + + - uses: apple-actions/import-codesign-certs@v2 + if: runner.os == 'macOS' + with: + p12-file-base64: ${{ secrets.CODE_SIGN_P12_BASE64 }} + p12-password: ${{ secrets.CODE_SIGN_P12_PASSWORD }} + + - name: Build + run: | + make build CMAKE_EXTRA_FLAGS="${{ matrix.cmake-flags }}" BUILD_DEPS_CMAKE_EXTRA_FLAGS="${{ matrix.build-deps-cmake-flags }}" + + - name: Pre-package + run: | + make pre-package + + - name: Code Signing macOS + if: runner.os == 'macOS' + run: | + make codesign CODE_SIGN=true DEVELOPER_ID="${{ secrets.DEVELOPER_ID }}" + + - uses: nick-fields/retry@v3 + with: + continue_on_error: true + retry_wait_seconds: 10 + timeout_minutes: 10 + max_attempts: 3 + shell: cmd + command: | + set PATH=%PATH%;%USERPROFILE%\.dotnet\tools + make codesign CODE_SIGN=true AZURE_KEY_VAULT_URI="${{ secrets.AZURE_KEY_VAULT_URI }}" AZURE_CLIENT_ID="${{ secrets.AZURE_CLIENT_ID }}" AZURE_TENANT_ID="${{ secrets.AZURE_TENANT_ID }}" AZURE_CLIENT_SECRET="${{ secrets.AZURE_CLIENT_SECRET }}" AZURE_CERT_NAME="${{ secrets.AZURE_CERT_NAME }}" + name: Code Signing Windows + if: runner.os == 'Windows' + + - name: Package + run: | + make package + + - uses: actions/upload-release-asset@v1.0.1 + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create-draft-release.outputs.upload_url }} + asset_path: ./tokenizer.tar.gz + asset_name: tokenizer-${{ needs.create-draft-release.outputs.version }}-${{ matrix.os }}-${{ matrix.name }}.tar.gz + asset_content_type: application/gzip + + - name: Remove build build-deps and build folder for windows + if: runner.os == 'Windows' + run: | + Remove-Item -Recurse -Force build diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml new file mode 100644 index 0000000..66b4257 --- /dev/null +++ b/.github/workflows/quality-gate.yml @@ -0,0 +1,70 @@ +name: CI Quality Gate Tokenizer.cpp + +on: + pull_request: + types: [opened, synchronize, reopened] + workflow_dispatch: + +env: + +jobs: + build-and-test: + runs-on: ${{ matrix.runs-on }} + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + include: + - os: "linux" + name: "amd64" + runs-on: "ubuntu-20-04-cuda-12-0" + cmake-flags: "-DTOKENIZER_VERSION=${{github.event.pull_request.head.sha}}" + build-deps-cmake-flags: "" + ccache-dir: '' + - os: "mac" + name: "amd64" + runs-on: "macos-13" + cmake-flags: "-DTOKENIZER_VERSION=${{github.event.pull_request.head.sha}}" + build-deps-cmake-flags: "" + ccache-dir: '' + - os: "mac" + name: "arm64" + runs-on: "macos-latest" + cmake-flags: "-DTOKENIZER_VERSION=${{github.event.pull_request.head.sha}}" + build-deps-cmake-flags: "" + ccache-dir: '' + - os: "windows" + name: "amd64" + runs-on: "windows-cuda-12-0" + cmake-flags: "-DTOKENIZER_VERSION=${{github.event.pull_request.head.sha}}" + build-deps-cmake-flags: "" + ccache-dir: '' + steps: + - name: Clone + id: checkout + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Install choco on Windows + if: runner.os == 'Windows' + run: | + choco install make pkgconfiglite ccache awscli 7zip ninja -y + + - name: Build + run: | + make build CMAKE_EXTRA_FLAGS="${{ matrix.cmake-flags }}" BUILD_DEPS_CMAKE_EXTRA_FLAGS="${{ matrix.build-deps-cmake-flags }}" + + - name: Pre-package + run: | + make pre-package + + - name: Package + run: | + make package + + - name: Upload Artifact + uses: actions/upload-artifact@v2 + with: + name: tokenizer-${{ matrix.os }}-${{ matrix.name }} + path: ./tokenizer \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..906b8f7 --- /dev/null +++ b/Makefile @@ -0,0 +1,78 @@ +# Makefile for Cortex llamacpp engine - Build, Lint, Test, and Clean +.PHONY: all build package + +BUILD_DEPS_CMAKE_EXTRA_FLAGS ?= "" +CMAKE_EXTRA_FLAGS ?= "" +RUN_TESTS ?= false +CODE_SIGN ?= false +AZURE_KEY_VAULT_URI ?= xxxx +AZURE_CLIENT_ID ?= xxxx +AZURE_TENANT_ID ?= xxxx +AZURE_CLIENT_SECRET ?= xxxx +AZURE_CERT_NAME ?= xxxx +DEVELOPER_ID ?= xxxx + +# Default target, does nothing +all: + @echo "Specify a target to run" + +# Build the library +build: +ifeq ($(OS),Windows_NT) + @powershell -Command "mkdir -p build; cd build; cmake .. $(CMAKE_EXTRA_FLAGS); cmake --build . --config Release -j4;" +else ifeq ($(shell uname -s),Linux) + @mkdir -p build && cd build; \ + cmake .. $(CMAKE_EXTRA_FLAGS); \ + make -j4; +else + @mkdir -p build && cd build; \ + cmake .. $(CMAKE_EXTRA_FLAGS); \ + make -j4; +endif + +# TODO(sang) header file +pre-package: +ifeq ($(OS),Windows_NT) + @powershell -Command "mkdir -p tokenizer;" + @powershell -Command "cp build\Release\tokenizer.dll .\tokenizer\;" +else ifeq ($(shell uname -s),Linux) + @mkdir -p tokenizer; \ + cp build/tokenizer tokenizer/; +else + @mkdir -p tokenizer; \ + cp build/tokenizer tokenizer/; +endif + +codesign: +ifeq ($(CODE_SIGN),false) + @echo "Skipping Code Sign" + @exit 0 +endif + +ifeq ($(OS),Windows_NT) + @powershell -Command "dotnet tool install --global AzureSignTool;" + @powershell -Command 'azuresigntool.exe sign -kvu "$(AZURE_KEY_VAULT_URI)" -kvi "$(AZURE_CLIENT_ID)" -kvt "$(AZURE_TENANT_ID)" -kvs "$(AZURE_CLIENT_SECRET)" -kvc "$(AZURE_CERT_NAME)" -tr http://timestamp.globalsign.com/tsa/r6advanced1 -v ".\tokenizer\tokenizer.dll";' +else ifeq ($(shell uname -s),Linux) + @echo "Skipping Code Sign for linux" + @exit 0 +else + find "tokenizer" -type f -exec codesign --force -s "$(DEVELOPER_ID)" --options=runtime {} \; +endif + +package: +ifeq ($(OS),Windows_NT) + @powershell -Command "7z a -ttar temp.tar tokenizer\*; 7z a -tgzip tokenizer.tar.gz temp.tar;" +else ifeq ($(shell uname -s),Linux) + tar -czvf tokenizer.tar.gz tokenizer; +else + tar -czvf tokenizer.tar.gz tokenizer; +endif + +clean: +ifeq ($(OS),Windows_NT) + @powershell -Command "rm -rf build; rm -rf build-deps; rm -rf tokenizer; rm -rf tokenizer.tar.gz;" +else ifeq ($(shell uname -s),Linux) + @rm -rf build; rm -rf build-deps; rm -rf tokenizer; rm -rf tokenizer.tar.gz; +else + @rm -rf build; rm -rf build-deps; rm -rf tokenizer; rm -rf tokenizer.tar.gz; +endif \ No newline at end of file