diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml index 3faf80e..bd7b726 100644 --- a/.github/workflows/publish_to_pypi.yml +++ b/.github/workflows/publish_to_pypi.yml @@ -27,9 +27,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-18.04] + os: [ubuntu-18.04, macos-10.15] cuda: ["10.1"] - gcc: ["5"] torch: ["1.8.1"] python-version: [3.6, 3.7, 3.8] steps: @@ -42,7 +41,16 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Install GCC 7 + if: startsWith(matrix.os, 'ubuntu') + run: | + sudo apt-get install -y gcc-7 g++-7 + echo "CC=/usr/bin/gcc-7" >> $GITHUB_ENV + echo "CXX=/usr/bin/g++-7" >> $GITHUB_ENV + + - name: Install CUDA Toolkit ${{ matrix.cuda }} + if: startsWith(matrix.os, 'ubuntu') shell: bash env: cuda: ${{ matrix.cuda }} @@ -53,18 +61,13 @@ jobs: echo "LD_LIBRARY_PATH=${CUDA_HOME}/lib:${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV - name: Display NVCC version + if: startsWith(matrix.os, 'ubuntu') run: | which nvcc nvcc --version - - name: Install GCC ${{ matrix.gcc }} - run: | - sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }} - echo "CC=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV - echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV - echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV - - name: Install PyTorch ${{ matrix.torch }} + if: startsWith(matrix.os, 'ubuntu') env: cuda: ${{ matrix.cuda }} torch: ${{ matrix.torch }} @@ -77,7 +80,18 @@ jobs: ./scripts/github_actions/install_torch.sh python3 -c "import torch; print('torch version:', torch.__version__)" + - name: Install PyTorch ${{ matrix.torch }} + if: startsWith(matrix.os, 'macos') + shell: bash + run: | + python3 -m pip install -qq --upgrade pip + python3 -m pip install -qq wheel twine typing_extensions + python3 -m pip install -qq torch==${{ matrix.torch }} + + python3 -c "import torch; print('torch version:', torch.__version__)" + - name: Download cudnn 8.0 + if: startsWith(matrix.os, 'ubuntu') env: cuda: ${{ matrix.cuda }} run: | diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 0000000..d1be1fc --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,117 @@ +# Copyright 2021 Xiaomi Corp. (author: Fangjun Kuang) + +# See ../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: Run tests + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + run_tests: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-18.04, macos-10.15] + cuda: ["10.1"] + torch: ["1.8.1"] + python-version: [3.6, 3.7, 3.8] + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install GCC 7 + if: startsWith(matrix.os, 'ubuntu') + run: | + sudo apt-get install -y gcc-7 g++-7 + echo "CC=/usr/bin/gcc-7" >> $GITHUB_ENV + echo "CXX=/usr/bin/g++-7" >> $GITHUB_ENV + + + - name: Install CUDA Toolkit ${{ matrix.cuda }} + if: startsWith(matrix.os, 'ubuntu') + shell: bash + env: + cuda: ${{ matrix.cuda }} + run: | + source ./scripts/github_actions/install_cuda.sh + echo "CUDA_HOME=${CUDA_HOME}" >> $GITHUB_ENV + echo "${CUDA_HOME}/bin" >> $GITHUB_PATH + echo "LD_LIBRARY_PATH=${CUDA_HOME}/lib:${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV + + - name: Display NVCC version + if: startsWith(matrix.os, 'ubuntu') + run: | + which nvcc + nvcc --version + + - name: Install PyTorch ${{ matrix.torch }} + if: startsWith(matrix.os, 'ubuntu') + env: + cuda: ${{ matrix.cuda }} + torch: ${{ matrix.torch }} + shell: bash + run: | + sudo apt update + sudo apt install libsndfile1-dev libsndfile1 ffmpeg + python3 -m pip install --upgrade pip + python3 -m pip install wheel twine typing_extensions soundfile + python3 -m pip install bs4 requests tqdm + + ./scripts/github_actions/install_torch.sh + python3 -c "import torch; print('torch version:', torch.__version__)" + + - name: Install PyTorch ${{ matrix.torch }} + if: startsWith(matrix.os, 'macos') + shell: bash + run: | + python3 -m pip install -qq --upgrade pip + python3 -m pip install -qq wheel twine typing_extensions soundfile + python3 -m pip install -qq torch==${{ matrix.torch }} + + python3 -c "import torch; print('torch version:', torch.__version__)" + + - name: Download cudnn 8.0 + if: startsWith(matrix.os, 'ubuntu') + env: + cuda: ${{ matrix.cuda }} + run: | + ./scripts/github_actions/install_cudnn.sh + + - name: Build + shell: bash + run: | + mkdir build_release + cd build_release + cmake .. + make VERBOSE=1 + + - name: Run tests + shell: bash + run: | + cd build_release + ctest -R py --output-on-failure diff --git a/.github/workflows/style_check.yml b/.github/workflows/style_check.yml new file mode 100644 index 0000000..20c3363 --- /dev/null +++ b/.github/workflows/style_check.yml @@ -0,0 +1,62 @@ +# Copyright 2021 Fangjun Kuang (csukuangfj@gmail.com) + +# See ../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: style_check + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + style_check: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-18.04, macos-10.15] + python-version: [3.7, 3.9] + fail-fast: false + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Python dependencies + run: | + python3 -m pip install --upgrade pip black flake8 + + - name: Run flake8 + shell: bash + working-directory: ${{github.workspace}} + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --show-source --statistics + flake8 . + + - name: Run black + shell: bash + working-directory: ${{github.workspace}} + run: | + black --check --diff . diff --git a/CMakeLists.txt b/CMakeLists.txt index 8813346..8263966 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ cmake_minimum_required(VERSION 3.8 FATAL_ERROR) project(kaldifeat) -set(kaldifeat_VERSION "1.3") +set(kaldifeat_VERSION "1.4") set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") diff --git a/README.md b/README.md index 408c346..cbc58f3 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # kaldifeat +
+ +
+ + diff --git a/cmake/cmake_extension.py b/cmake/cmake_extension.py index 413a87d..ceb0d61 100644 --- a/cmake/cmake_extension.py +++ b/cmake/cmake_extension.py @@ -2,6 +2,7 @@ import glob import os +import platform import shutil import sys from pathlib import Path @@ -15,6 +16,10 @@ def is_for_pypi(): return ans is not None +def is_macos(): + return platform.system() == "Darwin" + + try: from wheel.bdist_wheel import bdist_wheel as _bdist_wheel @@ -23,7 +28,7 @@ def finalize_options(self): _bdist_wheel.finalize_options(self) # In this case, the generated wheel has a name in the form # k2-xxx-pyxx-none-any.whl - if is_for_pypi(): + if is_for_pypi() and not is_macos(): self.root_is_pure = True else: # The generated wheel has a name ending with @@ -85,6 +90,7 @@ def build_extension(self, ext: setuptools.extension.Extension): ) lib_so = glob.glob(f"{self.build_temp}/lib/*kaldifeat*.so") + lib_so += glob.glob(f"{self.build_temp}/lib/*kaldifeat*.dylib") # macOS for so in lib_so: print(f"Copying {so} to {self.build_lib}/") shutil.copy(f"{so}", f"{self.build_lib}/") diff --git a/cmake/pybind11.cmake b/cmake/pybind11.cmake index 1677d11..4cad4e8 100644 --- a/cmake/pybind11.cmake +++ b/cmake/pybind11.cmake @@ -17,9 +17,6 @@ function(download_pybind11) FetchContent_Declare(pybind11 URL ${pybind11_URL} URL_HASH ${pybind11_HASH} - PATCH_COMMAND - sed -i s/\\${double_quotes}-flto\\\\${dollar}/\\${double_quotes}-Xcompiler=-flto${dollar}/g "tools/pybind11Tools.cmake" && - sed -i s/${seimcolon}-fno-fat-lto-objects/${seimcolon}-Xcompiler=-fno-fat-lto-objects/g "tools/pybind11Tools.cmake" ) FetchContent_GetProperties(pybind11) diff --git a/cmake/torch.cmake b/cmake/torch.cmake index 1377583..c35210b 100644 --- a/cmake/torch.cmake +++ b/cmake/torch.cmake @@ -23,17 +23,3 @@ execute_process( ) message(STATUS "PyTorch version: ${TORCH_VERSION}") - -# Solve the following error for NVCC: -# unknown option `-Wall` -# -# It contains only some -Wno-* flags, so it is OK -# to set them to empty -set_property(TARGET torch_cuda - PROPERTY - INTERFACE_COMPILE_OPTIONS "" -) -set_property(TARGET torch_cpu - PROPERTY - INTERFACE_COMPILE_OPTIONS "" -) diff --git a/doc/source/images/os.svg b/doc/source/images/os.svg new file mode 100644 index 0000000..314bf44 --- /dev/null +++ b/doc/source/images/os.svg @@ -0,0 +1 @@ +OS: Linux | macOSOSLinux | macOS diff --git a/kaldifeat/csrc/feature-window.cc b/kaldifeat/csrc/feature-window.cc index bec5e14..4d24c8e 100644 --- a/kaldifeat/csrc/feature-window.cc +++ b/kaldifeat/csrc/feature-window.cc @@ -157,7 +157,7 @@ torch::Tensor GetStrided(const torch::Tensor &wave, } torch::Tensor Dither(const torch::Tensor &wave, float dither_value) { - if (dither_value == 0.0f) wave; + if (dither_value == 0.0f) return wave; torch::Tensor rand_gauss = torch::randn_like(wave); #if 1 diff --git a/kaldifeat/python/CMakeLists.txt b/kaldifeat/python/CMakeLists.txt index 86735ca..60d6382 100644 --- a/kaldifeat/python/CMakeLists.txt +++ b/kaldifeat/python/CMakeLists.txt @@ -1 +1,2 @@ add_subdirectory(csrc) +add_subdirectory(tests) diff --git a/kaldifeat/python/csrc/CMakeLists.txt b/kaldifeat/python/csrc/CMakeLists.txt index d470522..bd1ef25 100644 --- a/kaldifeat/python/csrc/CMakeLists.txt +++ b/kaldifeat/python/csrc/CMakeLists.txt @@ -9,4 +9,7 @@ pybind11_add_module(_kaldifeat mel-computations.cc ) target_link_libraries(_kaldifeat PRIVATE kaldifeat_core) -target_link_libraries(_kaldifeat PRIVATE ${TORCH_DIR}/lib/libtorch_python.so) +if(UNIX AND NOT APPLE) + target_link_libraries(_kaldifeat PUBLIC ${TORCH_DIR}/lib/libtorch_python.so) + target_link_libraries(_kaldifeat PUBLIC ${PYTHON_LIBRARY}) +endif() diff --git a/kaldifeat/python/tests/CMakeLists.txt b/kaldifeat/python/tests/CMakeLists.txt new file mode 100644 index 0000000..9961b14 --- /dev/null +++ b/kaldifeat/python/tests/CMakeLists.txt @@ -0,0 +1,29 @@ +function(kaldifeat_add_py_test source) + get_filename_component(name ${source} NAME_WE) + set(name "${name}_py") + + add_test(NAME ${name} + COMMAND + "${PYTHON_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/${source}" + ) + + get_filename_component(kaldifeat_path ${CMAKE_CURRENT_LIST_DIR} DIRECTORY) + + set_property(TEST ${name} + PROPERTY ENVIRONMENT "PYTHONPATH=${kaldifeat_path}:$:$ENV{PYTHONPATH}" + ) +endfunction() + +# please sort the files in alphabetic order +set(py_test_files + test_fbank.py + test_mfcc.py + test_plp.py + test_spectrogram.py + test_options.py +) + +foreach(source IN LISTS py_test_files) + kaldifeat_add_py_test(${source}) +endforeach()