Support macOS. (#7)

* Support macOS. * Remove extra files. * Minor fixes. * Minor fixes. * fix an error. * Fix building pip packages on macOS. * Fix test errors. * Release v1.4. * Fix test errors.
csukuangfj · Jul 19, 2021 · 7b2702d · 7b2702d
1 parent d27b45b
commit 7b2702d
Show file tree

Hide file tree

Showing 13 changed files with 251 additions and 30 deletions.
diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml
@@ -27,9 +27,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-18.04]
+        os: [ubuntu-18.04, macos-10.15]
         cuda: ["10.1"]
-        gcc: ["5"]
         torch: ["1.8.1"]
         python-version: [3.6, 3.7, 3.8]
     steps:
@@ -42,7 +41,16 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Install GCC 7
+        if: startsWith(matrix.os, 'ubuntu')
+        run: |
+          sudo apt-get install -y gcc-7 g++-7
+          echo "CC=/usr/bin/gcc-7" >> $GITHUB_ENV
+          echo "CXX=/usr/bin/g++-7" >> $GITHUB_ENV
+
+
       - name: Install CUDA Toolkit ${{ matrix.cuda }}
+        if: startsWith(matrix.os, 'ubuntu')
         shell: bash
         env:
           cuda: ${{ matrix.cuda }}
@@ -53,18 +61,13 @@ jobs:
           echo "LD_LIBRARY_PATH=${CUDA_HOME}/lib:${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
 
       - name: Display NVCC version
+        if: startsWith(matrix.os, 'ubuntu')
         run: |
           which nvcc
           nvcc --version
 
-      - name: Install GCC ${{ matrix.gcc }}
-        run: |
-          sudo apt-get install -y gcc-${{ matrix.gcc }} g++-${{ matrix.gcc }}
-          echo "CC=/usr/bin/gcc-${{ matrix.gcc }}" >> $GITHUB_ENV
-          echo "CXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV
-          echo "CUDAHOSTCXX=/usr/bin/g++-${{ matrix.gcc }}" >> $GITHUB_ENV
-
       - name: Install PyTorch ${{ matrix.torch }}
+        if: startsWith(matrix.os, 'ubuntu')
         env:
           cuda: ${{ matrix.cuda }}
           torch: ${{ matrix.torch }}
@@ -77,7 +80,18 @@ jobs:
           ./scripts/github_actions/install_torch.sh
           python3 -c "import torch; print('torch version:', torch.__version__)"
 
+      - name: Install PyTorch ${{ matrix.torch }}
+        if: startsWith(matrix.os, 'macos')
+        shell: bash
+        run: |
+          python3 -m pip install -qq --upgrade pip
+          python3 -m pip install -qq wheel twine typing_extensions
+          python3 -m pip install -qq torch==${{ matrix.torch }}
+
+          python3 -c "import torch; print('torch version:', torch.__version__)"
+
       - name: Download cudnn 8.0
+        if: startsWith(matrix.os, 'ubuntu')
         env:
           cuda: ${{ matrix.cuda }}
         run: |

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
@@ -0,0 +1,117 @@
+# Copyright      2021  Xiaomi Corp.       (author: Fangjun Kuang)
+
+# See ../../LICENSE for clarification regarding multiple authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name: Run tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  run_tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-18.04, macos-10.15]
+        cuda: ["10.1"]
+        torch: ["1.8.1"]
+        python-version: [3.6, 3.7, 3.8]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install GCC 7
+        if: startsWith(matrix.os, 'ubuntu')
+        run: |
+          sudo apt-get install -y gcc-7 g++-7
+          echo "CC=/usr/bin/gcc-7" >> $GITHUB_ENV
+          echo "CXX=/usr/bin/g++-7" >> $GITHUB_ENV
+
+
+      - name: Install CUDA Toolkit ${{ matrix.cuda }}
+        if: startsWith(matrix.os, 'ubuntu')
+        shell: bash
+        env:
+          cuda: ${{ matrix.cuda }}
+        run: |
+          source ./scripts/github_actions/install_cuda.sh
+          echo "CUDA_HOME=${CUDA_HOME}" >> $GITHUB_ENV
+          echo "${CUDA_HOME}/bin" >> $GITHUB_PATH
+          echo "LD_LIBRARY_PATH=${CUDA_HOME}/lib:${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+
+      - name: Display NVCC version
+        if: startsWith(matrix.os, 'ubuntu')
+        run: |
+          which nvcc
+          nvcc --version
+
+      - name: Install PyTorch ${{ matrix.torch }}
+        if: startsWith(matrix.os, 'ubuntu')
+        env:
+          cuda: ${{ matrix.cuda }}
+          torch: ${{ matrix.torch }}
+        shell: bash
+        run: |
+          sudo apt update
+          sudo apt install libsndfile1-dev libsndfile1 ffmpeg
+          python3 -m pip install --upgrade pip
+          python3 -m pip install wheel twine typing_extensions soundfile
+          python3 -m pip install bs4 requests tqdm
+
+          ./scripts/github_actions/install_torch.sh
+          python3 -c "import torch; print('torch version:', torch.__version__)"
+
+      - name: Install PyTorch ${{ matrix.torch }}
+        if: startsWith(matrix.os, 'macos')
+        shell: bash
+        run: |
+          python3 -m pip install -qq --upgrade pip
+          python3 -m pip install -qq wheel twine typing_extensions soundfile
+          python3 -m pip install -qq torch==${{ matrix.torch }}
+
+          python3 -c "import torch; print('torch version:', torch.__version__)"
+
+      - name: Download cudnn 8.0
+        if: startsWith(matrix.os, 'ubuntu')
+        env:
+          cuda: ${{ matrix.cuda }}
+        run: |
+          ./scripts/github_actions/install_cudnn.sh
+
+      - name: Build
+        shell: bash
+        run: |
+          mkdir build_release
+          cd build_release
+          cmake ..
+          make VERBOSE=1
+
+      - name: Run tests
+        shell: bash
+        run: |
+          cd build_release
+          ctest -R py --output-on-failure
diff --git a/.github/workflows/style_check.yml b/.github/workflows/style_check.yml
@@ -0,0 +1,62 @@
+# Copyright      2021  Fangjun Kuang ([email protected])
+
+# See ../../LICENSE for clarification regarding multiple authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: style_check
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  style_check:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-18.04, macos-10.15]
+        python-version: [3.7, 3.9]
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Python dependencies
+        run: |
+          python3 -m pip install --upgrade pip black flake8
+
+      - name: Run flake8
+        shell: bash
+        working-directory: ${{github.workspace}}
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --show-source --statistics
+          flake8 .
+
+      - name: Run black
+        shell: bash
+        working-directory: ${{github.workspace}}
+        run: |
+          black --check --diff .
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -4,7 +4,7 @@ cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
 
 project(kaldifeat)
 
-set(kaldifeat_VERSION "1.3")
+set(kaldifeat_VERSION "1.4")
 
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")

diff --git a/README.md b/README.md
@@ -1,5 +1,10 @@
 # kaldifeat
 
+<div align="center">
+<img src="/doc/source/images/os.svg">
+</div>
+
+
 <sub>
 <table>
 <tr>

diff --git a/cmake/cmake_extension.py b/cmake/cmake_extension.py
@@ -2,6 +2,7 @@
 
 import glob
 import os
+import platform
 import shutil
 import sys
 from pathlib import Path
@@ -15,6 +16,10 @@ def is_for_pypi():
     return ans is not None
 
 
+def is_macos():
+    return platform.system() == "Darwin"
+
+
 try:
     from wheel.bdist_wheel import bdist_wheel as _bdist_wheel
 
@@ -23,7 +28,7 @@ def finalize_options(self):
             _bdist_wheel.finalize_options(self)
             # In this case, the generated wheel has a name in the form
             # k2-xxx-pyxx-none-any.whl
-            if is_for_pypi():
+            if is_for_pypi() and not is_macos():
                 self.root_is_pure = True
             else:
                 # The generated wheel has a name ending with
@@ -85,6 +90,7 @@ def build_extension(self, ext: setuptools.extension.Extension):
             )
 
         lib_so = glob.glob(f"{self.build_temp}/lib/*kaldifeat*.so")
+        lib_so += glob.glob(f"{self.build_temp}/lib/*kaldifeat*.dylib")  # macOS
         for so in lib_so:
             print(f"Copying {so} to {self.build_lib}/")
             shutil.copy(f"{so}", f"{self.build_lib}/")
diff --git a/cmake/pybind11.cmake b/cmake/pybind11.cmake
@@ -17,9 +17,6 @@ function(download_pybind11)
   FetchContent_Declare(pybind11
     URL               ${pybind11_URL}
     URL_HASH          ${pybind11_HASH}
-    PATCH_COMMAND
-      sed -i s/\\${double_quotes}-flto\\\\${dollar}/\\${double_quotes}-Xcompiler=-flto${dollar}/g "tools/pybind11Tools.cmake" &&
-      sed -i s/${seimcolon}-fno-fat-lto-objects/${seimcolon}-Xcompiler=-fno-fat-lto-objects/g "tools/pybind11Tools.cmake"
   )
 
   FetchContent_GetProperties(pybind11)

diff --git a/cmake/torch.cmake b/cmake/torch.cmake
@@ -23,17 +23,3 @@ execute_process(
 )
 
 message(STATUS "PyTorch version: ${TORCH_VERSION}")
-
-# Solve the following error for NVCC:
-#   unknown option `-Wall`
-#
-# It contains only some -Wno-* flags, so it is OK
-# to set them to empty
-set_property(TARGET torch_cuda
-  PROPERTY
-    INTERFACE_COMPILE_OPTIONS ""
-)
-set_property(TARGET torch_cpu
-  PROPERTY
-    INTERFACE_COMPILE_OPTIONS ""
-)
diff --git a/doc/source/images/os.svg b/doc/source/images/os.svg
diff --git a/kaldifeat/csrc/feature-window.cc b/kaldifeat/csrc/feature-window.cc
@@ -157,7 +157,7 @@ torch::Tensor GetStrided(const torch::Tensor &wave,
 }
 
 torch::Tensor Dither(const torch::Tensor &wave, float dither_value) {
-  if (dither_value == 0.0f) wave;
+  if (dither_value == 0.0f) return wave;
 
   torch::Tensor rand_gauss = torch::randn_like(wave);
 #if 1

diff --git a/kaldifeat/python/CMakeLists.txt b/kaldifeat/python/CMakeLists.txt
@@ -1 +1,2 @@
 add_subdirectory(csrc)
+add_subdirectory(tests)
diff --git a/kaldifeat/python/csrc/CMakeLists.txt b/kaldifeat/python/csrc/CMakeLists.txt
@@ -9,4 +9,7 @@ pybind11_add_module(_kaldifeat
   mel-computations.cc
 )
 target_link_libraries(_kaldifeat PRIVATE kaldifeat_core)
-target_link_libraries(_kaldifeat PRIVATE ${TORCH_DIR}/lib/libtorch_python.so)
+if(UNIX AND NOT APPLE)
+  target_link_libraries(_kaldifeat PUBLIC ${TORCH_DIR}/lib/libtorch_python.so)
+  target_link_libraries(_kaldifeat PUBLIC ${PYTHON_LIBRARY})
+endif()
diff --git a/kaldifeat/python/tests/CMakeLists.txt b/kaldifeat/python/tests/CMakeLists.txt
@@ -0,0 +1,29 @@
+function(kaldifeat_add_py_test source)
+  get_filename_component(name ${source} NAME_WE)
+  set(name "${name}_py")
+
+  add_test(NAME ${name}
+    COMMAND
+      "${PYTHON_EXECUTABLE}"
+      "${CMAKE_CURRENT_SOURCE_DIR}/${source}"
+  )
+
+  get_filename_component(kaldifeat_path ${CMAKE_CURRENT_LIST_DIR} DIRECTORY)
+
+  set_property(TEST ${name}
+    PROPERTY ENVIRONMENT "PYTHONPATH=${kaldifeat_path}:$<TARGET_FILE_DIR:_kaldifeat>:$ENV{PYTHONPATH}"
+  )
+endfunction()
+
+# please sort the files in alphabetic order
+set(py_test_files
+  test_fbank.py
+  test_mfcc.py
+  test_plp.py
+  test_spectrogram.py
+  test_options.py
+)
+
+foreach(source IN LISTS py_test_files)
+  kaldifeat_add_py_test(${source})
+endforeach()