-
Notifications
You must be signed in to change notification settings - Fork 3.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Re-add example of how to build, link, and test an external SWIG modul…
…e"" (#3981) Summary: Pull Request resolved: #3981 prior issues: 1. Nightlies were breaking due to yaml changes. Those are reverted. 2. AIX build broke (external). Fix is to add a conditional in tests/CMakeLists.txt. 3. Build issue #3944. Could not repro now. Reviewed By: mdouze Differential Revision: D64440629 fbshipit-source-id: a86b27c25ada0d07e9d3b4c6e4f00b2e6b637fbe
- Loading branch information
1 parent
ebe5a69
commit 9c3cd77
Showing
6 changed files
with
314 additions
and
27 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,141 @@ | ||
| /** | ||
| * Copyright (c) Meta Platforms, Inc. and its affiliates. | ||
| * | ||
| * This source code is licensed under the MIT license found in the | ||
| * LICENSE file in the root directory of this source tree. | ||
| */ | ||
|
|
||
| // This is an example of how an external module can be added via SWIG. | ||
|
|
||
| %module faiss_example_external_module; | ||
|
|
||
|
|
||
| // Put C++ includes here | ||
| %{ | ||
|
|
||
| #include <faiss/impl/FaissException.h> | ||
| #include <faiss/impl/IDSelector.h> | ||
|
|
||
| %} | ||
|
|
||
| #pragma SWIG nowarn=322 | ||
|
|
||
| typedef unsigned char uint8_t; | ||
| typedef unsigned short uint16_t; | ||
| typedef unsigned int uint32_t; | ||
|
|
||
| typedef signed char int8_t; | ||
| typedef short int16_t; | ||
| typedef int int32_t; | ||
|
|
||
| #ifdef SWIGWORDSIZE64 | ||
| typedef unsigned long uint64_t; | ||
| typedef long int64_t; | ||
| #else | ||
| typedef unsigned long long uint64_t; | ||
| typedef long long int64_t; | ||
| #endif | ||
|
|
||
| typedef uint64_t size_t; | ||
|
|
||
| // This means: assume what's declared in these .h files is provided | ||
| // by the Faiss module. | ||
| %import(module="faiss") "faiss/MetricType.h" | ||
| %import(module="faiss") "faiss/impl/IDSelector.h" | ||
|
|
||
| // functions to be parsed here | ||
|
|
||
| // This is important to release GIL and do Faiss exception handing | ||
| %exception { | ||
| Py_BEGIN_ALLOW_THREADS | ||
| try { | ||
| $action | ||
| } catch(faiss::FaissException & e) { | ||
| PyEval_RestoreThread(_save); | ||
|
|
||
| if (PyErr_Occurred()) { | ||
| // some previous code already set the error type. | ||
| } else { | ||
| PyErr_SetString(PyExc_RuntimeError, e.what()); | ||
| } | ||
| SWIG_fail; | ||
| } catch(std::bad_alloc & ba) { | ||
| PyEval_RestoreThread(_save); | ||
| PyErr_SetString(PyExc_MemoryError, "std::bad_alloc"); | ||
| SWIG_fail; | ||
| } | ||
| Py_END_ALLOW_THREADS | ||
| } | ||
|
|
||
|
|
||
| // any class or function declared below will be made available | ||
| // in the module. | ||
| %inline %{ | ||
|
|
||
| struct IDSelectorModulo : faiss::IDSelector { | ||
| int mod; | ||
|
|
||
| IDSelectorModulo(int mod): mod(mod) {} | ||
|
|
||
| bool is_member(faiss::idx_t id) const { | ||
| return id % mod == 0; | ||
| } | ||
|
|
||
| ~IDSelectorModulo() override {} | ||
| }; | ||
|
|
||
| faiss::idx_t sum_of_idx(size_t n, const faiss::idx_t *tab) { | ||
| faiss::idx_t sum = 0; | ||
| for(size_t i = 0; i < n; i++) { | ||
| sum += tab[i]; | ||
| } | ||
| return sum; | ||
| } | ||
|
|
||
| float sum_of_float32(size_t n, const float *tab) { | ||
| float sum = 0; | ||
| for(size_t i = 0; i < n; i++) { | ||
| sum += tab[i]; | ||
| } | ||
| return sum; | ||
| } | ||
|
|
||
| double sum_of_float64(size_t n, const double *tab) { | ||
| double sum = 0; | ||
| for(size_t i = 0; i < n; i++) { | ||
| sum += tab[i]; | ||
| } | ||
| return sum; | ||
| } | ||
|
|
||
| %} | ||
|
|
||
| /********************************************** | ||
| * To test if passing a swig_ptr on all array types works | ||
| **********************************************/ | ||
|
|
||
| %define SUM_OF_TYPE(ty) | ||
|
|
||
| %inline %{ | ||
|
|
||
| ty##_t sum_of_##ty (size_t n, const ty##_t * tab) { | ||
| ty##_t sum = 0; | ||
| for(size_t i = 0; i < n; i++) { | ||
| sum += tab[i]; | ||
| } | ||
| return sum; | ||
| } | ||
|
|
||
| %} | ||
|
|
||
| %enddef | ||
|
|
||
| SUM_OF_TYPE(uint8); | ||
| SUM_OF_TYPE(uint16); | ||
| SUM_OF_TYPE(uint32); | ||
| SUM_OF_TYPE(uint64); | ||
|
|
||
| SUM_OF_TYPE(int8); | ||
| SUM_OF_TYPE(int16); | ||
| SUM_OF_TYPE(int32); | ||
| SUM_OF_TYPE(int64); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,10 +4,12 @@ | |
| # LICENSE file in the root directory of this source tree. | ||
|
|
||
| from __future__ import print_function | ||
| from setuptools import setup, find_packages | ||
|
|
||
| import os | ||
| import shutil | ||
| import platform | ||
| import shutil | ||
|
|
||
| from setuptools import find_packages, setup | ||
|
|
||
| # make the faiss python package dir | ||
| shutil.rmtree("faiss", ignore_errors=True) | ||
|
|
@@ -20,25 +22,32 @@ | |
| shutil.copyfile("extra_wrappers.py", "faiss/extra_wrappers.py") | ||
| shutil.copyfile("array_conversions.py", "faiss/array_conversions.py") | ||
|
|
||
| ext = ".pyd" if platform.system() == 'Windows' else ".so" | ||
| prefix = "Release/" * (platform.system() == 'Windows') | ||
| ext = ".pyd" if platform.system() == "Windows" else ".so" | ||
| prefix = "Release/" * (platform.system() == "Windows") | ||
|
|
||
| swigfaiss_generic_lib = f"{prefix}_swigfaiss{ext}" | ||
| swigfaiss_avx2_lib = f"{prefix}_swigfaiss_avx2{ext}" | ||
| swigfaiss_avx512_lib = f"{prefix}_swigfaiss_avx512{ext}" | ||
| callbacks_lib = f"{prefix}libfaiss_python_callbacks{ext}" | ||
| swigfaiss_sve_lib = f"{prefix}_swigfaiss_sve{ext}" | ||
| faiss_example_external_module_lib = f"_faiss_example_external_module{ext}" | ||
|
|
||
| found_swigfaiss_generic = os.path.exists(swigfaiss_generic_lib) | ||
| found_swigfaiss_avx2 = os.path.exists(swigfaiss_avx2_lib) | ||
| found_swigfaiss_avx512 = os.path.exists(swigfaiss_avx512_lib) | ||
| found_callbacks = os.path.exists(callbacks_lib) | ||
| found_swigfaiss_sve = os.path.exists(swigfaiss_sve_lib) | ||
| found_faiss_example_external_module_lib = os.path.exists( | ||
| faiss_example_external_module_lib | ||
| ) | ||
|
|
||
| assert (found_swigfaiss_generic or found_swigfaiss_avx2 or found_swigfaiss_avx512 or found_swigfaiss_sve), \ | ||
| f"Could not find {swigfaiss_generic_lib} or " \ | ||
| f"{swigfaiss_avx2_lib} or {swigfaiss_avx512_lib} or {swigfaiss_sve_lib}. " \ | ||
| assert ( | ||
| found_swigfaiss_generic or found_swigfaiss_avx2 or found_swigfaiss_avx512 or found_swigfaiss_sve or found_faiss_example_external_module_lib | ||
| ), ( | ||
| f"Could not find {swigfaiss_generic_lib} or " | ||
| f"{swigfaiss_avx2_lib} or {swigfaiss_avx512_lib} or {swigfaiss_sve_lib} or {faiss_example_external_module_lib}. " | ||
| f"Faiss may not be compiled yet." | ||
| ) | ||
|
|
||
| if found_swigfaiss_generic: | ||
| print(f"Copying {swigfaiss_generic_lib}") | ||
|
|
@@ -64,7 +73,17 @@ | |
| shutil.copyfile("swigfaiss_sve.py", "faiss/swigfaiss_sve.py") | ||
| shutil.copyfile(swigfaiss_sve_lib, f"faiss/_swigfaiss_sve{ext}") | ||
|
|
||
| long_description=""" | ||
| if found_faiss_example_external_module_lib: | ||
| print(f"Copying {faiss_example_external_module_lib}") | ||
| shutil.copyfile( | ||
| "faiss_example_external_module.py", "faiss/faiss_example_external_module.py" | ||
| ) | ||
| shutil.copyfile( | ||
| faiss_example_external_module_lib, | ||
| f"faiss/_faiss_example_external_module{ext}", | ||
| ) | ||
|
|
||
| long_description = """ | ||
| Faiss is a library for efficient similarity search and clustering of dense | ||
| vectors. It contains algorithms that search in sets of vectors of any size, | ||
| up to ones that possibly do not fit in RAM. It also contains supporting | ||
|
|
@@ -73,20 +92,19 @@ | |
| are implemented on the GPU. It is developed by Facebook AI Research. | ||
| """ | ||
| setup( | ||
| name='faiss', | ||
| version='1.9.0', | ||
| description='A library for efficient similarity search and clustering of dense vectors', | ||
| name="faiss", | ||
| version="1.9.0", | ||
| description="A library for efficient similarity search and clustering of dense vectors", | ||
| long_description=long_description, | ||
| url='https://github.com/facebookresearch/faiss', | ||
| author='Matthijs Douze, Jeff Johnson, Herve Jegou, Lucas Hosseini', | ||
| author_email='[email protected]', | ||
| license='MIT', | ||
| keywords='search nearest neighbors', | ||
|
|
||
| install_requires=['numpy', 'packaging'], | ||
| packages=['faiss', 'faiss.contrib', 'faiss.contrib.torch'], | ||
| url="https://github.com/facebookresearch/faiss", | ||
| author="Matthijs Douze, Jeff Johnson, Herve Jegou, Lucas Hosseini", | ||
| author_email="[email protected]", | ||
| license="MIT", | ||
| keywords="search nearest neighbors", | ||
| install_requires=["numpy", "packaging"], | ||
| packages=["faiss", "faiss.contrib", "faiss.contrib.torch"], | ||
| package_data={ | ||
| 'faiss': ['*.so', '*.pyd'], | ||
| "faiss": ["*.so", "*.pyd"], | ||
| }, | ||
| zip_safe=False, | ||
| ) | ||
Oops, something went wrong.