From 2e9bc3e7eb418c1bce6108396d54cffbb64310de Mon Sep 17 00:00:00 2001 From: Chris Marslender Date: Thu, 7 Mar 2024 15:26:21 -0600 Subject: [PATCH] Add c bindings + Updated camke builds (#166) * Add c bindings for create_discriminant, prove, and verify_n_wesolowski * Add chiavdfc shared library target * Don't build the c wrapper library by default (avoid it in the wheel) + add workflow for the c libs * no build vdf client * mpir for windows + unique artifact names per platform * Fix checkout * remove .git * Fix mpir path * Include mpir on windows * mpir before forcing gmp * Fix path to mpir in cmake * Add a windows specific prefix path to help find gmp and gmpxx * Split mpir/gmp(xx) for windows/non-windows * Try cmake --build * Missed . * -pthread flag doesn't apply to windows * Fix windows compile warnings * Update path for dynamic dll * Cleanup setup.py to build windows via cmake, similar to the chiapos cleanup * remove unused import --- .github/workflows/build-c-libraries.yml | 92 ++++++++++++++ .gitignore | 5 + setup.py | 154 +++--------------------- src/CMakeLists.txt | 74 ++++++++++-- src/c_bindings/c_wrapper.cpp | 46 +++++++ src/c_bindings/c_wrapper.h | 23 ++++ src/c_bindings/readme.md | 4 + 7 files changed, 253 insertions(+), 145 deletions(-) create mode 100644 .github/workflows/build-c-libraries.yml create mode 100644 src/c_bindings/c_wrapper.cpp create mode 100644 src/c_bindings/c_wrapper.h create mode 100644 src/c_bindings/readme.md diff --git a/.github/workflows/build-c-libraries.yml b/.github/workflows/build-c-libraries.yml new file mode 100644 index 00000000..3ca1f443 --- /dev/null +++ b/.github/workflows/build-c-libraries.yml @@ -0,0 +1,92 @@ +name: Build C Libraries + +on: + push: + branches: + - main + release: + types: [published] + pull_request: + branches: + - '**' + +concurrency: + # SHA is added to the end if on `main` to let all main workflows run + group: ${{ github.ref }}-${{ github.workflow }}-${{ github.event_name }}-${{ (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/') || startsWith(github.ref, 'refs/heads/long_lived/')) && github.sha || '' }} + cancel-in-progress: true + +permissions: + contents: read + id-token: write + +jobs: + build-c-libraries: + name: C Libraries - ${{ matrix.os.name }} ${{ matrix.arch.name }} + runs-on: ${{ matrix.os.runs-on[matrix.arch.matrix] }} + strategy: + fail-fast: false + matrix: + os: + - name: macOS + matrix: macos + runs-on: + arm: [macOS, ARM64] + intel: [macos-11] + - name: Ubuntu + matrix: ubuntu + runs-on: + arm: [Linux, ARM64] + intel: [ubuntu-latest] + - name: Windows + matrix: windows + runs-on: + intel: [windows-latest] + arch: + - name: ARM + matrix: arm + - name: Intel + matrix: intel + exclude: + # Only partial entries are required here by GitHub Actions so generally I + # only specify the `matrix:` entry. The super linter complains so for now + # all entries are included to avoid that. Reported at + # https://github.com/github/super-linter/issues/3016 + - os: + name: Windows + matrix: windows + runs-on: + intel: [windows-latest] + arch: + name: ARM + matrix: arm + + steps: + - name: Clean workspace + uses: Chia-Network/actions/clean-workspace@main + + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Checkout mpir for windows + if: matrix.os.matrix == 'windows' + uses: actions/checkout@v4 + with: + repository: Chia-Network/mpir_gc_x64 + fetch-depth: 1 + path: mpir_gc_x64 + + - name: Build + working-directory: src + env: + BUILD_VDF_CLIENT: "N" + run: | + cmake . -DBUILD_CHIAVDFC=ON + cmake --build . + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: c-libraries-${{ matrix.os.matrix }}-${{ matrix.arch.matrix }} + path: ./src/lib diff --git a/.gitignore b/.gitignore index 77796548..63471188 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,11 @@ __pycache__/ *.py[cod] *$py.class +*.so +*.dylib +*.dll +*.a +src/verifier_test # Generated assembly file /asm_compiled.s diff --git a/setup.py b/setup.py index 47a68a35..8604fff1 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ import subprocess import sys -from setuptools import Command, Extension, setup, errors +from setuptools import Command, Extension, setup from setuptools.command.build import build from setuptools.command.build_ext import build_ext from setuptools.command.install import install @@ -116,7 +116,7 @@ def build_extension(self, ext): cmake_args += [ "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}".format(cfg.upper(), extdir) ] - if sys.maxsize > 2 ** 32: + if sys.maxsize > 2**32: cmake_args += ["-A", "x64"] build_args += ["--", "/m"] else: @@ -131,136 +131,22 @@ def build_extension(self, ext): subprocess.check_call(["cmake", "--build", "."] + build_args) -class get_pybind_include(object): - """Helper class to determine the pybind11 include path - - The purpose of this class is to postpone importing pybind11 - until it is actually installed, so that the ``get_include()`` - method can be invoked.""" - - def __init__(self, user=False): - self.user = user - - def __str__(self): - import pybind11 - - return pybind11.get_include(self.user) - - -ext_modules = [ - Extension( - "chiavdf", - sorted( - [ - "src/python_bindings/fastvdf.cpp", - "src/refcode/lzcnt.c", - ] - ), - include_dirs=[ - # Path to pybind11 headers - get_pybind_include(), - get_pybind_include(user=True), - "mpir_gc_x64", - ], - library_dirs=["mpir_gc_x64"], - libraries=["mpir"], - language="c++", +build.sub_commands.append(("build_hook", lambda x: True)) # type: ignore +install.sub_commands.append(("install_hook", lambda x: True)) + +setup( + name="chiavdf", + author="Florin Chirica", + author_email="florin@chia.net", + description="Chia vdf verification (wraps C++)", + license="Apache License", + python_requires=">=3.8", + long_description=open("README.md").read(), + long_description_content_type="text/markdown", + url="https://github.com/Chia-Network/chiavdf", + ext_modules=[CMakeExtension("chiavdf", "src")], + cmdclass=dict( + build_ext=CMakeBuild, install_hook=install_hook, build_hook=build_hook ), -] - - -# As of Python 3.6, CCompiler has a `has_flag` method. -# cf http://bugs.python.org/issue26689 -def has_flag(compiler, flagname): - """Return a boolean indicating whether a flag name is supported on - the specified compiler. - """ - import tempfile - - with tempfile.NamedTemporaryFile("w", suffix=".cpp") as f: - f.write("int main (int argc, char **argv) { return 0; }") - try: - compiler.compile([f.name], extra_postargs=[flagname]) - except errors.CompileError: - return False - return True - - -def cpp_flag(compiler): - """Return the -std=c++[11/14/17] compiler flag. - - The newer version is prefered over c++11 (when it is available). - """ - flags = ["-std=c++17", "-std=c++14", "-std=c++11"] - - for flag in flags: - if has_flag(compiler, flag): - return flag - - raise RuntimeError("Unsupported compiler -- at least C++11 support " "is needed!") - - -class BuildExt(build_ext): - """A custom build extension for adding compiler-specific options.""" - - c_opts = { - "msvc": ["/EHsc", "/std:c++17"], - "unix": [""], - } - l_opts = { - "msvc": [], - "unix": [""], - } - - def build_extensions(self): - ct = self.compiler.compiler_type - opts = self.c_opts.get(ct, []) - link_opts = self.l_opts.get(ct, []) - if ct == "unix": - opts.append('-DVERSION_INFO="%s"' % self.distribution.get_version()) - opts.append(cpp_flag(self.compiler)) - if has_flag(self.compiler, "-fvisibility=hidden"): - opts.append("-fvisibility=hidden") - elif ct == "msvc": - opts.append('/DVERSION_INFO=\\"%s\\"' % self.distribution.get_version()) - for ext in self.extensions: - ext.extra_compile_args = opts - ext.extra_link_args = link_opts - build_ext.build_extensions(self) - - -if platform.system() == "Windows": - setup( - name="chiavdf", - author="Mariano Sorgente", - author_email="mariano@chia.net", - description="Chia vdf verification (wraps C++)", - license="Apache License", - python_requires=">=3.8", - long_description=open("README.md").read(), - long_description_content_type="text/markdown", - url="https://github.com/Chia-Network/chiavdf", - ext_modules=ext_modules, - cmdclass={"build_ext": BuildExt}, - zip_safe=False, - ) -else: - build.sub_commands.append(("build_hook", lambda x: True)) # type: ignore - install.sub_commands.append(("install_hook", lambda x: True)) - - setup( - name="chiavdf", - author="Florin Chirica", - author_email="florin@chia.net", - description="Chia vdf verification (wraps C++)", - license="Apache License", - python_requires=">=3.8", - long_description=open("README.md").read(), - long_description_content_type="text/markdown", - url="https://github.com/Chia-Network/chiavdf", - ext_modules=[CMakeExtension("chiavdf", "src")], - cmdclass=dict( - build_ext=CMakeBuild, install_hook=install_hook, build_hook=build_hook - ), - zip_safe=False, - ) + zip_safe=False, +) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 95c51bd2..2ac31123 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,6 +1,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.14 FATAL_ERROR) +option(BUILD_CHIAVDFC "Build the chiavdfc shared library" OFF) set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_POSITION_INDEPENDENT_CODE ON) IF(NOT CMAKE_BUILD_TYPE) @@ -14,17 +16,38 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ) -find_package(GMP REQUIRED) -find_package(GMPXX REQUIRED) +if(MSVC) + add_compile_options(/EHsc) +endif() -include_directories( - ${INCLUDE_DIRECTORIES} - ${CMAKE_CURRENT_SOURCE_DIR} - ${GMP_INCLUDE_DIR} - ${GMPXX_INCLUDE_DIR} -) +if(WIN32) + set(MPIR_LIBRARY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../mpir_gc_x64") + set(MPIR_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../mpir_gc_x64") + include_directories( + ${INCLUDE_DIRECTORIES} + ${CMAKE_CURRENT_SOURCE_DIR} + ${MPIR_INCLUDE_DIR} + ) + find_library(MPIR_LIBRARY NAMES mpir PATHS ${MPIR_LIBRARY_DIR} NO_DEFAULT_PATH) + if(MPIR_LIBRARY) + message(STATUS "MPIR library found at ${MPIR_LIBRARY}") + link_libraries(${MPIR_LIBRARY}) + else() + message(FATAL_ERROR "MPIR library not found") + endif() + + list(APPEND CMAKE_PREFIX_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../mpir_gc_x64") +else() + find_package(GMP REQUIRED) + find_package(GMPXX REQUIRED) -set (CMAKE_CXX_FLAGS "-std=c++1z") + include_directories( + ${INCLUDE_DIRECTORIES} + ${CMAKE_CURRENT_SOURCE_DIR} + ${GMP_INCLUDE_DIR} + ${GMPXX_INCLUDE_DIR} + ) +endif() # CMake 3.14+ include(FetchContent) @@ -46,5 +69,34 @@ add_executable(verifier_test ${CMAKE_CURRENT_SOURCE_DIR}/refcode/lzcnt.c ) -target_link_libraries(chiavdf PRIVATE ${GMP_LIBRARIES} ${GMPXX_LIBRARIES} -pthread) -target_link_libraries(verifier_test ${GMP_LIBRARIES} ${GMPXX_LIBRARIES} -pthread) +target_link_libraries(chiavdf PRIVATE ${GMP_LIBRARIES} ${GMPXX_LIBRARIES}) +target_link_libraries(verifier_test PRIVATE ${GMP_LIBRARIES} ${GMPXX_LIBRARIES}) + +if(UNIX) + target_link_libraries(chiavdf PRIVATE -pthread) + target_link_libraries(verifier_test PRIVATE -pthread) +endif() + +if(BUILD_CHIAVDFC) + add_library(chiavdfc_shared SHARED + ${CMAKE_CURRENT_SOURCE_DIR}/c_bindings/c_wrapper.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/refcode/lzcnt.c + ) + add_library(chiavdfc_static STATIC + ${CMAKE_CURRENT_SOURCE_DIR}/c_bindings/c_wrapper.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/refcode/lzcnt.c + ) + target_link_libraries(chiavdfc_shared ${GMP_LIBRARIES} ${GMPXX_LIBRARIES}) + target_link_libraries(chiavdfc_static ${GMP_LIBRARIES} ${GMPXX_LIBRARIES}) + + set_target_properties(chiavdfc_shared PROPERTIES + OUTPUT_NAME chiavdfc + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib/shared" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib/shared" + ) + + set_target_properties(chiavdfc_static PROPERTIES + OUTPUT_NAME chiavdfc + ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib/static" + ) +endif() diff --git a/src/c_bindings/c_wrapper.cpp b/src/c_bindings/c_wrapper.cpp new file mode 100644 index 00000000..b6d8588a --- /dev/null +++ b/src/c_bindings/c_wrapper.cpp @@ -0,0 +1,46 @@ +#include "c_wrapper.h" +#include +#include +#include "../verifier.h" +#include "../prover_slow.h" + +extern "C" { + // C wrapper function + const char* create_discriminant_wrapper(const uint8_t* seed, size_t seed_size, int length) { + std::vector seedVector(seed, seed + seed_size); + integer result = CreateDiscriminant(seedVector, length); + + // Serialize the 'result' to a string + std::string resultStr = result.to_string(); + + // Allocate a new C-string to hold the serialized result + char* cResultStr = new char[resultStr.length() + 1]; + std::strcpy(cResultStr, resultStr.c_str()); + + return cResultStr; // Return the C-string + } + + ByteArray prove_wrapper(const uint8_t* challenge_hash, size_t challenge_size, const uint8_t* x_s, size_t x_s_size, int discriminant_size_bits, uint64_t num_iterations) { + std::vector challenge_hash_bytes(challenge_hash, challenge_hash + challenge_size); + integer D = CreateDiscriminant(challenge_hash_bytes, discriminant_size_bits); + form x = DeserializeForm(D, (const uint8_t*)x_s, x_s_size); + std::vector result = ProveSlow(D, x, num_iterations); + + // Allocate memory for the result and copy data + uint8_t* resultData = new uint8_t[result.size()]; + std::copy(result.begin(), result.end(), resultData); + + // Create and return a ByteArray struct + ByteArray resultArray = { resultData, result.size() }; + return resultArray; + } + + int verify_n_wesolowski_wrapper(const char* discriminant_str, size_t discriminant_size, const char* x_s, size_t x_s_size, const char* proof_blob, size_t proof_blob_size, uint64_t num_iterations, uint64_t disc_size_bits, uint64_t recursion) { + std::vector x_s_v(x_s, x_s + x_s_size); + std::vector proof_blob_v(proof_blob, proof_blob + proof_blob_size); + + bool result = CheckProofOfTimeNWesolowski(integer(discriminant_str), x_s_v.data(), proof_blob_v.data(), proof_blob_v.size(), num_iterations, disc_size_bits, recursion); + + return result ? 1 : 0; + } +} diff --git a/src/c_bindings/c_wrapper.h b/src/c_bindings/c_wrapper.h new file mode 100644 index 00000000..e17bb192 --- /dev/null +++ b/src/c_bindings/c_wrapper.h @@ -0,0 +1,23 @@ +#pragma once +#include + +#ifdef __cplusplus +#include // for size_t +#include // for uint8_t +extern "C" { +#endif + +const char* create_discriminant_wrapper(const uint8_t* seed, size_t seed_size, int length); + +// Define a struct to hold the byte array and its length +typedef struct { + uint8_t* data; + size_t length; +} ByteArray; +ByteArray prove_wrapper(const uint8_t* challenge_hash, size_t challenge_size, const uint8_t* x_s, size_t x_s_size, int discriminant_size_bits, uint64_t num_iterations); + +int verify_n_wesolowski_wrapper(const char* discriminant_str, size_t discriminant_size, const char* x_s, size_t x_s_size, const char* proof_blob, size_t proof_blob_size, uint64_t num_iterations, uint64_t disc_size_bits, uint64_t recursion); + +#ifdef __cplusplus +} +#endif diff --git a/src/c_bindings/readme.md b/src/c_bindings/readme.md new file mode 100644 index 00000000..45d8b812 --- /dev/null +++ b/src/c_bindings/readme.md @@ -0,0 +1,4 @@ +# C Bindings + +C bindings here wrap a few of the C++ functions with purely c compatible types, so that other languages which require +c bindings (golang for example) can create bindings to the C++ functions.