From 0af3ba0250336db3e549cc03a2fce902f8397fe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Wed, 14 Nov 2018 16:00:28 -0700 Subject: [PATCH 001/158] Add lld recipe from conda-forge --- conda-recipes/lld/bld.bat | 18 +++++++++++++ conda-recipes/lld/build.sh | 13 ++++++++++ conda-recipes/lld/meta.yaml | 51 +++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 conda-recipes/lld/bld.bat create mode 100644 conda-recipes/lld/build.sh create mode 100644 conda-recipes/lld/meta.yaml diff --git a/conda-recipes/lld/bld.bat b/conda-recipes/lld/bld.bat new file mode 100644 index 000000000..e5708ee74 --- /dev/null +++ b/conda-recipes/lld/bld.bat @@ -0,0 +1,18 @@ +mkdir build +cd build + +cmake -G "Ninja" ^ + -DCMAKE_BUILD_TYPE="Release" ^ + -DCMAKE_PREFIX_PATH=%LIBRARY_PREFIX% ^ + -DCMAKE_INSTALL_PREFIX:PATH=%LIBRARY_PREFIX% ^ + -DCLANG_INCLUDE_TESTS=OFF ^ + -DCLANG_INCLUDE_DOCS=OFF ^ + -DLLVM_INCLUDE_TESTS=OFF ^ + -DLLVM_INCLUDE_DOCS=OFF ^ + -DLLVM_TARGETS_TO_BUILD=X86 ^ + %SRC_DIR% + +if errorlevel 1 exit 1 + +cmake --build . --target install +if errorlevel 1 exit 1 \ No newline at end of file diff --git a/conda-recipes/lld/build.sh b/conda-recipes/lld/build.sh new file mode 100644 index 000000000..57876c4ae --- /dev/null +++ b/conda-recipes/lld/build.sh @@ -0,0 +1,13 @@ +mkdir build +cd build + +cmake \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DCMAKE_PREFIX_PATH=$PREFIX \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_RTTI=ON \ + -DLLVM_INCLUDE_TESTS=OFF \ + .. + +make -j${CPU_COUNT} +make install diff --git a/conda-recipes/lld/meta.yaml b/conda-recipes/lld/meta.yaml new file mode 100644 index 000000000..8bb99f27e --- /dev/null +++ b/conda-recipes/lld/meta.yaml @@ -0,0 +1,51 @@ +{% set version = "7.0.0" %} +{% set sha256 = "fbcf47c5e543f4cdac6bb9bbbc6327ff24217cd7eafc5571549ad6d237287f9c" %} + +package: + name: lld + version: {{ version }} + +source: + fn: lld-{{ version }}.src.tar.xz + url: http://releases.llvm.org/{{ version }}/lld-{{ version }}.src.tar.xz + sha256: {{ sha256 }} + +build: + number: 0 + skip: true # [win and py<35] + +requirements: + build: + - cmake >=3.4.3 + - ninja # [win] + - {{ compiler('c') }} + - {{ compiler('cxx') }} + host: + - python + - llvmdev =={{ version }} + - libxml2 + - swig + - libcxx # [osx] + run: + - libcxx # [osx] + - llvmdev =={{ version }} + - python + +test: + requires: + - python {{ environ['PY_VER'] + '*' }} # [win] + - cmake >=3.4.3 + + commands: + - ld.lld --version # [linux] + - lld-link /? # [win] + +about: + home: http://llvm.org/ + license: NCSA + license_file: LICENSE.TXT + summary: The LLVM Linker + +extra: + recipe-maintainers: + - isuruf From 730fc2f21724e4fce406eead720fc1c0d351d46f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Wed, 14 Nov 2018 16:14:26 -0700 Subject: [PATCH 002/158] Downgrade to LLD 6.0.0 --- conda-recipes/lld/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda-recipes/lld/meta.yaml b/conda-recipes/lld/meta.yaml index 8bb99f27e..3ea4a04ff 100644 --- a/conda-recipes/lld/meta.yaml +++ b/conda-recipes/lld/meta.yaml @@ -1,5 +1,5 @@ -{% set version = "7.0.0" %} -{% set sha256 = "fbcf47c5e543f4cdac6bb9bbbc6327ff24217cd7eafc5571549ad6d237287f9c" %} +{% set version = "6.0.0" %} +{% set sha256 = "6b8c4a833cf30230c0213d78dbac01af21387b298225de90ab56032ca79c0e0b" %} package: name: lld From 63313d5b451bb92b41da0931acff3eac9b95451b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Wed, 14 Nov 2018 16:25:50 -0700 Subject: [PATCH 003/158] Disable RTTI for LLD This fixes the following link errors: .../miniconda3/envs/build/conda-bld/lld_1542237425154/_build_env/bin/../lib/gcc/x86_64-conda_cos6-linux-gnu/7.3.0/../../../../x86_64-conda_cos6-linux-gnu/bin/ld: ../../lib/liblldCOFF.a(MapFile.cpp.o):(.data.rel.ro._ZTIN4llvm13format_objectIJmmmEEE[_ZTIN4llvm13format_objectIJmmmEEE]+0x10): undefined reference to `typeinfo for llvm::format_object_base' --- conda-recipes/lld/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-recipes/lld/build.sh b/conda-recipes/lld/build.sh index 57876c4ae..e6e367e10 100644 --- a/conda-recipes/lld/build.sh +++ b/conda-recipes/lld/build.sh @@ -5,7 +5,7 @@ cmake \ -DCMAKE_INSTALL_PREFIX=$PREFIX \ -DCMAKE_PREFIX_PATH=$PREFIX \ -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_ENABLE_RTTI=ON \ + -DLLVM_ENABLE_RTTI=OFF \ -DLLVM_INCLUDE_TESTS=OFF \ .. From f1e8a88b70694403d2209f65da7fbaac61e1ca2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Wed, 14 Nov 2018 16:35:54 -0700 Subject: [PATCH 004/158] Add initial LLD wrapper in C++ For now we just call lld::elf::link. --- ffi/CMakeLists.txt | 3 ++- ffi/Makefile.linux | 2 +- ffi/lld.cpp | 8 ++++++++ 3 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 ffi/lld.cpp diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index a387fea35..d6a583b0f 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -34,7 +34,8 @@ endif() # Define our shared library add_library(llvmlite SHARED assembly.cpp bitcode.cpp core.cpp initfini.cpp module.cpp value.cpp executionengine.cpp transforms.cpp - passmanagers.cpp targets.cpp dylib.cpp linker.cpp object_file.cpp) + passmanagers.cpp targets.cpp dylib.cpp linker.cpp object_file.cpp + lld.cpp) # Find the libraries that correspond to the LLVM components # that we wish to use diff --git a/ffi/Makefile.linux b/ffi/Makefile.linux index 334e9ffd0..f6f5d3a4b 100644 --- a/ffi/Makefile.linux +++ b/ffi/Makefile.linux @@ -11,7 +11,7 @@ LIBS = $(LLVM_LIBS) INCLUDE = core.h SRC = assembly.cpp bitcode.cpp core.cpp initfini.cpp module.cpp value.cpp \ executionengine.cpp transforms.cpp passmanagers.cpp targets.cpp dylib.cpp \ - linker.cpp object_file.cpp + linker.cpp object_file.cpp lld.cpp OUTPUT = libllvmlite.so all: $(OUTPUT) diff --git a/ffi/lld.cpp b/ffi/lld.cpp new file mode 100644 index 000000000..33716ffcb --- /dev/null +++ b/ffi/lld.cpp @@ -0,0 +1,8 @@ +#include "lld/Common/Driver.h" +#include + +int main_test(int Argc, const char **Argv) { + //InitLLVM X(Argc, Argv); + std::vector Args(Argv, Argv + Argc); + return !lld::elf::link(Args, false); +} From 9f3889f5314af7372b4c8b4f0b28199198d176a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Wed, 14 Nov 2018 16:57:54 -0700 Subject: [PATCH 005/158] Add LLD libraries to link This is a temporary workaround to get it working. We have to figure out how to obtain these automatically. --- ffi/build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ffi/build.py b/ffi/build.py index 7d5247dad..1f2bc1532 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -121,7 +121,8 @@ def main_posix(kind, library_ext): # Get LLVM information for building libs = run_llvm_config(llvm_config, "--system-libs --libs all".split()) # Normalize whitespace (trim newlines) - os.environ['LLVM_LIBS'] = ' '.join(libs.split()) + os.environ['LLVM_LIBS'] = ' '.join(libs.split()) + \ + ' -llldCOFF -llldCommon -llldCore -llldDriver -llldELF -llldMachO -llldMinGW -llldReaderWriter -llldWasm -llldYAML' cxxflags = run_llvm_config(llvm_config, ["--cxxflags"]) # on OSX cxxflags has null bytes at the end of the string, remove them From e162690a17e7c053ed33385aebed1f4002d9f838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Wed, 14 Nov 2018 17:32:55 -0700 Subject: [PATCH 006/158] Add the .a libraries first --- ffi/build.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ffi/build.py b/ffi/build.py index 1f2bc1532..482f37547 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -121,8 +121,9 @@ def main_posix(kind, library_ext): # Get LLVM information for building libs = run_llvm_config(llvm_config, "--system-libs --libs all".split()) # Normalize whitespace (trim newlines) - os.environ['LLVM_LIBS'] = ' '.join(libs.split()) + \ - ' -llldCOFF -llldCommon -llldCore -llldDriver -llldELF -llldMachO -llldMinGW -llldReaderWriter -llldWasm -llldYAML' + os.environ['LLVM_LIBS'] = \ + '-llldCOFF -llldCommon -llldCore -llldDriver -llldELF -llldMachO -llldMinGW -llldReaderWriter -llldWasm -llldYAML ' + \ + ' '.join(libs.split()) cxxflags = run_llvm_config(llvm_config, ["--cxxflags"]) # on OSX cxxflags has null bytes at the end of the string, remove them From 5a2b50b6991c1461ce23017e685a66d06f7e1e19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Wed, 14 Nov 2018 17:39:18 -0700 Subject: [PATCH 007/158] Rename the stub to lld_main --- ffi/lld.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ffi/lld.cpp b/ffi/lld.cpp index 33716ffcb..226da02f2 100644 --- a/ffi/lld.cpp +++ b/ffi/lld.cpp @@ -1,7 +1,6 @@ #include "lld/Common/Driver.h" -#include -int main_test(int Argc, const char **Argv) { +int lld_main(int Argc, const char **Argv) { //InitLLVM X(Argc, Argv); std::vector Args(Argv, Argv + Argc); return !lld::elf::link(Args, false); From 5c04cfd288c43c3bda76517a5f5f28f060ea996d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Wed, 14 Nov 2018 17:43:04 -0700 Subject: [PATCH 008/158] Add lld_main_help() for testing --- ffi/lld.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ffi/lld.cpp b/ffi/lld.cpp index 226da02f2..2d5c400df 100644 --- a/ffi/lld.cpp +++ b/ffi/lld.cpp @@ -5,3 +5,8 @@ int lld_main(int Argc, const char **Argv) { std::vector Args(Argv, Argv + Argc); return !lld::elf::link(Args, false); } + +int lld_main_help() { + std::vector Args = {"ld.lld", "--help"}; + return !lld::elf::link(Args, false); +} From 708d9358040cc118ce1d598338c2d605edcb607b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Wed, 14 Nov 2018 17:43:46 -0700 Subject: [PATCH 009/158] Add extern "C" --- ffi/lld.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ffi/lld.cpp b/ffi/lld.cpp index 2d5c400df..5ca58203a 100644 --- a/ffi/lld.cpp +++ b/ffi/lld.cpp @@ -1,5 +1,7 @@ #include "lld/Common/Driver.h" +extern "C" { + int lld_main(int Argc, const char **Argv) { //InitLLVM X(Argc, Argv); std::vector Args(Argv, Argv + Argc); @@ -10,3 +12,5 @@ int lld_main_help() { std::vector Args = {"ld.lld", "--help"}; return !lld::elf::link(Args, false); } + +} // end extern "C" From fc096d07e1b493a8e4c16d398d2318f75638d657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Wed, 14 Nov 2018 17:45:52 -0700 Subject: [PATCH 010/158] Add a Python wrapper for lld_main_help() --- llvmlite/binding/initfini.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/llvmlite/binding/initfini.py b/llvmlite/binding/initfini.py index 7e9e48700..297f3eb02 100644 --- a/llvmlite/binding/initfini.py +++ b/llvmlite/binding/initfini.py @@ -2,6 +2,12 @@ from . import ffi +def lld_main_help(): + """ + Shows help for lld. + """ + ffi.lib.lld_main_help() + def initialize(): """ From af1cba0996301e3cf7f48e3254a721b6580a4881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Fri, 16 Nov 2018 14:44:46 -0700 Subject: [PATCH 011/158] Pass the strings from Python, start with two --- ffi/lld.cpp | 5 +++++ llvmlite/binding/initfini.py | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/ffi/lld.cpp b/ffi/lld.cpp index 5ca58203a..f4797796c 100644 --- a/ffi/lld.cpp +++ b/ffi/lld.cpp @@ -13,4 +13,9 @@ int lld_main_help() { return !lld::elf::link(Args, false); } +int lld_main_2(const char *a, const char *b) { + std::vector Args = {a, b}; + return !lld::elf::link(Args, false); +} + } // end extern "C" diff --git a/llvmlite/binding/initfini.py b/llvmlite/binding/initfini.py index 297f3eb02..285bbdedb 100644 --- a/llvmlite/binding/initfini.py +++ b/llvmlite/binding/initfini.py @@ -1,12 +1,18 @@ from ctypes import c_uint +from ctypes import POINTER, c_char_p, c_int from . import ffi +ffi.lib.lld_main_2.restype = c_int +ffi.lib.lld_main_2.argtypes = [c_char_p, c_char_p] def lld_main_help(): """ Shows help for lld. """ - ffi.lib.lld_main_help() + r = ffi.lib.lld_main_2(c_char_p("ld.lld".encode()), + c_char_p("--help".encode())) + if r != 0: + raise Exception("lld_main() failed, error code: %d" % r) def initialize(): From a0c6cf561455fc63cbe6c40a2d37a1871b8b3d51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Fri, 16 Nov 2018 14:57:42 -0700 Subject: [PATCH 012/158] Make it work with arbitrary number of arguments --- llvmlite/binding/initfini.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/llvmlite/binding/initfini.py b/llvmlite/binding/initfini.py index 285bbdedb..fe6a1bd5d 100644 --- a/llvmlite/binding/initfini.py +++ b/llvmlite/binding/initfini.py @@ -1,16 +1,18 @@ -from ctypes import c_uint -from ctypes import POINTER, c_char_p, c_int +from ctypes import c_uint, POINTER, c_char_p, c_int from . import ffi -ffi.lib.lld_main_2.restype = c_int -ffi.lib.lld_main_2.argtypes = [c_char_p, c_char_p] +ffi.lib.lld_main.restype = c_int +ffi.lib.lld_main.argtypes = [c_int, POINTER(c_char_p)] def lld_main_help(): """ Shows help for lld. """ - r = ffi.lib.lld_main_2(c_char_p("ld.lld".encode()), - c_char_p("--help".encode())) + ldd_args = ["ld.lld", "--help"] + args = (c_char_p*len(ldd_args))() + for i, arg in enumerate(ldd_args): + args[i] = arg.encode() + r = ffi.lib.lld_main(len(ldd_args), args) if r != 0: raise Exception("lld_main() failed, error code: %d" % r) From 87089341273050ba32abce23952e67a0611ef1c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Fri, 16 Nov 2018 14:58:08 -0700 Subject: [PATCH 013/158] Remove the two unneeded functions --- ffi/lld.cpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/ffi/lld.cpp b/ffi/lld.cpp index f4797796c..0c5d4a816 100644 --- a/ffi/lld.cpp +++ b/ffi/lld.cpp @@ -8,14 +8,4 @@ int lld_main(int Argc, const char **Argv) { return !lld::elf::link(Args, false); } -int lld_main_help() { - std::vector Args = {"ld.lld", "--help"}; - return !lld::elf::link(Args, false); -} - -int lld_main_2(const char *a, const char *b) { - std::vector Args = {a, b}; - return !lld::elf::link(Args, false); -} - } // end extern "C" From ba232945bf16f7dcc43d2f1fd3df9b3fe9b0f4f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Fri, 16 Nov 2018 15:01:20 -0700 Subject: [PATCH 014/158] Refactor the code --- llvmlite/binding/initfini.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/llvmlite/binding/initfini.py b/llvmlite/binding/initfini.py index fe6a1bd5d..1e8ffcf6b 100644 --- a/llvmlite/binding/initfini.py +++ b/llvmlite/binding/initfini.py @@ -2,17 +2,24 @@ from . import ffi -ffi.lib.lld_main.restype = c_int -ffi.lib.lld_main.argtypes = [c_int, POINTER(c_char_p)] def lld_main_help(): """ Shows help for lld. """ - ldd_args = ["ld.lld", "--help"] - args = (c_char_p*len(ldd_args))() - for i, arg in enumerate(ldd_args): + lld_main(["ld.lld", "--help"]) + +ffi.lib.lld_main.restype = c_int +ffi.lib.lld_main.argtypes = [c_int, POINTER(c_char_p)] +def lld_main(lld_args): + """ + Calls the `lld::elf::link()` with the `lld_args` arguments. + + lld_args ... A list of strings + """ + args = (c_char_p*len(lld_args))() + for i, arg in enumerate(lld_args): args[i] = arg.encode() - r = ffi.lib.lld_main(len(ldd_args), args) + r = ffi.lib.lld_main(len(lld_args), args) if r != 0: raise Exception("lld_main() failed, error code: %d" % r) From 77e454801d06e3e15c146811f237929906db3f09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Fri, 16 Nov 2018 15:08:22 -0700 Subject: [PATCH 015/158] Move lld_main_help() into docstring --- llvmlite/binding/initfini.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/llvmlite/binding/initfini.py b/llvmlite/binding/initfini.py index 1e8ffcf6b..fea7c0418 100644 --- a/llvmlite/binding/initfini.py +++ b/llvmlite/binding/initfini.py @@ -2,19 +2,22 @@ from . import ffi -def lld_main_help(): - """ - Shows help for lld. - """ - lld_main(["ld.lld", "--help"]) - ffi.lib.lld_main.restype = c_int ffi.lib.lld_main.argtypes = [c_int, POINTER(c_char_p)] def lld_main(lld_args): """ - Calls the `lld::elf::link()` with the `lld_args` arguments. + Calls LLD - the LLVM linker. + + The function calls the `lld::elf::link()` with the `lld_args` (list of + strings) as arguments. + + Examples + ~~~~~~~~ + + Shows help: - lld_args ... A list of strings + >>> from llvmlite.binding import lld_main + >>> lld_main(["ld.lld", "--help"]) """ args = (c_char_p*len(lld_args))() for i, arg in enumerate(lld_args): From b95a72a020af935263ee7f55e01dbc60c4a3dc7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Sat, 17 Nov 2018 16:27:54 -0700 Subject: [PATCH 016/158] Add a test for LLD in x86-64 --- llvmlite/tests/test_binding.py | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/llvmlite/tests/test_binding.py b/llvmlite/tests/test_binding.py index 443b269ee..8d55dc32e 100644 --- a/llvmlite/tests/test_binding.py +++ b/llvmlite/tests/test_binding.py @@ -10,6 +10,7 @@ import re import subprocess import sys +from tempfile import TemporaryDirectory import unittest from contextlib import contextmanager from tempfile import mkstemp @@ -155,6 +156,23 @@ def no_de_locale(): @llvm.global_dtors = appending global [1 x {{i32, void ()*, i8*}}] [{{i32, void ()*, i8*}} {{i32 0, void ()* @dtor_A, i8* null}}] """ +asm_lld_executable = r""" + ; exit(int exit_code) + define void @exit(i64 %exit_code) { + call i64 asm sideeffect "syscall", + "={rax},{rax},{rdi},~{rcx},~{r11},~{dirflag},~{fpsr},~{flags}" + ( i64 60 ; {rax} SYSCALL_EXIT + , i64 %exit_code ; {rdi} exit_code + ) + ret void + } + + define void @_start() { + call void @exit(i64 42) + ret void + } + """ + class BaseTest(TestCase): @@ -1391,5 +1409,26 @@ def test_add_object_file_from_filesystem(self): self.assertEqual(sum_twice(2, 3), 10) +@unittest.skipUnless(platform.machine().startswith('x86'), "only on x86") +class TestLLD_x86(BaseTest): + def test_standalone_executable(self): + with TemporaryDirectory() as tmpdir: + objfile = os.path.join(tmpdir, "test1.o") + binfile = os.path.join(tmpdir, "test1") + llvm.initialize() + llvm.initialize_native_asmprinter() + llvm.initialize_native_asmparser() + llvm.initialize_native_target() + target = llvm.Target.from_triple(llvm.get_default_triple()) + target_machine = target.create_target_machine() + mod = llvm.parse_assembly(asm_lld_executable) + mod.verify() + with open(objfile, "wb") as o: + o.write(target_machine.emit_object(mod)) + llvm.lld_main(["ld.lld", "-o", binfile, objfile]) + r = subprocess.call("%s" % binfile) + self.assertEqual(r, 42) + + if __name__ == "__main__": unittest.main() From caa3d44bf7a8ed18fbc892725e00dff3335d8d2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Sat, 17 Nov 2018 17:29:23 -0700 Subject: [PATCH 017/158] Add a workaround for Python 2.7 --- buildscripts/incremental/setup_conda_environment.sh | 3 +++ llvmlite/tests/test_binding.py | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index bb69424bd..64a7eca35 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -28,6 +28,9 @@ set -v # Install llvmdev (separate channel, for now) $CONDA_INSTALL -c numba llvmdev="7.0*" +if [ "$PYTHON" == "2.7" ]; then + pip install backports.tempfile +fi # Install the compiler toolchain, for osx, bootstrapping needed # which happens in build.sh diff --git a/llvmlite/tests/test_binding.py b/llvmlite/tests/test_binding.py index 8d55dc32e..dab2d6661 100644 --- a/llvmlite/tests/test_binding.py +++ b/llvmlite/tests/test_binding.py @@ -10,7 +10,11 @@ import re import subprocess import sys -from tempfile import TemporaryDirectory +try: + from tempfile import TemporaryDirectory +except ImportError: + # Python 2.7 + from backports.tempfile import TemporaryDirectory import unittest from contextlib import contextmanager from tempfile import mkstemp From 5f09c703d9f028a4c3f7ae1191dad409421df3de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Sat, 17 Nov 2018 17:56:43 -0700 Subject: [PATCH 018/158] Remove unnecessary initializers --- llvmlite/tests/test_binding.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/llvmlite/tests/test_binding.py b/llvmlite/tests/test_binding.py index dab2d6661..3b93055a1 100644 --- a/llvmlite/tests/test_binding.py +++ b/llvmlite/tests/test_binding.py @@ -1419,12 +1419,7 @@ def test_standalone_executable(self): with TemporaryDirectory() as tmpdir: objfile = os.path.join(tmpdir, "test1.o") binfile = os.path.join(tmpdir, "test1") - llvm.initialize() - llvm.initialize_native_asmprinter() - llvm.initialize_native_asmparser() - llvm.initialize_native_target() - target = llvm.Target.from_triple(llvm.get_default_triple()) - target_machine = target.create_target_machine() + target_machine = self.target_machine() mod = llvm.parse_assembly(asm_lld_executable) mod.verify() with open(objfile, "wb") as o: From e7c7bbd05db77f9bc6ae10edb67e848af2343e07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Sat, 26 Jan 2019 14:43:46 -0700 Subject: [PATCH 019/158] Remove lld (it's in llvmdev) --- conda-recipes/lld/bld.bat | 18 ------------- conda-recipes/lld/build.sh | 13 ---------- conda-recipes/lld/meta.yaml | 51 ------------------------------------- 3 files changed, 82 deletions(-) delete mode 100644 conda-recipes/lld/bld.bat delete mode 100644 conda-recipes/lld/build.sh delete mode 100644 conda-recipes/lld/meta.yaml diff --git a/conda-recipes/lld/bld.bat b/conda-recipes/lld/bld.bat deleted file mode 100644 index e5708ee74..000000000 --- a/conda-recipes/lld/bld.bat +++ /dev/null @@ -1,18 +0,0 @@ -mkdir build -cd build - -cmake -G "Ninja" ^ - -DCMAKE_BUILD_TYPE="Release" ^ - -DCMAKE_PREFIX_PATH=%LIBRARY_PREFIX% ^ - -DCMAKE_INSTALL_PREFIX:PATH=%LIBRARY_PREFIX% ^ - -DCLANG_INCLUDE_TESTS=OFF ^ - -DCLANG_INCLUDE_DOCS=OFF ^ - -DLLVM_INCLUDE_TESTS=OFF ^ - -DLLVM_INCLUDE_DOCS=OFF ^ - -DLLVM_TARGETS_TO_BUILD=X86 ^ - %SRC_DIR% - -if errorlevel 1 exit 1 - -cmake --build . --target install -if errorlevel 1 exit 1 \ No newline at end of file diff --git a/conda-recipes/lld/build.sh b/conda-recipes/lld/build.sh deleted file mode 100644 index e6e367e10..000000000 --- a/conda-recipes/lld/build.sh +++ /dev/null @@ -1,13 +0,0 @@ -mkdir build -cd build - -cmake \ - -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DCMAKE_PREFIX_PATH=$PREFIX \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_ENABLE_RTTI=OFF \ - -DLLVM_INCLUDE_TESTS=OFF \ - .. - -make -j${CPU_COUNT} -make install diff --git a/conda-recipes/lld/meta.yaml b/conda-recipes/lld/meta.yaml deleted file mode 100644 index 3ea4a04ff..000000000 --- a/conda-recipes/lld/meta.yaml +++ /dev/null @@ -1,51 +0,0 @@ -{% set version = "6.0.0" %} -{% set sha256 = "6b8c4a833cf30230c0213d78dbac01af21387b298225de90ab56032ca79c0e0b" %} - -package: - name: lld - version: {{ version }} - -source: - fn: lld-{{ version }}.src.tar.xz - url: http://releases.llvm.org/{{ version }}/lld-{{ version }}.src.tar.xz - sha256: {{ sha256 }} - -build: - number: 0 - skip: true # [win and py<35] - -requirements: - build: - - cmake >=3.4.3 - - ninja # [win] - - {{ compiler('c') }} - - {{ compiler('cxx') }} - host: - - python - - llvmdev =={{ version }} - - libxml2 - - swig - - libcxx # [osx] - run: - - libcxx # [osx] - - llvmdev =={{ version }} - - python - -test: - requires: - - python {{ environ['PY_VER'] + '*' }} # [win] - - cmake >=3.4.3 - - commands: - - ld.lld --version # [linux] - - lld-link /? # [win] - -about: - home: http://llvm.org/ - license: NCSA - license_file: LICENSE.TXT - summary: The LLVM Linker - -extra: - recipe-maintainers: - - isuruf From 2739f3d825fe4f2ee7035ba7131b2377f56e2898 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Sat, 26 Jan 2019 15:00:34 -0700 Subject: [PATCH 020/158] Install backports.tempfile in build_llvmlite.sh --- buildscripts/manylinux1/build_llvmlite.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/buildscripts/manylinux1/build_llvmlite.sh b/buildscripts/manylinux1/build_llvmlite.sh index d8b109e59..0970b2c11 100755 --- a/buildscripts/manylinux1/build_llvmlite.sh +++ b/buildscripts/manylinux1/build_llvmlite.sh @@ -20,6 +20,9 @@ conda create -y -n $envname source activate $envname # Install llvmdev conda install -y -c numba/label/manylinux1 llvmdev +if [ "$PYTHON" == "2.7" ]; then + pip install backports.tempfile +fi # Prepend builtin Python Path export PATH=/opt/python/$pyver/bin:$PATH From 734590c159919c986dff2d3c24c2c101b4727cfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Sat, 26 Jan 2019 15:11:48 -0700 Subject: [PATCH 021/158] Use PYVER and the proper value for Python 2.7 --- buildscripts/manylinux1/build_llvmlite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/manylinux1/build_llvmlite.sh b/buildscripts/manylinux1/build_llvmlite.sh index 0970b2c11..714d7e328 100755 --- a/buildscripts/manylinux1/build_llvmlite.sh +++ b/buildscripts/manylinux1/build_llvmlite.sh @@ -20,7 +20,7 @@ conda create -y -n $envname source activate $envname # Install llvmdev conda install -y -c numba/label/manylinux1 llvmdev -if [ "$PYTHON" == "2.7" ]; then +if [[ "$PYVER"="cp27-cp27mu" ]]; then pip install backports.tempfile fi From 364fb9d9924de3b67b24d22acba0d5b287e459e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Sat, 26 Jan 2019 15:35:10 -0700 Subject: [PATCH 022/158] Install backports.tempfile using conda --- buildscripts/manylinux1/build_llvmlite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/manylinux1/build_llvmlite.sh b/buildscripts/manylinux1/build_llvmlite.sh index 714d7e328..484b60263 100755 --- a/buildscripts/manylinux1/build_llvmlite.sh +++ b/buildscripts/manylinux1/build_llvmlite.sh @@ -21,7 +21,7 @@ source activate $envname # Install llvmdev conda install -y -c numba/label/manylinux1 llvmdev if [[ "$PYVER"="cp27-cp27mu" ]]; then - pip install backports.tempfile + conda install -y -c backports.tempfile fi # Prepend builtin Python Path From 80f2414b8bdc62f2f8d5d589262b858c20395ab3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Sat, 26 Jan 2019 15:39:18 -0700 Subject: [PATCH 023/158] Fix a typo --- buildscripts/manylinux1/build_llvmlite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/manylinux1/build_llvmlite.sh b/buildscripts/manylinux1/build_llvmlite.sh index 484b60263..fa3c1b4b2 100755 --- a/buildscripts/manylinux1/build_llvmlite.sh +++ b/buildscripts/manylinux1/build_llvmlite.sh @@ -21,7 +21,7 @@ source activate $envname # Install llvmdev conda install -y -c numba/label/manylinux1 llvmdev if [[ "$PYVER"="cp27-cp27mu" ]]; then - conda install -y -c backports.tempfile + conda install -y backports.tempfile fi # Prepend builtin Python Path From 09cf908cc57785a9202c4747a1f2923d36fd9b24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Sat, 26 Jan 2019 15:43:51 -0700 Subject: [PATCH 024/158] Try conda-forge --- buildscripts/manylinux1/build_llvmlite.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/manylinux1/build_llvmlite.sh b/buildscripts/manylinux1/build_llvmlite.sh index fa3c1b4b2..5701e6a96 100755 --- a/buildscripts/manylinux1/build_llvmlite.sh +++ b/buildscripts/manylinux1/build_llvmlite.sh @@ -21,7 +21,7 @@ source activate $envname # Install llvmdev conda install -y -c numba/label/manylinux1 llvmdev if [[ "$PYVER"="cp27-cp27mu" ]]; then - conda install -y backports.tempfile + conda install -y -c conda-forge backports.tempfile fi # Prepend builtin Python Path From c6ff417700908a61749fbf734c4aa9e4da389f3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Sat, 26 Jan 2019 15:46:16 -0700 Subject: [PATCH 025/158] Pip install it using the right pip --- buildscripts/manylinux1/build_llvmlite.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/buildscripts/manylinux1/build_llvmlite.sh b/buildscripts/manylinux1/build_llvmlite.sh index 5701e6a96..a1c5e1af3 100755 --- a/buildscripts/manylinux1/build_llvmlite.sh +++ b/buildscripts/manylinux1/build_llvmlite.sh @@ -20,15 +20,16 @@ conda create -y -n $envname source activate $envname # Install llvmdev conda install -y -c numba/label/manylinux1 llvmdev -if [[ "$PYVER"="cp27-cp27mu" ]]; then - conda install -y -c conda-forge backports.tempfile -fi # Prepend builtin Python Path export PATH=/opt/python/$pyver/bin:$PATH echo "Using python: $(which python)" +if [[ "$PYVER"="cp27-cp27mu" ]]; then + pip install backports.tempfile +fi + # Clean up git clean -xdf llvmlite build python setup.py clean From 75c73dd735fb246ab964048053eb6031fbdffede Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Tue, 10 Jan 2023 21:16:27 -0500 Subject: [PATCH 026/158] Update lld_main() function to work in modern llvmlite --- ffi/build.py | 2 +- ffi/lld.cpp | 6 ++++-- llvmlite/binding/initfini.py | 6 +++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/ffi/build.py b/ffi/build.py index e4380e9f6..06edeeff5 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -207,7 +207,7 @@ def main_posix(kind, library_ext): except NotImplementedError: default_makeopts = "" makeopts = os.environ.get('LLVMLITE_MAKEOPTS', default_makeopts).split() - subprocess.check_call(['make', '-f', makefile] + makeopts) + subprocess.check_call(['make', '-f', makefile] + makeopts) #? should this also have `-B`? shutil.copy('libllvmlite' + library_ext, target_dir) diff --git a/ffi/lld.cpp b/ffi/lld.cpp index 0c5d4a816..96c78129f 100644 --- a/ffi/lld.cpp +++ b/ffi/lld.cpp @@ -1,11 +1,13 @@ #include "lld/Common/Driver.h" +#include "llvm/Support/raw_ostream.h" extern "C" { -int lld_main(int Argc, const char **Argv) { +bool lld_main(int Argc, const char **Argv) { //InitLLVM X(Argc, Argv); + llvm::raw_ostream &output = llvm::outs(); std::vector Args(Argv, Argv + Argc); - return !lld::elf::link(Args, false); + return !lld::elf::link(Args, false, output, output); } } // end extern "C" diff --git a/llvmlite/binding/initfini.py b/llvmlite/binding/initfini.py index da00ffa5d..94e51d647 100644 --- a/llvmlite/binding/initfini.py +++ b/llvmlite/binding/initfini.py @@ -1,8 +1,8 @@ -from ctypes import c_uint, POINTER, c_char_p, c_int +from ctypes import c_bool, c_uint, POINTER, c_char_p, c_int from llvmlite.binding import ffi -ffi.lib.lld_main.restype = c_int +ffi.lib.lld_main.restype = c_bool ffi.lib.lld_main.argtypes = [c_int, POINTER(c_char_p)] def lld_main(lld_args): """ @@ -23,7 +23,7 @@ def lld_main(lld_args): for i, arg in enumerate(lld_args): args[i] = arg.encode() r = ffi.lib.lld_main(len(lld_args), args) - if r != 0: + if r != False: raise Exception("lld_main() failed, error code: %d" % r) From 4a721165a9e84575ce490bb3c821e1bd8218b514 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Wed, 11 Jan 2023 00:53:47 -0500 Subject: [PATCH 027/158] functions added for each platform lld drivers exist for --- ffi/lld.cpp | 13 ++++-- llvmlite/binding/__init__.py | 3 +- llvmlite/binding/initfini.py | 28 +---------- llvmlite/binding/lld.py | 85 ++++++++++++++++++++++++++++++++++ llvmlite/tests/test_binding.py | 2 +- 5 files changed, 100 insertions(+), 31 deletions(-) create mode 100644 llvmlite/binding/lld.py diff --git a/ffi/lld.cpp b/ffi/lld.cpp index 96c78129f..3034a720b 100644 --- a/ffi/lld.cpp +++ b/ffi/lld.cpp @@ -1,13 +1,20 @@ +#include "core.h" #include "lld/Common/Driver.h" #include "llvm/Support/raw_ostream.h" extern "C" { -bool lld_main(int Argc, const char **Argv) { +bool lld_main(int Argc, const char **Argv, const char **outstr) { //InitLLVM X(Argc, Argv); - llvm::raw_ostream &output = llvm::outs(); + std::string command_output; + llvm::raw_string_ostream command_stream(command_output); std::vector Args(Argv, Argv + Argc); - return !lld::elf::link(Args, false, output, output); + + bool linker_output = !lld::elf::link(Args, false, command_stream, command_stream); + + *outstr = LLVMPY_CreateString(command_output.c_str()); + + return linker_output; } } // end extern "C" diff --git a/llvmlite/binding/__init__.py b/llvmlite/binding/__init__.py index 9c6eedf92..f1d3e52df 100644 --- a/llvmlite/binding/__init__.py +++ b/llvmlite/binding/__init__.py @@ -13,4 +13,5 @@ from .value import * from .analysis import * from .object_file import * -from .context import * \ No newline at end of file +from .context import * +from .lld import * \ No newline at end of file diff --git a/llvmlite/binding/initfini.py b/llvmlite/binding/initfini.py index 94e51d647..7bd935aff 100644 --- a/llvmlite/binding/initfini.py +++ b/llvmlite/binding/initfini.py @@ -1,31 +1,7 @@ -from ctypes import c_bool, c_uint, POINTER, c_char_p, c_int +from ctypes import byref, c_bool, c_uint, POINTER, c_char_p, c_int from llvmlite.binding import ffi - -ffi.lib.lld_main.restype = c_bool -ffi.lib.lld_main.argtypes = [c_int, POINTER(c_char_p)] -def lld_main(lld_args): - """ - Calls LLD - the LLVM linker. - - The function calls the `lld::elf::link()` with the `lld_args` (list of - strings) as arguments. - - Examples - ~~~~~~~~ - - Shows help: - - >>> from llvmlite.binding import lld_main - >>> lld_main(["ld.lld", "--help"]) - """ - args = (c_char_p*len(lld_args))() - for i, arg in enumerate(lld_args): - args[i] = arg.encode() - r = ffi.lib.lld_main(len(lld_args), args) - if r != False: - raise Exception("lld_main() failed, error code: %d" % r) - +from llvmlite.binding.common import _encode_string def initialize(): """ diff --git a/llvmlite/binding/lld.py b/llvmlite/binding/lld.py new file mode 100644 index 000000000..bf694b126 --- /dev/null +++ b/llvmlite/binding/lld.py @@ -0,0 +1,85 @@ +from ctypes import c_bool, c_uint, POINTER, c_char_p, c_int, c_wchar_p +import platform + +from llvmlite.binding import ffi + +ffi.lib.lld_main.restype = c_bool +ffi.lib.lld_main.argtypes = [c_int, POINTER(c_char_p), POINTER(c_char_p)] +def lld_main(lld_args) -> str: + """ + Calls LLD - the LLVM linker. + + The function calls the `lld::elf::link()` with the `lld_args` (list of + strings) as arguments. + + Examples + ~~~~~~~~ + + Shows help: + + >>> from llvmlite.binding import lld_main + >>> lld_main(["ld.lld", "--help"]) + """ + args = (c_char_p*len(lld_args))() + for i, arg in enumerate(lld_args): + args[i] = arg.encode() + with ffi.OutputString() as outstr: + r = ffi.lib.lld_main(len(lld_args), args, outstr) + if r != False: + raise Exception("lld_main() failed, error code: %d" % r) + + return str(outstr) + +def lld_windows(output: str, objects: list[str], args: list[str] = []) -> str: + ''' + runs the command `lld-link -o {output-file} {*input_files} {*args}` + output: output as a str + object: a list of input .o files as strings + args: additional arguments for the command + ''' + return lld_main(["lld-link", "-o", output, *objects, *args]) + +def lld_macos(output: str, objects: list[str], args: list[str] = []) -> str: + ''' + runs the command `ld64.lld -o {output-file} {*input_files} {*args}` + output: output as a str + object: a list of input .o files as strings + args: additional arguments for the command + ''' + return lld_main(["ld64.lld", "-o", output, *objects, *args]) + +def lld_linux(output: str, objects: list[str], args: list[str] = []) -> str: + ''' + runs the command `ld.lld -o {output-file} {*input_files} {*args}` + output: output as a str + object: a list of input .o files as strings + args: additional arguments for the command + ''' + return lld_main(["ld.lld", "-o", output, *objects, *args]) + +def lld_wasm(output: str, objects: list[str], args: list[str] = []) -> str: + ''' + runs the command `ld.lld -o {output-file} {*input_files} {*args}` + output: output as a str + object: a list of input .o files as strings + args: additional arguments for the command + ''' + return lld_main(["wasm-ld", "-o", output, *objects, *args]) + +def lld_auto(output: str, objects: list[str], args: list[str] = [], add_extension = True) -> str: + ''' + Automatically determines which lld function to run based on the hosts system. + Does not use `lld_wasm()` + + add_extension: adds `.exe` and other file endings automatically + ''' + system = platform.system() + + if system in "Linux": + return lld_linux(output, objects, args) + if system == "Windows": + name = output+('.exe'*add_extension) + return lld_windows(name, objects, args) + if system == "Darwin": # Macos + name = output+('.app'*add_extension) + return lld_macos(name, objects, args) \ No newline at end of file diff --git a/llvmlite/tests/test_binding.py b/llvmlite/tests/test_binding.py index c8e17187f..66df2f4fc 100644 --- a/llvmlite/tests/test_binding.py +++ b/llvmlite/tests/test_binding.py @@ -2017,7 +2017,7 @@ def test_standalone_executable(self): with TemporaryDirectory() as tmpdir: objfile = os.path.join(tmpdir, "test1.o") binfile = os.path.join(tmpdir, "test1") - target_machine = self.target_machine() + target_machine = self.target_machine(jit=False) mod = llvm.parse_assembly(asm_lld_executable) mod.verify() with open(objfile, "wb") as o: From 4f5b563b1e38474188eb22977312c492b7e05e15 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Wed, 11 Jan 2023 20:40:02 -0500 Subject: [PATCH 028/158] Rmove unused imports and better follow flake8 --- llvmlite/binding/initfini.py | 4 +- llvmlite/binding/lld.py | 83 +++++++++++++++++------------------- 2 files changed, 41 insertions(+), 46 deletions(-) diff --git a/llvmlite/binding/initfini.py b/llvmlite/binding/initfini.py index 7bd935aff..4466d9da2 100644 --- a/llvmlite/binding/initfini.py +++ b/llvmlite/binding/initfini.py @@ -1,7 +1,7 @@ -from ctypes import byref, c_bool, c_uint, POINTER, c_char_p, c_int +from ctypes import c_uint from llvmlite.binding import ffi -from llvmlite.binding.common import _encode_string + def initialize(): """ diff --git a/llvmlite/binding/lld.py b/llvmlite/binding/lld.py index bf694b126..6878c4e61 100644 --- a/llvmlite/binding/lld.py +++ b/llvmlite/binding/lld.py @@ -1,10 +1,14 @@ -from ctypes import c_bool, c_uint, POINTER, c_char_p, c_int, c_wchar_p +from ctypes import c_bool, POINTER, c_char_p, c_int import platform +from typing import List from llvmlite.binding import ffi + ffi.lib.lld_main.restype = c_bool ffi.lib.lld_main.argtypes = [c_int, POINTER(c_char_p), POINTER(c_char_p)] + + def lld_main(lld_args) -> str: """ Calls LLD - the LLVM linker. @@ -20,66 +24,57 @@ def lld_main(lld_args) -> str: >>> from llvmlite.binding import lld_main >>> lld_main(["ld.lld", "--help"]) """ - args = (c_char_p*len(lld_args))() + args = (c_char_p * len(lld_args))() for i, arg in enumerate(lld_args): args[i] = arg.encode() - with ffi.OutputString() as outstr: + with ffi.OutputString() as outstr: r = ffi.lib.lld_main(len(lld_args), args, outstr) - if r != False: + if not r: raise Exception("lld_main() failed, error code: %d" % r) - + return str(outstr) -def lld_windows(output: str, objects: list[str], args: list[str] = []) -> str: - ''' - runs the command `lld-link -o {output-file} {*input_files} {*args}` - output: output as a str - object: a list of input .o files as strings - args: additional arguments for the command - ''' - return lld_main(["lld-link", "-o", output, *objects, *args]) -def lld_macos(output: str, objects: list[str], args: list[str] = []) -> str: - ''' - runs the command `ld64.lld -o {output-file} {*input_files} {*args}` - output: output as a str - object: a list of input .o files as strings - args: additional arguments for the command - ''' - return lld_main(["ld64.lld", "-o", output, *objects, *args]) +def lld_runner(command: str): + '''creates lld functions while still allowing docstrings for users to see.''' + def wrapped(output: str, objects: List[str], args: List[str] = []) -> str: + ''' + runs the command "{platform's lld command} -o {output-file} {*input_files} {*args}" + output: output file as a str + object: a list of input .o files as strings + args: additional arguments for the command + ''' + return lld_main([command, "-o", output, *objects, *args]) + return wrapped -def lld_linux(output: str, objects: list[str], args: list[str] = []) -> str: - ''' - runs the command `ld.lld -o {output-file} {*input_files} {*args}` - output: output as a str - object: a list of input .o files as strings - args: additional arguments for the command - ''' - return lld_main(["ld.lld", "-o", output, *objects, *args]) -def lld_wasm(output: str, objects: list[str], args: list[str] = []) -> str: - ''' - runs the command `ld.lld -o {output-file} {*input_files} {*args}` - output: output as a str - object: a list of input .o files as strings - args: additional arguments for the command - ''' - return lld_main(["wasm-ld", "-o", output, *objects, *args]) +lld_windows = lld_runner("lld-link") +lld_macos = lld_runner("ld64.lld") +lld_linux = lld_runner("ld.lld") +lld_wasm = lld_runner("wasm-ld") + def lld_auto(output: str, objects: list[str], args: list[str] = [], add_extension = True) -> str: ''' Automatically determines which lld function to run based on the hosts system. Does not use `lld_wasm()` - add_extension: adds `.exe` and other file endings automatically + add_extension: adds `.exe` and other file endings automatically ''' system = platform.system() - if system in "Linux": + if system == "Linux": + # has no standard file extension return lld_linux(output, objects, args) - if system == "Windows": - name = output+('.exe'*add_extension) + elif system == "Windows": + name = output + ('.exe' * add_extension) return lld_windows(name, objects, args) - if system == "Darwin": # Macos - name = output+('.app'*add_extension) - return lld_macos(name, objects, args) \ No newline at end of file + elif system == "Darwin": # Macos + name = output + ('.app' * add_extension) + return lld_macos(name, objects, args) + else: + msg = (f'''Invalid host system name: {system} + If you are sure a driver exists for this system, + please use the `lld_main()` function + '''.strip()) + raise RuntimeError(msg) From 831b3c6a3da3b96752351241ac013c7033fd3f95 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Wed, 11 Jan 2023 22:04:10 -0500 Subject: [PATCH 029/158] remove py27 code --- buildscripts/manylinux_2_17/build_llvmlite.sh | 4 ---- llvmlite/tests/test_binding.py | 6 +----- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/buildscripts/manylinux_2_17/build_llvmlite.sh b/buildscripts/manylinux_2_17/build_llvmlite.sh index ee97a71c1..1a8baa0bc 100644 --- a/buildscripts/manylinux_2_17/build_llvmlite.sh +++ b/buildscripts/manylinux_2_17/build_llvmlite.sh @@ -31,10 +31,6 @@ export PATH=/opt/python/$pyver/bin:$PATH echo "Using python: $(which python)" -if [[ "$PYVER"="cp27-cp27mu" ]]; then - pip install backports.tempfile -fi - # Clean up git clean -xdf llvmlite build python setup.py clean diff --git a/llvmlite/tests/test_binding.py b/llvmlite/tests/test_binding.py index 66df2f4fc..dec66798f 100644 --- a/llvmlite/tests/test_binding.py +++ b/llvmlite/tests/test_binding.py @@ -8,11 +8,7 @@ import re import subprocess import sys -try: - from tempfile import TemporaryDirectory -except ImportError: - # Python 2.7 - from backports.tempfile import TemporaryDirectory +from tempfile import TemporaryDirectory import unittest from contextlib import contextmanager from tempfile import mkstemp From dc777a0e82ff226153c30a586123457502e9a946 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Wed, 11 Jan 2023 22:19:17 -0500 Subject: [PATCH 030/158] hopefully made lld.cpp work on all platforms --- ffi/lld.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ffi/lld.cpp b/ffi/lld.cpp index 3034a720b..75ad0c075 100644 --- a/ffi/lld.cpp +++ b/ffi/lld.cpp @@ -10,7 +10,13 @@ bool lld_main(int Argc, const char **Argv, const char **outstr) { llvm::raw_string_ostream command_stream(command_output); std::vector Args(Argv, Argv + Argc); - bool linker_output = !lld::elf::link(Args, false, command_stream, command_stream); + #if defined __linux__ || __unix__ + bool linker_output = !lld::elf::link(Args, false, command_stream, command_stream); + #elif defined(WIN32) || defined(_WIN32) || defined(__WIN32__) + bool linker_output = !lld::coff::link(Args, false, command_stream, command_stream); + #else + bool linker_output = !lld::macho::link(Args, false, command_stream, command_stream); + #endif *outstr = LLVMPY_CreateString(command_output.c_str()); From b5bb1fcbdcd6ac9ab6c6ea28124e6bde930ca565 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Wed, 11 Jan 2023 23:51:03 -0500 Subject: [PATCH 031/158] hopefully got windows to build --- ffi/build.py | 3 ++- llvmlite/tests/test_binding.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ffi/build.py b/ffi/build.py index 06edeeff5..8e62ce4ef 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -97,7 +97,8 @@ def main_windows(): config = 'Release' if not os.path.exists(build_dir): os.mkdir(build_dir) - # Run configuration step + os.environ['LLVM_LIBS'] += \ + '-llldCOFF -llldCommon -llldCore -llldDriver -llldELF -llldMachO -llldMinGW -llldReaderWriter -llldWasm -llldYAML' try_cmake(here_dir, build_dir, *generator) subprocess.check_call(['cmake', '--build', build_dir, '--config', config]) shutil.copy(os.path.join(build_dir, config, 'llvmlite.dll'), target_dir) diff --git a/llvmlite/tests/test_binding.py b/llvmlite/tests/test_binding.py index dec66798f..bb44d9385 100644 --- a/llvmlite/tests/test_binding.py +++ b/llvmlite/tests/test_binding.py @@ -2018,7 +2018,7 @@ def test_standalone_executable(self): mod.verify() with open(objfile, "wb") as o: o.write(target_machine.emit_object(mod)) - llvm.lld_main(["ld.lld", "-o", binfile, objfile]) + llvm.lld.lld_auto(binfile, [objfile]) r = subprocess.call("%s" % binfile) self.assertEqual(r, 42) From beeb8b5c129f4eef85fcf04325b1be1cb8d1d204 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Thu, 12 Jan 2023 00:23:37 -0500 Subject: [PATCH 032/158] attempt to add lld options into Cmake --- ffi/CMakeLists.txt | 1 + ffi/build.py | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index f1a210c95..a940ee217 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -31,6 +31,7 @@ CHECK_INCLUDE_FILES("llvm/IR/SVML.inc" HAVE_SVML) if(HAVE_SVML) message(STATUS "SVML found") add_definitions(-DHAVE_SVML) + add_compile_options(-llldCOFF -llldCommon -llldCore -llldDriver -llldELF -llldMachO -llldMinGW -llldReaderWriter -llldWasm -llldYAML) else() message(STATUS "SVML not found") endif() diff --git a/ffi/build.py b/ffi/build.py index 8e62ce4ef..1c01e2c7f 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -97,8 +97,6 @@ def main_windows(): config = 'Release' if not os.path.exists(build_dir): os.mkdir(build_dir) - os.environ['LLVM_LIBS'] += \ - '-llldCOFF -llldCommon -llldCore -llldDriver -llldELF -llldMachO -llldMinGW -llldReaderWriter -llldWasm -llldYAML' try_cmake(here_dir, build_dir, *generator) subprocess.check_call(['cmake', '--build', build_dir, '--config', config]) shutil.copy(os.path.join(build_dir, config, 'llvmlite.dll'), target_dir) From 11cbea7d3ec26c64afa0222cf67c960f5f0345ad Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Thu, 12 Jan 2023 10:21:56 -0500 Subject: [PATCH 033/158] add -DLLVM_ENABLE_PROJECTS=lld to see if it works --- ffi/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index a940ee217..1ad5b3c78 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -30,8 +30,8 @@ set(CMAKE_REQUIRED_INCLUDES ${LLVM_INCLUDE_DIRS}) CHECK_INCLUDE_FILES("llvm/IR/SVML.inc" HAVE_SVML) if(HAVE_SVML) message(STATUS "SVML found") - add_definitions(-DHAVE_SVML) - add_compile_options(-llldCOFF -llldCommon -llldCore -llldDriver -llldELF -llldMachO -llldMinGW -llldReaderWriter -llldWasm -llldYAML) + add_definitions(-DHAVE_SVML -DLLVM_ENABLE_PROJECTS=lld) + # add_compile_options(-llldCOFF -llldCommon -llldCore -llldDriver -llldELF -llldMachO -llldMinGW -llldReaderWriter -llldWasm -llldYAML) else() message(STATUS "SVML not found") endif() From e9f58507ab3b6e0d6fcc3a28b708a6cfe1c6fbf4 Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Thu, 12 Jan 2023 10:34:12 -0500 Subject: [PATCH 034/158] Another attempt to get windows to compile --- ffi/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index 1ad5b3c78..ec4f33e06 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -12,6 +12,7 @@ if(NOT MSVC) endif() find_package(LLVM REQUIRED CONFIG) +find_package(LLD REQUIRED CONFIG) message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") @@ -31,7 +32,7 @@ CHECK_INCLUDE_FILES("llvm/IR/SVML.inc" HAVE_SVML) if(HAVE_SVML) message(STATUS "SVML found") add_definitions(-DHAVE_SVML -DLLVM_ENABLE_PROJECTS=lld) - # add_compile_options(-llldCOFF -llldCommon -llldCore -llldDriver -llldELF -llldMachO -llldMinGW -llldReaderWriter -llldWasm -llldYAML) + add_compile_options(-llldCOFF -llldCommon -llldCore -llldDriver -llldELF -llldMachO -llldMinGW -llldReaderWriter -llldWasm -llldYAML) else() message(STATUS "SVML not found") endif() From 5ca4cab584c5c374ac967c2adcbd246b961ea08a Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Thu, 12 Jan 2023 10:54:03 -0500 Subject: [PATCH 035/158] Add quotes to LLVM_ENABLE_PROJECTS list --- ffi/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index ec4f33e06..03c9d0d4b 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -31,7 +31,7 @@ set(CMAKE_REQUIRED_INCLUDES ${LLVM_INCLUDE_DIRS}) CHECK_INCLUDE_FILES("llvm/IR/SVML.inc" HAVE_SVML) if(HAVE_SVML) message(STATUS "SVML found") - add_definitions(-DHAVE_SVML -DLLVM_ENABLE_PROJECTS=lld) + add_definitions(-DHAVE_SVML -DLLVM_ENABLE_PROJECTS="lld") add_compile_options(-llldCOFF -llldCommon -llldCore -llldDriver -llldELF -llldMachO -llldMinGW -llldReaderWriter -llldWasm -llldYAML) else() message(STATUS "SVML not found") From e91c501838c8423e7798ee02b81cf388cb4e9f2a Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Thu, 12 Jan 2023 11:18:01 -0500 Subject: [PATCH 036/158] Attempt to add ${LLD_SOURCE_DIR}/include --- ffi/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index 03c9d0d4b..9a9e83033 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -22,7 +22,7 @@ message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") # you will need to enable C++11 support # for your compiler. -include_directories(${LLVM_INCLUDE_DIRS}) +include_directories(${LLVM_INCLUDE_DIRS} ${LLD_SOURCE_DIR}/include) add_definitions(${LLVM_DEFINITIONS}) # Look for SVML From 7c54827c1ff2af99840cb7d5e02b23370c51f24c Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Thu, 12 Jan 2023 13:08:17 -0500 Subject: [PATCH 037/158] trying more random things --- ffi/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index 9a9e83033..28a5d4515 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -26,13 +26,13 @@ include_directories(${LLVM_INCLUDE_DIRS} ${LLD_SOURCE_DIR}/include) add_definitions(${LLVM_DEFINITIONS}) # Look for SVML -set(CMAKE_REQUIRED_INCLUDES ${LLVM_INCLUDE_DIRS}) +set(CMAKE_REQUIRED_INCLUDES ${LLVM_INCLUDE_DIRS} ${LLD_SOURCE_DIR}/include) CHECK_INCLUDE_FILES("llvm/IR/SVML.inc" HAVE_SVML) if(HAVE_SVML) message(STATUS "SVML found") add_definitions(-DHAVE_SVML -DLLVM_ENABLE_PROJECTS="lld") - add_compile_options(-llldCOFF -llldCommon -llldCore -llldDriver -llldELF -llldMachO -llldMinGW -llldReaderWriter -llldWasm -llldYAML) + #add_compile_options(-llldCOFF -llldCommon -llldCore -llldDriver -llldELF -llldMachO -llldMinGW -llldReaderWriter -llldWasm -llldYAML) else() message(STATUS "SVML not found") endif() @@ -51,7 +51,7 @@ add_library(llvmlite SHARED assembly.cpp bitcode.cpp core.cpp initfini.cpp # Bug reported upstream at: https://bugs.llvm.org/show_bug.cgi?id=47003 # BROKEN: llvm_map_components_to_libnames(llvm_libs all) if ($ENV{LLVMLITE_SHARED}) - set(llvm_libs LLVM) + set(llvm_libs LLVM LLD) else() set(llvm_libs ${LLVM_AVAILABLE_LIBS}) endif() From 8a214f6d85a55b393b139e6fa6efb4e3da86fe0a Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Thu, 12 Jan 2023 21:31:34 -0500 Subject: [PATCH 038/158] change linking stuff --- ffi/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index a940ee217..455f618a5 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -31,7 +31,7 @@ CHECK_INCLUDE_FILES("llvm/IR/SVML.inc" HAVE_SVML) if(HAVE_SVML) message(STATUS "SVML found") add_definitions(-DHAVE_SVML) - add_compile_options(-llldCOFF -llldCommon -llldCore -llldDriver -llldELF -llldMachO -llldMinGW -llldReaderWriter -llldWasm -llldYAML) + add_library(lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) else() message(STATUS "SVML not found") endif() From b63546189a31ef7cc146be463bc55218c0df761f Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Thu, 12 Jan 2023 21:42:32 -0500 Subject: [PATCH 039/158] attempt to fix latest link error --- ffi/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index 81895e4ba..d712c979d 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -12,7 +12,7 @@ if(NOT MSVC) endif() find_package(LLVM REQUIRED CONFIG) -find_package(LLD REQUIRED CONFIG) +# find_package(LLD REQUIRED CONFIG) message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") @@ -31,7 +31,7 @@ set(CMAKE_REQUIRED_INCLUDES ${LLVM_INCLUDE_DIRS} ${LLD_SOURCE_DIR}/include) CHECK_INCLUDE_FILES("llvm/IR/SVML.inc" HAVE_SVML) if(HAVE_SVML) message(STATUS "SVML found") - add_definitions(-DHAVE_SVML -DLLVM_ENABLE_PROJECTS="lld") + add_definitions(-DHAVE_SVML) add_library(lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) else() message(STATUS "SVML not found") From 514d9b986f058accb799ef53650656a54aa2d731 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Thu, 12 Jan 2023 21:50:21 -0500 Subject: [PATCH 040/158] Another attempt --- ffi/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index d712c979d..efb94f407 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -22,16 +22,16 @@ message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") # you will need to enable C++11 support # for your compiler. -include_directories(${LLVM_INCLUDE_DIRS} ${LLD_SOURCE_DIR}/include) +include_directories(${LLVM_INCLUDE_DIRS} ${LLD_SOURCE_DIR}) add_definitions(${LLVM_DEFINITIONS}) # Look for SVML -set(CMAKE_REQUIRED_INCLUDES ${LLVM_INCLUDE_DIRS} ${LLD_SOURCE_DIR}/include) +set(CMAKE_REQUIRED_INCLUDES ${LLVM_INCLUDE_DIRS} ${LLD_SOURCE_DIR}) CHECK_INCLUDE_FILES("llvm/IR/SVML.inc" HAVE_SVML) if(HAVE_SVML) message(STATUS "SVML found") - add_definitions(-DHAVE_SVML) + add_definitions(-DHAVE_SVML -DLLVM_ENABLE_PROJECTS="lld") add_library(lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) else() message(STATUS "SVML not found") From 5faf2485a98a308ad8594beb20c6b4165e4b3565 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Thu, 12 Jan 2023 22:00:06 -0500 Subject: [PATCH 041/158] hmmm. Not sure whats happening --- ffi/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index efb94f407..ce24610a5 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -12,7 +12,7 @@ if(NOT MSVC) endif() find_package(LLVM REQUIRED CONFIG) -# find_package(LLD REQUIRED CONFIG) +find_package(LLD REQUIRED CONFIG) message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") From ae7f2121ac34a81064ccab2b3695e3d4122f0c55 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Thu, 12 Jan 2023 22:07:44 -0500 Subject: [PATCH 042/158] remove lldCOFF to see what happens --- ffi/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index ce24610a5..1f033b4a3 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -32,7 +32,7 @@ CHECK_INCLUDE_FILES("llvm/IR/SVML.inc" HAVE_SVML) if(HAVE_SVML) message(STATUS "SVML found") add_definitions(-DHAVE_SVML -DLLVM_ENABLE_PROJECTS="lld") - add_library(lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) + add_library(lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) else() message(STATUS "SVML not found") endif() From fdeee7abc0bc9f0e5f26dc3a136e7de3791adacc Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Thu, 12 Jan 2023 22:13:47 -0500 Subject: [PATCH 043/158] random commit name --- ffi/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index 1f033b4a3..6093e0a73 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -32,7 +32,7 @@ CHECK_INCLUDE_FILES("llvm/IR/SVML.inc" HAVE_SVML) if(HAVE_SVML) message(STATUS "SVML found") add_definitions(-DHAVE_SVML -DLLVM_ENABLE_PROJECTS="lld") - add_library(lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) + # add_library(lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) else() message(STATUS "SVML not found") endif() From cf4d4b6e0ddb7f18b0ec8b613e0a1511b8d177e9 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Thu, 12 Jan 2023 22:22:31 -0500 Subject: [PATCH 044/158] try add_lld_library macro --- ffi/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index 6093e0a73..116bd1cf7 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -32,7 +32,7 @@ CHECK_INCLUDE_FILES("llvm/IR/SVML.inc" HAVE_SVML) if(HAVE_SVML) message(STATUS "SVML found") add_definitions(-DHAVE_SVML -DLLVM_ENABLE_PROJECTS="lld") - # add_library(lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) + add_lld_library(lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) else() message(STATUS "SVML not found") endif() From 1354956565139409f60ce9a2b817362cf6fd6837 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Thu, 12 Jan 2023 22:53:03 -0500 Subject: [PATCH 045/158] Another attempt --- ffi/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index 116bd1cf7..bf5ae7c2f 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -22,17 +22,17 @@ message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") # you will need to enable C++11 support # for your compiler. -include_directories(${LLVM_INCLUDE_DIRS} ${LLD_SOURCE_DIR}) +include_directories(${LLVM_INCLUDE_DIRS} ${LLD_INCLUDE_DIRS}) add_definitions(${LLVM_DEFINITIONS}) # Look for SVML -set(CMAKE_REQUIRED_INCLUDES ${LLVM_INCLUDE_DIRS} ${LLD_SOURCE_DIR}) +set(CMAKE_REQUIRED_INCLUDES ${LLVM_INCLUDE_DIRS} ${LLD_INCLUDE_DIRS}) CHECK_INCLUDE_FILES("llvm/IR/SVML.inc" HAVE_SVML) if(HAVE_SVML) message(STATUS "SVML found") add_definitions(-DHAVE_SVML -DLLVM_ENABLE_PROJECTS="lld") - add_lld_library(lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) + # add_lld_library(lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) else() message(STATUS "SVML not found") endif() From 2ec96baf6393980479aba89e74b23f112ca1c203 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Thu, 12 Jan 2023 23:06:20 -0500 Subject: [PATCH 046/158] trying random shit. --- ffi/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index bf5ae7c2f..aaf07f130 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -51,7 +51,7 @@ add_library(llvmlite SHARED assembly.cpp bitcode.cpp core.cpp initfini.cpp # Bug reported upstream at: https://bugs.llvm.org/show_bug.cgi?id=47003 # BROKEN: llvm_map_components_to_libnames(llvm_libs all) if ($ENV{LLVMLITE_SHARED}) - set(llvm_libs LLVM LLD) + set(llvm_libs LLVM lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) else() set(llvm_libs ${LLVM_AVAILABLE_LIBS}) endif() From e4cc1d657ccb45b108b22e9a00cf04675dc9f3e0 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Thu, 12 Jan 2023 23:16:35 -0500 Subject: [PATCH 047/158] debug message for testing --- ffi/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index aaf07f130..b8b85457b 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -51,8 +51,10 @@ add_library(llvmlite SHARED assembly.cpp bitcode.cpp core.cpp initfini.cpp # Bug reported upstream at: https://bugs.llvm.org/show_bug.cgi?id=47003 # BROKEN: llvm_map_components_to_libnames(llvm_libs all) if ($ENV{LLVMLITE_SHARED}) - set(llvm_libs LLVM lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) + message("ENV LLVMLITE_SHARED") + set(llvm_libs lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML LLVM) else() + message("NOT ENV LLVMLITE_SHARED") set(llvm_libs ${LLVM_AVAILABLE_LIBS}) endif() From af791026fae355de7a4f7c955b3bfde5f27907e5 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Thu, 12 Jan 2023 23:29:29 -0500 Subject: [PATCH 048/158] more message stuff --- ffi/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index b8b85457b..0f3fe2cf1 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -55,6 +55,8 @@ if ($ENV{LLVMLITE_SHARED}) set(llvm_libs lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML LLVM) else() message("NOT ENV LLVMLITE_SHARED") + message(${LLVM_AVAILABLE_LIBS}) + message("------------------------") set(llvm_libs ${LLVM_AVAILABLE_LIBS}) endif() From 9dfb01250ad3aa3ae13ed320961ef16b3ebc9f7e Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Thu, 12 Jan 2023 23:38:12 -0500 Subject: [PATCH 049/158] Another test --- ffi/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index 0f3fe2cf1..96d452c35 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -75,4 +75,6 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") set(LLVM_EXPORTED_SYMBOLS "-Wl,-exported_symbol,_LLVMPY_*") set_property(TARGET llvmlite APPEND_STRING PROPERTY LINK_FLAGS "${LLVM_EXPORTED_SYMBOLS}") +else() + set_property(TARGET llvmlite APPEND_STRING PROPERTY LINK_FLAGS "-flto -Wl,--exclude-libs,ALL") endif() From 97bf22bf32028da0f21a19fefdf32d3c6a9b8036 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Fri, 13 Jan 2023 00:06:03 -0500 Subject: [PATCH 050/158] Another attempt --- ffi/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index 96d452c35..a3de5c66e 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -76,5 +76,5 @@ elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") set(LLVM_EXPORTED_SYMBOLS "-Wl,-exported_symbol,_LLVMPY_*") set_property(TARGET llvmlite APPEND_STRING PROPERTY LINK_FLAGS "${LLVM_EXPORTED_SYMBOLS}") else() - set_property(TARGET llvmlite APPEND_STRING PROPERTY LINK_FLAGS "-flto -Wl,--exclude-libs,ALL") + set_property(TARGET llvmlite APPEND_STRING PROPERTY LINK_FLAGS "${LLVM_EXPORTED_SYMBOLS}") endif() From 1e234d4ea1e0685658042c600da0abdade489e7e Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Fri, 13 Jan 2023 00:13:16 -0500 Subject: [PATCH 051/158] attempt number... idk --- ffi/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index a3de5c66e..8fa3e745a 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -76,5 +76,5 @@ elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") set(LLVM_EXPORTED_SYMBOLS "-Wl,-exported_symbol,_LLVMPY_*") set_property(TARGET llvmlite APPEND_STRING PROPERTY LINK_FLAGS "${LLVM_EXPORTED_SYMBOLS}") else() - set_property(TARGET llvmlite APPEND_STRING PROPERTY LINK_FLAGS "${LLVM_EXPORTED_SYMBOLS}") + set_property(TARGET llvmlite APPEND_STRING PROPERTY LINK_FLAGS "${LLVM_EXPORTED_SYMBOLS}" /FORCE:UNRESOLVED) endif() From 5f21d65fc9e039447e5ab5fb3411c499ad750217 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Fri, 13 Jan 2023 00:26:00 -0500 Subject: [PATCH 052/158] add_link_options test --- ffi/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index 8fa3e745a..7c9113d75 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -76,5 +76,6 @@ elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") set(LLVM_EXPORTED_SYMBOLS "-Wl,-exported_symbol,_LLVMPY_*") set_property(TARGET llvmlite APPEND_STRING PROPERTY LINK_FLAGS "${LLVM_EXPORTED_SYMBOLS}") else() - set_property(TARGET llvmlite APPEND_STRING PROPERTY LINK_FLAGS "${LLVM_EXPORTED_SYMBOLS}" /FORCE:UNRESOLVED) + add_link_options("/FORCE:UNRESOLVED") + set_property(TARGET llvmlite APPEND_STRING PROPERTY LINK_FLAGS "${LLVM_EXPORTED_SYMBOLS}") endif() From 47f4f5b8d72deacfe8ce0cfa46ebff649d4b9f34 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Fri, 13 Jan 2023 00:35:30 -0500 Subject: [PATCH 053/158] trying random stuff again --- ffi/CMakeLists.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index 7c9113d75..5bc9e81e6 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -32,7 +32,7 @@ CHECK_INCLUDE_FILES("llvm/IR/SVML.inc" HAVE_SVML) if(HAVE_SVML) message(STATUS "SVML found") add_definitions(-DHAVE_SVML -DLLVM_ENABLE_PROJECTS="lld") - # add_lld_library(lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) + add_library(lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) else() message(STATUS "SVML not found") endif() @@ -75,7 +75,4 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") set(LLVM_EXPORTED_SYMBOLS "-Wl,-exported_symbol,_LLVMPY_*") set_property(TARGET llvmlite APPEND_STRING PROPERTY LINK_FLAGS "${LLVM_EXPORTED_SYMBOLS}") -else() - add_link_options("/FORCE:UNRESOLVED") - set_property(TARGET llvmlite APPEND_STRING PROPERTY LINK_FLAGS "${LLVM_EXPORTED_SYMBOLS}") endif() From c11d26c13c122efcbedad1cc4ef0cf66a98cf408 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Fri, 13 Jan 2023 00:46:23 -0500 Subject: [PATCH 054/158] trying more stuff --- ffi/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index 5bc9e81e6..f4c7920f4 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -32,7 +32,7 @@ CHECK_INCLUDE_FILES("llvm/IR/SVML.inc" HAVE_SVML) if(HAVE_SVML) message(STATUS "SVML found") add_definitions(-DHAVE_SVML -DLLVM_ENABLE_PROJECTS="lld") - add_library(lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) + add_library(lld SHARED lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) else() message(STATUS "SVML not found") endif() From 0ad8163e26e069aff8b3e03ff50777df394aa577 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Fri, 13 Jan 2023 00:52:56 -0500 Subject: [PATCH 055/158] I don't even know anymore --- ffi/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index f4c7920f4..7d81010ef 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -32,7 +32,7 @@ CHECK_INCLUDE_FILES("llvm/IR/SVML.inc" HAVE_SVML) if(HAVE_SVML) message(STATUS "SVML found") add_definitions(-DHAVE_SVML -DLLVM_ENABLE_PROJECTS="lld") - add_library(lld SHARED lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) + add_library(random_name SHARED lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) else() message(STATUS "SVML not found") endif() From af1ceba9dc84f176a72210c86ff88ac48aca1a37 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 14 Jan 2023 20:32:06 -0500 Subject: [PATCH 056/158] going to test if this works --- buildscripts/azure/azure-windows.yml | 2 +- ffi/CMakeLists.txt | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/buildscripts/azure/azure-windows.yml b/buildscripts/azure/azure-windows.yml index e51f39fc9..fcdc1cba2 100644 --- a/buildscripts/azure/azure-windows.yml +++ b/buildscripts/azure/azure-windows.yml @@ -31,7 +31,7 @@ jobs: call conda remove --all -q -y -n %CONDA_ENV% call conda create -n %CONDA_ENV% -q -y python=%PYTHON% cmake call activate %CONDA_ENV% - call conda install -y -q -c numba llvmdev="11.*" + call conda install -y -q -c numba llvmdev="11.*" lld="11.*" displayName: 'Before Install' - script: | diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index 7d81010ef..7c5428698 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -32,7 +32,7 @@ CHECK_INCLUDE_FILES("llvm/IR/SVML.inc" HAVE_SVML) if(HAVE_SVML) message(STATUS "SVML found") add_definitions(-DHAVE_SVML -DLLVM_ENABLE_PROJECTS="lld") - add_library(random_name SHARED lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) + # link_libraries(lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) else() message(STATUS "SVML not found") endif() @@ -57,6 +57,7 @@ else() message("NOT ENV LLVMLITE_SHARED") message(${LLVM_AVAILABLE_LIBS}) message("------------------------") + # set(llvm_libs ${LLVM_AVAILABLE_LIBS}) set(llvm_libs ${LLVM_AVAILABLE_LIBS}) endif() From 641f53577b4ad48b8c40e784797ab1b0185084a4 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 14 Jan 2023 21:18:46 -0500 Subject: [PATCH 057/158] This works locally --- buildscripts/azure/azure-windows.yml | 2 +- ffi/CMakeLists.txt | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/buildscripts/azure/azure-windows.yml b/buildscripts/azure/azure-windows.yml index fcdc1cba2..e51f39fc9 100644 --- a/buildscripts/azure/azure-windows.yml +++ b/buildscripts/azure/azure-windows.yml @@ -31,7 +31,7 @@ jobs: call conda remove --all -q -y -n %CONDA_ENV% call conda create -n %CONDA_ENV% -q -y python=%PYTHON% cmake call activate %CONDA_ENV% - call conda install -y -q -c numba llvmdev="11.*" lld="11.*" + call conda install -y -q -c numba llvmdev="11.*" displayName: 'Before Install' - script: | diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index 7c5428698..959249f98 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -16,6 +16,8 @@ find_package(LLD REQUIRED CONFIG) message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") +message(STATUS "Found LLD ${LLD_PACKAGE_VERSION}") +message(STATUS "Using LLDConfig.cmake in: ${LLD_DIR}") # Set your project compile flags. # E.g. if using the C++ header files @@ -58,7 +60,7 @@ else() message(${LLVM_AVAILABLE_LIBS}) message("------------------------") # set(llvm_libs ${LLVM_AVAILABLE_LIBS}) - set(llvm_libs ${LLVM_AVAILABLE_LIBS}) + set(llvm_libs ${LLVM_AVAILABLE_LIBS} lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) endif() # Since LLVM 8 "OptRemarks" is built as a shared library only and also appears From cd3a691c19468c1b5e189813843ed2b5ef21ab83 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 14 Jan 2023 23:12:39 -0500 Subject: [PATCH 058/158] OMG I THINK I DID IT! --- ffi/CMakeLists.txt | 6 +++--- ffi/lld.cpp | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index 959249f98..870b3a07c 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -24,17 +24,17 @@ message(STATUS "Using LLDConfig.cmake in: ${LLD_DIR}") # you will need to enable C++11 support # for your compiler. -include_directories(${LLVM_INCLUDE_DIRS} ${LLD_INCLUDE_DIRS}) +include_directories(${LLVM_INCLUDE_DIRS}) add_definitions(${LLVM_DEFINITIONS}) # Look for SVML -set(CMAKE_REQUIRED_INCLUDES ${LLVM_INCLUDE_DIRS} ${LLD_INCLUDE_DIRS}) +set(CMAKE_REQUIRED_INCLUDES ${LLVM_INCLUDE_DIRS}) CHECK_INCLUDE_FILES("llvm/IR/SVML.inc" HAVE_SVML) if(HAVE_SVML) message(STATUS "SVML found") add_definitions(-DHAVE_SVML -DLLVM_ENABLE_PROJECTS="lld") - # link_libraries(lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) + # link_libraries(lldCommon lldCore lldDriver lldCOFF lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) else() message(STATUS "SVML not found") endif() diff --git a/ffi/lld.cpp b/ffi/lld.cpp index 75ad0c075..5a13668ca 100644 --- a/ffi/lld.cpp +++ b/ffi/lld.cpp @@ -4,22 +4,22 @@ extern "C" { -bool lld_main(int Argc, const char **Argv, const char **outstr) { +API_EXPORT(bool) lld_main(int Argc, const char **Argv, const char **outstr) { //InitLLVM X(Argc, Argv); std::string command_output; llvm::raw_string_ostream command_stream(command_output); std::vector Args(Argv, Argv + Argc); #if defined __linux__ || __unix__ - bool linker_output = !lld::elf::link(Args, false, command_stream, command_stream); + bool linker_output = lld::elf::link(Args, false, command_stream, command_stream); #elif defined(WIN32) || defined(_WIN32) || defined(__WIN32__) - bool linker_output = !lld::coff::link(Args, false, command_stream, command_stream); + bool linker_output = lld::coff::link(Args, false, command_stream, command_stream); #else - bool linker_output = !lld::macho::link(Args, false, command_stream, command_stream); + bool linker_output = lld::macho::link(Args, false, command_stream, command_stream); #endif *outstr = LLVMPY_CreateString(command_output.c_str()); - + return linker_output; } From beb0c22cb6b8b401527486bb9e988dbbe60abe5f Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sun, 15 Jan 2023 01:12:08 -0500 Subject: [PATCH 059/158] formatting --- ffi/CMakeLists.txt | 2 -- ffi/build.py | 2 +- ffi/lld.cpp | 22 ++++++++++++---------- llvmlite/binding/lld.py | 7 +++++-- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index 870b3a07c..33f3382f3 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -34,7 +34,6 @@ CHECK_INCLUDE_FILES("llvm/IR/SVML.inc" HAVE_SVML) if(HAVE_SVML) message(STATUS "SVML found") add_definitions(-DHAVE_SVML -DLLVM_ENABLE_PROJECTS="lld") - # link_libraries(lldCommon lldCore lldDriver lldCOFF lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) else() message(STATUS "SVML not found") endif() @@ -59,7 +58,6 @@ else() message("NOT ENV LLVMLITE_SHARED") message(${LLVM_AVAILABLE_LIBS}) message("------------------------") - # set(llvm_libs ${LLVM_AVAILABLE_LIBS}) set(llvm_libs ${LLVM_AVAILABLE_LIBS} lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) endif() diff --git a/ffi/build.py b/ffi/build.py index 1c01e2c7f..295a1badc 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -206,7 +206,7 @@ def main_posix(kind, library_ext): except NotImplementedError: default_makeopts = "" makeopts = os.environ.get('LLVMLITE_MAKEOPTS', default_makeopts).split() - subprocess.check_call(['make', '-f', makefile] + makeopts) #? should this also have `-B`? + subprocess.check_call(['make', '-f', makefile] + makeopts) shutil.copy('libllvmlite' + library_ext, target_dir) diff --git a/ffi/lld.cpp b/ffi/lld.cpp index 5a13668ca..df8b8b7db 100644 --- a/ffi/lld.cpp +++ b/ffi/lld.cpp @@ -5,22 +5,24 @@ extern "C" { API_EXPORT(bool) lld_main(int Argc, const char **Argv, const char **outstr) { - //InitLLVM X(Argc, Argv); + // InitLLVM X(Argc, Argv); std::string command_output; llvm::raw_string_ostream command_stream(command_output); std::vector Args(Argv, Argv + Argc); - #if defined __linux__ || __unix__ - bool linker_output = lld::elf::link(Args, false, command_stream, command_stream); - #elif defined(WIN32) || defined(_WIN32) || defined(__WIN32__) - bool linker_output = lld::coff::link(Args, false, command_stream, command_stream); - #else - bool linker_output = lld::macho::link(Args, false, command_stream, command_stream); - #endif +#if defined __linux__ || __unix__ + bool linker_output = + lld::elf::link(Args, false, command_stream, command_stream); +#elif defined(WIN32) || defined(_WIN32) || defined(__WIN32__) + bool linker_output = + lld::coff::link(Args, false, command_stream, command_stream); +#else + bool linker_output = + lld::macho::link(Args, false, command_stream, command_stream); +#endif *outstr = LLVMPY_CreateString(command_output.c_str()); - return linker_output; } -} // end extern "C" +} // end extern "C" \ No newline at end of file diff --git a/llvmlite/binding/lld.py b/llvmlite/binding/lld.py index 6878c4e61..34f4ec660 100644 --- a/llvmlite/binding/lld.py +++ b/llvmlite/binding/lld.py @@ -39,7 +39,8 @@ def lld_runner(command: str): '''creates lld functions while still allowing docstrings for users to see.''' def wrapped(output: str, objects: List[str], args: List[str] = []) -> str: ''' - runs the command "{platform's lld command} -o {output-file} {*input_files} {*args}" + runs the command: + "{platform's lld command} -o {output-file} {*input_files} {*args}" output: output file as a str object: a list of input .o files as strings args: additional arguments for the command @@ -56,7 +57,9 @@ def wrapped(output: str, objects: List[str], args: List[str] = []) -> str: def lld_auto(output: str, objects: list[str], args: list[str] = [], add_extension = True) -> str: ''' - Automatically determines which lld function to run based on the hosts system. + Automatically determines which lld function + to run based on the hosts system. + Does not use `lld_wasm()` add_extension: adds `.exe` and other file endings automatically From 9743d140817258ed86c084f4851c4cab1169f98e Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sun, 15 Jan 2023 01:31:05 -0500 Subject: [PATCH 060/158] Make LLD test work on all platforms --- llvmlite/tests/test_binding.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/llvmlite/tests/test_binding.py b/llvmlite/tests/test_binding.py index bb44d9385..f25ad7a7d 100644 --- a/llvmlite/tests/test_binding.py +++ b/llvmlite/tests/test_binding.py @@ -2007,21 +2007,27 @@ def rel(): llvm.ffi.unregister_lock_callback(acq, rel) -@unittest.skipUnless(platform.machine().startswith('x86'), "only on x86") -class TestLLD_x86(BaseTest): +class TestLLD(BaseTest): def test_standalone_executable(self): + test_ir = """ + ;ModuleID = + target triple = "{triple}" + + define void @_start() {{ + %.3 = add i32 0, 12 + ret void + }} + """ with TemporaryDirectory() as tmpdir: objfile = os.path.join(tmpdir, "test1.o") binfile = os.path.join(tmpdir, "test1") target_machine = self.target_machine(jit=False) - mod = llvm.parse_assembly(asm_lld_executable) + mod = self.module(test_ir) mod.verify() with open(objfile, "wb") as o: o.write(target_machine.emit_object(mod)) - llvm.lld.lld_auto(binfile, [objfile]) - r = subprocess.call("%s" % binfile) - self.assertEqual(r, 42) - + print(llvm.lld.lld_auto(binfile, [objfile])) + subprocess.call("%s" % binfile) if __name__ == "__main__": unittest.main() From d9484c62973bbccc211eb37fa9c47ad432fd40f4 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sun, 15 Jan 2023 01:42:52 -0500 Subject: [PATCH 061/158] add lld.cpp to other make files --- ffi/Makefile.freebsd | 2 +- ffi/Makefile.osx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ffi/Makefile.freebsd b/ffi/Makefile.freebsd index ba727e331..84733f70d 100644 --- a/ffi/Makefile.freebsd +++ b/ffi/Makefile.freebsd @@ -11,7 +11,7 @@ LIBS = $(LLVM_LIBS) INCLUDE = core.h SRC = assembly.cpp bitcode.cpp core.cpp initfini.cpp module.cpp value.cpp \ executionengine.cpp transforms.cpp passmanagers.cpp targets.cpp dylib.cpp \ - linker.cpp object_file.cpp + linker.cpp object_file.cpp lld.cpp OUTPUT = libllvmlite.so all: $(OUTPUT) diff --git a/ffi/Makefile.osx b/ffi/Makefile.osx index bc192071e..37c58e552 100644 --- a/ffi/Makefile.osx +++ b/ffi/Makefile.osx @@ -8,7 +8,7 @@ LIBS = $(LLVM_LIBS) INCLUDE = core.h SRC = assembly.cpp bitcode.cpp core.cpp initfini.cpp module.cpp value.cpp \ executionengine.cpp transforms.cpp passmanagers.cpp targets.cpp dylib.cpp \ - linker.cpp object_file.cpp custom_passes.cpp + linker.cpp object_file.cpp custom_passes.cpp lld.cpp OUTPUT = libllvmlite.dylib MACOSX_DEPLOYMENT_TARGET ?= 10.9 From 1935042a8c3efdb04168cab129f2cd664a325614 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sun, 15 Jan 2023 12:49:31 -0500 Subject: [PATCH 062/158] Another test + command output included on lld exceptions --- llvmlite/binding/lld.py | 2 +- llvmlite/tests/test_binding.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/llvmlite/binding/lld.py b/llvmlite/binding/lld.py index 34f4ec660..202ed3b8c 100644 --- a/llvmlite/binding/lld.py +++ b/llvmlite/binding/lld.py @@ -30,7 +30,7 @@ def lld_main(lld_args) -> str: with ffi.OutputString() as outstr: r = ffi.lib.lld_main(len(lld_args), args, outstr) if not r: - raise Exception("lld_main() failed, error code: %d" % r) + raise Exception("lld_main() failed, error code: %d\nCommand Output: %s" % (r, str(outstr))) return str(outstr) diff --git a/llvmlite/tests/test_binding.py b/llvmlite/tests/test_binding.py index f25ad7a7d..da312e85b 100644 --- a/llvmlite/tests/test_binding.py +++ b/llvmlite/tests/test_binding.py @@ -2007,6 +2007,19 @@ def rel(): llvm.ffi.unregister_lock_callback(acq, rel) +class TestLLD_Help(BaseTest): + def test_lld_help(self): + system = platform.system() + + if system == "Linux": + llvm.lld.lld_main(["ld.lld", "--help"]) + elif system == "Windows": + llvm.lld.lld_main(["lld-link", "--help"]) + elif system == "Darwin": # Macos + llvm.lld.lld_main(["ld64.lld", "--help"]) + + +# @unittest.skipUnless(platform.machine().startswith('x86'), "only on x86") class TestLLD(BaseTest): def test_standalone_executable(self): test_ir = """ From 6a4f8ccedb0ae0c2eaaff116e22de7a468e36c61 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sun, 15 Jan 2023 13:20:05 -0500 Subject: [PATCH 063/158] Accounting for differences in windows lld. --- llvmlite/binding/lld.py | 6 +++--- llvmlite/tests/test_binding.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/llvmlite/binding/lld.py b/llvmlite/binding/lld.py index 202ed3b8c..1f08d4dbf 100644 --- a/llvmlite/binding/lld.py +++ b/llvmlite/binding/lld.py @@ -35,7 +35,7 @@ def lld_main(lld_args) -> str: return str(outstr) -def lld_runner(command: str): +def lld_runner(command: str, out_arg="-o"): '''creates lld functions while still allowing docstrings for users to see.''' def wrapped(output: str, objects: List[str], args: List[str] = []) -> str: ''' @@ -45,11 +45,11 @@ def wrapped(output: str, objects: List[str], args: List[str] = []) -> str: object: a list of input .o files as strings args: additional arguments for the command ''' - return lld_main([command, "-o", output, *objects, *args]) + return lld_main([command, out_arg, output, *objects, *args]) return wrapped -lld_windows = lld_runner("lld-link") +lld_windows = lld_runner("lld-link", out_arg="-out") lld_macos = lld_runner("ld64.lld") lld_linux = lld_runner("ld.lld") lld_wasm = lld_runner("wasm-ld") diff --git a/llvmlite/tests/test_binding.py b/llvmlite/tests/test_binding.py index da312e85b..f36ac818e 100644 --- a/llvmlite/tests/test_binding.py +++ b/llvmlite/tests/test_binding.py @@ -2014,7 +2014,7 @@ def test_lld_help(self): if system == "Linux": llvm.lld.lld_main(["ld.lld", "--help"]) elif system == "Windows": - llvm.lld.lld_main(["lld-link", "--help"]) + llvm.lld.lld_main(["lld-link", "-help"]) elif system == "Darwin": # Macos llvm.lld.lld_main(["ld64.lld", "--help"]) From 8d8d29db1f605c8378507fd618f17f7325d02915 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sun, 15 Jan 2023 14:46:13 -0500 Subject: [PATCH 064/158] Fix tests to work on windows and linux. --- ffi/lld.cpp | 3 ++- llvmlite/binding/lld.py | 9 +++++---- llvmlite/tests/test_binding.py | 36 ++++++++++++---------------------- 3 files changed, 20 insertions(+), 28 deletions(-) diff --git a/ffi/lld.cpp b/ffi/lld.cpp index df8b8b7db..b5b7ab3d8 100644 --- a/ffi/lld.cpp +++ b/ffi/lld.cpp @@ -11,8 +11,9 @@ API_EXPORT(bool) lld_main(int Argc, const char **Argv, const char **outstr) { std::vector Args(Argv, Argv + Argc); #if defined __linux__ || __unix__ + // command output needs to be inverted on linux for some reason bool linker_output = - lld::elf::link(Args, false, command_stream, command_stream); + !lld::elf::link(Args, false, command_stream, command_stream); #elif defined(WIN32) || defined(_WIN32) || defined(__WIN32__) bool linker_output = lld::coff::link(Args, false, command_stream, command_stream); diff --git a/llvmlite/binding/lld.py b/llvmlite/binding/lld.py index 1f08d4dbf..8cfa359cb 100644 --- a/llvmlite/binding/lld.py +++ b/llvmlite/binding/lld.py @@ -29,13 +29,13 @@ def lld_main(lld_args) -> str: args[i] = arg.encode() with ffi.OutputString() as outstr: r = ffi.lib.lld_main(len(lld_args), args, outstr) - if not r: + if r: raise Exception("lld_main() failed, error code: %d\nCommand Output: %s" % (r, str(outstr))) return str(outstr) -def lld_runner(command: str, out_arg="-o"): +def lld_runner(command: str, out_arg="-o "): '''creates lld functions while still allowing docstrings for users to see.''' def wrapped(output: str, objects: List[str], args: List[str] = []) -> str: ''' @@ -45,11 +45,12 @@ def wrapped(output: str, objects: List[str], args: List[str] = []) -> str: object: a list of input .o files as strings args: additional arguments for the command ''' - return lld_main([command, out_arg, output, *objects, *args]) + out_args = (out_arg + output).split(' ') + return lld_main([command, *out_args, *objects, *args]) return wrapped -lld_windows = lld_runner("lld-link", out_arg="-out") +lld_windows = lld_runner("lld-link", out_arg="/out:") lld_macos = lld_runner("ld64.lld") lld_linux = lld_runner("ld.lld") lld_wasm = lld_runner("wasm-ld") diff --git a/llvmlite/tests/test_binding.py b/llvmlite/tests/test_binding.py index f36ac818e..7f7f2ed53 100644 --- a/llvmlite/tests/test_binding.py +++ b/llvmlite/tests/test_binding.py @@ -2006,21 +2006,12 @@ def rel(): with self.assertRaises(ValueError): llvm.ffi.unregister_lock_callback(acq, rel) - -class TestLLD_Help(BaseTest): - def test_lld_help(self): - system = platform.system() - - if system == "Linux": - llvm.lld.lld_main(["ld.lld", "--help"]) - elif system == "Windows": - llvm.lld.lld_main(["lld-link", "-help"]) - elif system == "Darwin": # Macos - llvm.lld.lld_main(["ld64.lld", "--help"]) - - # @unittest.skipUnless(platform.machine().startswith('x86'), "only on x86") class TestLLD(BaseTest): + def target_machine(self, *, jit): + target = llvm.Target.from_default_triple() + return target.create_target_machine(jit=jit, codemodel="default") + def test_standalone_executable(self): test_ir = """ ;ModuleID = @@ -2031,16 +2022,15 @@ def test_standalone_executable(self): ret void }} """ - with TemporaryDirectory() as tmpdir: - objfile = os.path.join(tmpdir, "test1.o") - binfile = os.path.join(tmpdir, "test1") - target_machine = self.target_machine(jit=False) - mod = self.module(test_ir) - mod.verify() - with open(objfile, "wb") as o: - o.write(target_machine.emit_object(mod)) - print(llvm.lld.lld_auto(binfile, [objfile])) - subprocess.call("%s" % binfile) + objfile = "test1.o" + binfile = "test1" + target_machine = self.target_machine(jit=False) + mod = self.module(test_ir) + mod.verify() + with open(objfile, "wb") as o: + o.write(target_machine.emit_object(mod)) + print(llvm.lld.lld_auto(binfile, [objfile])) + subprocess.call("./%s" % binfile) if __name__ == "__main__": unittest.main() From e13cfe14c524b500906859165efc6b19a2a9b9f1 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sun, 15 Jan 2023 15:02:56 -0500 Subject: [PATCH 065/158] run tests with detail --- buildscripts/incremental/test.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/incremental/test.cmd b/buildscripts/incremental/test.cmd index 488961a5c..9adf8122e 100644 --- a/buildscripts/incremental/test.cmd +++ b/buildscripts/incremental/test.cmd @@ -1,4 +1,4 @@ call activate %CONDA_ENV% -python runtests.py -v +python runtests.py From b42d0cd1a9e7309caa9bfa135ce174e086ea12e9 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sun, 15 Jan 2023 15:16:29 -0500 Subject: [PATCH 066/158] fix error? --- ffi/lld.cpp | 6 +++--- llvmlite/binding/lld.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ffi/lld.cpp b/ffi/lld.cpp index b5b7ab3d8..7c2949748 100644 --- a/ffi/lld.cpp +++ b/ffi/lld.cpp @@ -10,16 +10,16 @@ API_EXPORT(bool) lld_main(int Argc, const char **Argv, const char **outstr) { llvm::raw_string_ostream command_stream(command_output); std::vector Args(Argv, Argv + Argc); + // command output needs to be inverted for some reason #if defined __linux__ || __unix__ - // command output needs to be inverted on linux for some reason bool linker_output = !lld::elf::link(Args, false, command_stream, command_stream); #elif defined(WIN32) || defined(_WIN32) || defined(__WIN32__) bool linker_output = - lld::coff::link(Args, false, command_stream, command_stream); + !lld::coff::link(Args, false, command_stream, command_stream); #else bool linker_output = - lld::macho::link(Args, false, command_stream, command_stream); + !lld::macho::link(Args, false, command_stream, command_stream); #endif *outstr = LLVMPY_CreateString(command_output.c_str()); diff --git a/llvmlite/binding/lld.py b/llvmlite/binding/lld.py index 8cfa359cb..f35b2a70d 100644 --- a/llvmlite/binding/lld.py +++ b/llvmlite/binding/lld.py @@ -30,13 +30,13 @@ def lld_main(lld_args) -> str: with ffi.OutputString() as outstr: r = ffi.lib.lld_main(len(lld_args), args, outstr) if r: - raise Exception("lld_main() failed, error code: %d\nCommand Output: %s" % (r, str(outstr))) + raise Exception("lld_main() failed, error code: \ + %d\nCommand Output: %s" % (r, str(outstr))) return str(outstr) def lld_runner(command: str, out_arg="-o "): - '''creates lld functions while still allowing docstrings for users to see.''' def wrapped(output: str, objects: List[str], args: List[str] = []) -> str: ''' runs the command: From 95c9e85769bd4cbafd08fd7c5a8597740fc5d51c Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sun, 15 Jan 2023 17:36:43 -0500 Subject: [PATCH 067/158] fix premature exit on windows --- ffi/lld.cpp | 9 --------- llvmlite/binding/lld.py | 6 +++--- llvmlite/tests/test_binding.py | 15 ++++++++++----- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/ffi/lld.cpp b/ffi/lld.cpp index 7c2949748..810d97d35 100644 --- a/ffi/lld.cpp +++ b/ffi/lld.cpp @@ -10,17 +10,8 @@ API_EXPORT(bool) lld_main(int Argc, const char **Argv, const char **outstr) { llvm::raw_string_ostream command_stream(command_output); std::vector Args(Argv, Argv + Argc); - // command output needs to be inverted for some reason -#if defined __linux__ || __unix__ bool linker_output = !lld::elf::link(Args, false, command_stream, command_stream); -#elif defined(WIN32) || defined(_WIN32) || defined(__WIN32__) - bool linker_output = - !lld::coff::link(Args, false, command_stream, command_stream); -#else - bool linker_output = - !lld::macho::link(Args, false, command_stream, command_stream); -#endif *outstr = LLVMPY_CreateString(command_output.c_str()); return linker_output; diff --git a/llvmlite/binding/lld.py b/llvmlite/binding/lld.py index f35b2a70d..8056b7a26 100644 --- a/llvmlite/binding/lld.py +++ b/llvmlite/binding/lld.py @@ -31,8 +31,8 @@ def lld_main(lld_args) -> str: r = ffi.lib.lld_main(len(lld_args), args, outstr) if r: raise Exception("lld_main() failed, error code: \ - %d\nCommand Output: %s" % (r, str(outstr))) - + %d\nCommand Output: %s" % (r, str(outstr))) + return str(outstr) @@ -50,7 +50,7 @@ def wrapped(output: str, objects: List[str], args: List[str] = []) -> str: return wrapped -lld_windows = lld_runner("lld-link", out_arg="/out:") +lld_windows = lld_runner("lld-link", out_arg="-o ") lld_macos = lld_runner("ld64.lld") lld_linux = lld_runner("ld.lld") lld_wasm = lld_runner("wasm-ld") diff --git a/llvmlite/tests/test_binding.py b/llvmlite/tests/test_binding.py index 7f7f2ed53..78dcbcb87 100644 --- a/llvmlite/tests/test_binding.py +++ b/llvmlite/tests/test_binding.py @@ -2008,10 +2008,6 @@ def rel(): # @unittest.skipUnless(platform.machine().startswith('x86'), "only on x86") class TestLLD(BaseTest): - def target_machine(self, *, jit): - target = llvm.Target.from_default_triple() - return target.create_target_machine(jit=jit, codemodel="default") - def test_standalone_executable(self): test_ir = """ ;ModuleID = @@ -2030,7 +2026,16 @@ def test_standalone_executable(self): with open(objfile, "wb") as o: o.write(target_machine.emit_object(mod)) print(llvm.lld.lld_auto(binfile, [objfile])) - subprocess.call("./%s" % binfile) + system = platform.system() + + if system == "Linux": + # has no standard file extension + self.assertTrue(os.path.exists(binfile)) + elif system == "Windows": + self.assertTrue(os.path.exists(f"{binfile}.exe")) + elif system == "Darwin": # Macos + self.assertTrue(os.path.exists(f"{binfile}.app")) + if __name__ == "__main__": unittest.main() From 626429b73c2def0e5f1eab786f2ccbd3b508d29b Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sun, 15 Jan 2023 18:26:18 -0500 Subject: [PATCH 068/158] Reduce the amount of needed llvmlibs --- ffi/CMakeLists.txt | 4 ++-- ffi/build.py | 2 +- llvmlite/binding/lld.py | 12 +++++------- llvmlite/tests/test_binding.py | 4 ++-- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index 33f3382f3..fe101a733 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -53,12 +53,12 @@ add_library(llvmlite SHARED assembly.cpp bitcode.cpp core.cpp initfini.cpp # BROKEN: llvm_map_components_to_libnames(llvm_libs all) if ($ENV{LLVMLITE_SHARED}) message("ENV LLVMLITE_SHARED") - set(llvm_libs lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML LLVM) + set(llvm_libs lldCommon lldCore lldDriver lldELF lldReaderWriter lldWasm lldYAML LLVM) else() message("NOT ENV LLVMLITE_SHARED") message(${LLVM_AVAILABLE_LIBS}) message("------------------------") - set(llvm_libs ${LLVM_AVAILABLE_LIBS} lldCOFF lldCommon lldCore lldDriver lldELF lldMachO lldMinGW lldReaderWriter lldWasm lldYAML) + set(llvm_libs ${LLVM_AVAILABLE_LIBS} lldCommon lldCore lldDriver lldELF lldReaderWriter lldWasm lldYAML) endif() # Since LLVM 8 "OptRemarks" is built as a shared library only and also appears diff --git a/ffi/build.py b/ffi/build.py index 295a1badc..9f6dbe67d 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -175,7 +175,7 @@ def main_posix(kind, library_ext): libs = run_llvm_config(llvm_config, "--system-libs --libs all".split()) # Normalize whitespace (trim newlines) os.environ['LLVM_LIBS'] = \ - '-llldCOFF -llldCommon -llldCore -llldDriver -llldELF -llldMachO -llldMinGW -llldReaderWriter -llldWasm -llldYAML ' + \ + '-llldCommon -llldCore -llldDriver -llldELF -llldReaderWriter -llldWasm -llldYAML ' + \ ' '.join(libs.split()) cxxflags = run_llvm_config(llvm_config, ["--cxxflags"]) diff --git a/llvmlite/binding/lld.py b/llvmlite/binding/lld.py index 8056b7a26..1a141c211 100644 --- a/llvmlite/binding/lld.py +++ b/llvmlite/binding/lld.py @@ -30,13 +30,12 @@ def lld_main(lld_args) -> str: with ffi.OutputString() as outstr: r = ffi.lib.lld_main(len(lld_args), args, outstr) if r: - raise Exception("lld_main() failed, error code: \ - %d\nCommand Output: %s" % (r, str(outstr))) - + raise Exception("lld_main() failed, error code: %d\ + \nCommand Output: %s" % (r, str(outstr))) return str(outstr) -def lld_runner(command: str, out_arg="-o "): +def lld_runner(command: str): def wrapped(output: str, objects: List[str], args: List[str] = []) -> str: ''' runs the command: @@ -45,12 +44,11 @@ def wrapped(output: str, objects: List[str], args: List[str] = []) -> str: object: a list of input .o files as strings args: additional arguments for the command ''' - out_args = (out_arg + output).split(' ') - return lld_main([command, *out_args, *objects, *args]) + return lld_main([command, '-o', output, *objects, *args]) return wrapped -lld_windows = lld_runner("lld-link", out_arg="-o ") +lld_windows = lld_runner("lld-link") lld_macos = lld_runner("ld64.lld") lld_linux = lld_runner("ld.lld") lld_wasm = lld_runner("wasm-ld") diff --git a/llvmlite/tests/test_binding.py b/llvmlite/tests/test_binding.py index 78dcbcb87..f014e5e3f 100644 --- a/llvmlite/tests/test_binding.py +++ b/llvmlite/tests/test_binding.py @@ -2006,7 +2006,7 @@ def rel(): with self.assertRaises(ValueError): llvm.ffi.unregister_lock_callback(acq, rel) -# @unittest.skipUnless(platform.machine().startswith('x86'), "only on x86") + class TestLLD(BaseTest): def test_standalone_executable(self): test_ir = """ @@ -2035,7 +2035,7 @@ def test_standalone_executable(self): self.assertTrue(os.path.exists(f"{binfile}.exe")) elif system == "Darwin": # Macos self.assertTrue(os.path.exists(f"{binfile}.app")) - + if __name__ == "__main__": unittest.main() From d8bf405ca48edae2f8a8148f02ac576f7e549f68 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sun, 15 Jan 2023 23:46:21 -0500 Subject: [PATCH 069/158] Docs and formatting --- docs/source/user-guide/binding/lld.rst | 82 ++++++++++++++++++++++++++ llvmlite/binding/lld.py | 6 +- 2 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 docs/source/user-guide/binding/lld.rst diff --git a/docs/source/user-guide/binding/lld.rst b/docs/source/user-guide/binding/lld.rst new file mode 100644 index 000000000..cdd451463 --- /dev/null +++ b/docs/source/user-guide/binding/lld.rst @@ -0,0 +1,82 @@ +================= +Linking with LLD +================= + +.. currentmodule:: llvmlite.binding + +The LLD linker is automatically built into llvmlite to provide easy-to-use, cross-platform linking. + +Known limitations +================== + +Currently, only ``lld::elf::link`` is used and thus COFF and MACHO object files will **not** link. + +The following llvm drivers are usable from :function:`lld_main` + +* ld.lld (Linux) +* ld64.lld (macos/darwin) +* lld-link (windows) +* wasm-ld (Web Assembly) + +Functions +========== + +* .. function:: lld_main(args) + + * ``args`` is a list of strings passed to the lld linker as arguments + * returns the output of the specific lld command being run. If there is an error, an ``Exception`` will be thrown. + + example: ``binding.lld_main(["ld.lld", "--help"])`` + + +* .. function:: lld_linux(ouput, objects, args=[]) + + * ``output`` is the name of the output file as a **string** + * ``objects`` is a list of object files' names as **strings**. + * ``args`` is a list of strings passed to the lld linker as arguments (at the end of the command) + * returns the output of lld_main for the given arguments + + Links given object files into an executable using ``lld_main()`` + + +* .. function:: lld_windows(ouput, objects, args=[]) + + * ``output`` is the name of the output file as a **string** + * ``objects`` is a list of object files' names as **strings**. + * ``args`` is a list of strings passed to the lld linker as arguments (at the end of the command) + * returns the output of lld_main for the given arguments + + Links given object files into an executable using ``lld_main()`` + + +* .. function:: lld_macos(ouput, objects, args=[]) + + * ``output`` is the name of the output file as a **string** + * ``objects`` is a list of object files' names as **strings**. + * ``args`` is a list of strings passed to the lld linker as arguments (at the end of the command) + * returns the output of lld_main for the given arguments + + Links given object files into an executable using ``lld_main()`` + + +* .. function:: lld_wasm(ouput, objects, args=[]) + + * ``output`` is the name of the output file as a **string** + * ``objects`` is a list of object files' names as **strings**. + * ``args`` is a list of strings passed to the lld linker as arguments (at the end of the command) + * returns the output of lld_main for the given arguments + + Links given object files into an executable using ``lld_main()`` + + +* .. function:: lld_auto(ouput, objects, args=[], add_extension=True) + + * ``output`` is the name of the output file as a **string** + * ``objects`` is a list of object files' names as **strings**. + * ``args`` is a list of strings passed to the lld linker as arguments (at the end of the command) + * ``add_extension`` should this function automatically add **.exe** and **.app** for windows and macos targets + * returns the output of lld_main for the given arguments + + Automatically determines which function to use given the host operating system. + This will only use the ``lld_linux``, ``lld_windows``, and ``lld_macos`` functions. + Creates and exception if the host operating system isn't ``Darwin``, ``Windows``, or ``Linux`` \ No newline at end of file diff --git a/llvmlite/binding/lld.py b/llvmlite/binding/lld.py index 1a141c211..439cac15a 100644 --- a/llvmlite/binding/lld.py +++ b/llvmlite/binding/lld.py @@ -31,7 +31,7 @@ def lld_main(lld_args) -> str: r = ffi.lib.lld_main(len(lld_args), args, outstr) if r: raise Exception("lld_main() failed, error code: %d\ - \nCommand Output: %s" % (r, str(outstr))) + \nCommand Output: %s" % (r, str(outstr))) return str(outstr) @@ -54,7 +54,9 @@ def wrapped(output: str, objects: List[str], args: List[str] = []) -> str: lld_wasm = lld_runner("wasm-ld") -def lld_auto(output: str, objects: list[str], args: list[str] = [], add_extension = True) -> str: +def lld_auto(output: str, objects: list[str], + args: list[str] = [], + add_extension=True) -> str: ''' Automatically determines which lld function to run based on the hosts system. From 17c11f35bfdf26821c29f7d296d9dd0b6e1d3fd1 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sun, 15 Jan 2023 23:50:01 -0500 Subject: [PATCH 070/158] Forgotten file from previous commit --- docs/source/user-guide/binding/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/user-guide/binding/index.rst b/docs/source/user-guide/binding/index.rst index 5199ef149..62053d8f3 100644 --- a/docs/source/user-guide/binding/index.rst +++ b/docs/source/user-guide/binding/index.rst @@ -32,5 +32,6 @@ implement Numba_'s JIT compiler. analysis-utilities pass_timings misc + lld examples From 86534f42504efc7a1c79098dc54643eb945b15d7 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sun, 15 Jan 2023 23:56:23 -0500 Subject: [PATCH 071/158] spelling and formatting --- docs/source/user-guide/binding/lld.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/source/user-guide/binding/lld.rst b/docs/source/user-guide/binding/lld.rst index cdd451463..6246a0b0f 100644 --- a/docs/source/user-guide/binding/lld.rst +++ b/docs/source/user-guide/binding/lld.rst @@ -11,7 +11,7 @@ Known limitations Currently, only ``lld::elf::link`` is used and thus COFF and MACHO object files will **not** link. -The following llvm drivers are usable from :function:`lld_main` +The following llvm drivers are usable from :func:`lld_main` * ld.lld (Linux) * ld64.lld (macos/darwin) @@ -24,12 +24,12 @@ Functions * .. function:: lld_main(args) * ``args`` is a list of strings passed to the lld linker as arguments - * returns the output of the specific lld command being run. If there is an error, an ``Exception`` will be thrown. + * returns the output of the specific lld command being run. If there is an error, an :exc:`Exception` will be thrown. example: ``binding.lld_main(["ld.lld", "--help"])`` -* .. function:: lld_linux(ouput, objects, args=[]) +* .. function:: lld_linux(output_file, objects, args=[]) * ``output`` is the name of the output file as a **string** * ``objects`` is a list of object files' names as **strings**. @@ -39,7 +39,7 @@ Functions Links given object files into an executable using ``lld_main()`` -* .. function:: lld_windows(ouput, objects, args=[]) +* .. function:: lld_windows(output_file, objects, args=[]) * ``output`` is the name of the output file as a **string** * ``objects`` is a list of object files' names as **strings**. @@ -49,7 +49,7 @@ Functions Links given object files into an executable using ``lld_main()`` -* .. function:: lld_macos(ouput, objects, args=[]) +* .. function:: lld_macos(output_file, objects, args=[]) * ``output`` is the name of the output file as a **string** * ``objects`` is a list of object files' names as **strings**. @@ -59,7 +59,7 @@ Functions Links given object files into an executable using ``lld_main()`` -* .. function:: lld_wasm(ouput, objects, args=[]) +* .. function:: lld_wasm(output_file, objects, args=[]) * ``output`` is the name of the output file as a **string** * ``objects`` is a list of object files' names as **strings**. @@ -69,7 +69,7 @@ Functions Links given object files into an executable using ``lld_main()`` -* .. function:: lld_auto(ouput, objects, args=[], add_extension=True) +* .. function:: lld_auto(output_file, objects, args=[], add_extension=True) * ``output`` is the name of the output file as a **string** * ``objects`` is a list of object files' names as **strings**. @@ -79,4 +79,4 @@ Functions Automatically determines which function to use given the host operating system. This will only use the ``lld_linux``, ``lld_windows``, and ``lld_macos`` functions. - Creates and exception if the host operating system isn't ``Darwin``, ``Windows``, or ``Linux`` \ No newline at end of file + Throws :exc:`Exception` if the host operating system isn't ``Darwin``, ``Windows``, or ``Linux`` \ No newline at end of file From 22bb4c206442ea681e78a72f7c54f9eadd4af488 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 16 Jan 2023 00:00:44 -0500 Subject: [PATCH 072/158] More concise docs --- docs/source/user-guide/binding/lld.rst | 29 +++++--------------------- 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/docs/source/user-guide/binding/lld.rst b/docs/source/user-guide/binding/lld.rst index 6246a0b0f..6fba2959b 100644 --- a/docs/source/user-guide/binding/lld.rst +++ b/docs/source/user-guide/binding/lld.rst @@ -31,49 +31,30 @@ Functions * .. function:: lld_linux(output_file, objects, args=[]) - * ``output`` is the name of the output file as a **string** * ``objects`` is a list of object files' names as **strings**. * ``args`` is a list of strings passed to the lld linker as arguments (at the end of the command) * returns the output of lld_main for the given arguments - Links given object files into an executable using ``lld_main()`` + Links given object files into an executable using :func:`lld_main()` * .. function:: lld_windows(output_file, objects, args=[]) - * ``output`` is the name of the output file as a **string** - * ``objects`` is a list of object files' names as **strings**. - * ``args`` is a list of strings passed to the lld linker as arguments (at the end of the command) - * returns the output of lld_main for the given arguments - - Links given object files into an executable using ``lld_main()`` + Link for Windows target * .. function:: lld_macos(output_file, objects, args=[]) - * ``output`` is the name of the output file as a **string** - * ``objects`` is a list of object files' names as **strings**. - * ``args`` is a list of strings passed to the lld linker as arguments (at the end of the command) - * returns the output of lld_main for the given arguments - - Links given object files into an executable using ``lld_main()`` + Link for Macos target * .. function:: lld_wasm(output_file, objects, args=[]) - * ``output`` is the name of the output file as a **string** - * ``objects`` is a list of object files' names as **strings**. - * ``args`` is a list of strings passed to the lld linker as arguments (at the end of the command) - * returns the output of lld_main for the given arguments - - Links given object files into an executable using ``lld_main()`` + Link for wasm target * .. function:: lld_auto(output_file, objects, args=[], add_extension=True) - - * ``output`` is the name of the output file as a **string** - * ``objects`` is a list of object files' names as **strings**. - * ``args`` is a list of strings passed to the lld linker as arguments (at the end of the command) + * ``add_extension`` should this function automatically add **.exe** and **.app** for windows and macos targets * returns the output of lld_main for the given arguments From 488a382cf80ddf3d4d9f09455eaaffc1d02a47b1 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 16 Jan 2023 00:13:22 -0500 Subject: [PATCH 073/158] remove unused import --- llvmlite/tests/test_binding.py | 1 - 1 file changed, 1 deletion(-) diff --git a/llvmlite/tests/test_binding.py b/llvmlite/tests/test_binding.py index f014e5e3f..1f2393cab 100644 --- a/llvmlite/tests/test_binding.py +++ b/llvmlite/tests/test_binding.py @@ -8,7 +8,6 @@ import re import subprocess import sys -from tempfile import TemporaryDirectory import unittest from contextlib import contextmanager from tempfile import mkstemp From 576dccb0bf398af55723b635210fb66967709c56 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 16 Jan 2023 11:38:54 -0500 Subject: [PATCH 074/158] Make Type Hinting compatible with older python --- llvmlite/binding/lld.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvmlite/binding/lld.py b/llvmlite/binding/lld.py index 439cac15a..dba3f8aa3 100644 --- a/llvmlite/binding/lld.py +++ b/llvmlite/binding/lld.py @@ -54,8 +54,8 @@ def wrapped(output: str, objects: List[str], args: List[str] = []) -> str: lld_wasm = lld_runner("wasm-ld") -def lld_auto(output: str, objects: list[str], - args: list[str] = [], +def lld_auto(output: str, objects: List[str], + args: List[str] = [], add_extension=True) -> str: ''' Automatically determines which lld function From d03f6a244490d803e6c718a69fd971bbe521234e Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 16 Jan 2023 13:02:19 -0500 Subject: [PATCH 075/158] trying something to make osx work --- ffi/Makefile.osx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/Makefile.osx b/ffi/Makefile.osx index 37c58e552..1f28c9455 100644 --- a/ffi/Makefile.osx +++ b/ffi/Makefile.osx @@ -13,7 +13,7 @@ OUTPUT = libllvmlite.dylib MACOSX_DEPLOYMENT_TARGET ?= 10.9 all: $(SRC) $(INCLUDE) - MACOSX_DEPLOYMENT_TARGET=$(MACOSX_DEPLOYMENT_TARGET) $(CXX) -dynamiclib $(CXXFLAGS) $(SRC) -o $(OUTPUT) $(LDFLAGS) $(LIBS) + MACOSX_DEPLOYMENT_TARGET=$(MACOSX_DEPLOYMENT_TARGET) $(CXX) -dynamiclib -undefined dynamic_lookup -flat_namespace $(CXXFLAGS) $(SRC) -o $(OUTPUT) $(LDFLAGS) $(LIBS) clean: rm -rf test From 3c23bbdaff33605932ef766abdff533e09d3e2fe Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 16 Jan 2023 19:38:58 -0500 Subject: [PATCH 076/158] I think this should fix mac builds --- ffi/Makefile.osx | 2 +- ffi/lld.cpp | 2 +- llvmlite/binding/lld.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ffi/Makefile.osx b/ffi/Makefile.osx index 1f28c9455..37c58e552 100644 --- a/ffi/Makefile.osx +++ b/ffi/Makefile.osx @@ -13,7 +13,7 @@ OUTPUT = libllvmlite.dylib MACOSX_DEPLOYMENT_TARGET ?= 10.9 all: $(SRC) $(INCLUDE) - MACOSX_DEPLOYMENT_TARGET=$(MACOSX_DEPLOYMENT_TARGET) $(CXX) -dynamiclib -undefined dynamic_lookup -flat_namespace $(CXXFLAGS) $(SRC) -o $(OUTPUT) $(LDFLAGS) $(LIBS) + MACOSX_DEPLOYMENT_TARGET=$(MACOSX_DEPLOYMENT_TARGET) $(CXX) -dynamiclib $(CXXFLAGS) $(SRC) -o $(OUTPUT) $(LDFLAGS) $(LIBS) clean: rm -rf test diff --git a/ffi/lld.cpp b/ffi/lld.cpp index 810d97d35..8e1c9ff6b 100644 --- a/ffi/lld.cpp +++ b/ffi/lld.cpp @@ -4,7 +4,7 @@ extern "C" { -API_EXPORT(bool) lld_main(int Argc, const char **Argv, const char **outstr) { +API_EXPORT(bool) LLVMPY_lld_main(int Argc, const char **Argv, const char **outstr) { // InitLLVM X(Argc, Argv); std::string command_output; llvm::raw_string_ostream command_stream(command_output); diff --git a/llvmlite/binding/lld.py b/llvmlite/binding/lld.py index dba3f8aa3..34ce2ee4a 100644 --- a/llvmlite/binding/lld.py +++ b/llvmlite/binding/lld.py @@ -5,8 +5,8 @@ from llvmlite.binding import ffi -ffi.lib.lld_main.restype = c_bool -ffi.lib.lld_main.argtypes = [c_int, POINTER(c_char_p), POINTER(c_char_p)] +ffi.lib.LLVMPY_lld_main.restype = c_bool +ffi.lib.LLVMPY_lld_main.argtypes = [c_int, POINTER(c_char_p), POINTER(c_char_p)] def lld_main(lld_args) -> str: @@ -28,7 +28,7 @@ def lld_main(lld_args) -> str: for i, arg in enumerate(lld_args): args[i] = arg.encode() with ffi.OutputString() as outstr: - r = ffi.lib.lld_main(len(lld_args), args, outstr) + r = ffi.lib.LLVMPY_lld_main(len(lld_args), args, outstr) if r: raise Exception("lld_main() failed, error code: %d\ \nCommand Output: %s" % (r, str(outstr))) From c50e618e9f48a1ffc24e41b8cb5e5161655c01e1 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 16 Jan 2023 19:41:17 -0500 Subject: [PATCH 077/158] cpp formatting fix --- ffi/lld.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ffi/lld.cpp b/ffi/lld.cpp index 8e1c9ff6b..476d33dd2 100644 --- a/ffi/lld.cpp +++ b/ffi/lld.cpp @@ -4,7 +4,8 @@ extern "C" { -API_EXPORT(bool) LLVMPY_lld_main(int Argc, const char **Argv, const char **outstr) { +API_EXPORT(bool) +LLVMPY_lld_main(int Argc, const char **Argv, const char **outstr) { // InitLLVM X(Argc, Argv); std::string command_output; llvm::raw_string_ostream command_stream(command_output); From b8ec5f5788be84b94a65d6ded0e011ce307d3dcb Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 16 Jan 2023 20:01:30 -0500 Subject: [PATCH 078/158] Readd macos lld linker --- ffi/CMakeLists.txt | 4 ++-- ffi/build.py | 2 +- ffi/lld.cpp | 5 +++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index fe101a733..fec0400af 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -53,12 +53,12 @@ add_library(llvmlite SHARED assembly.cpp bitcode.cpp core.cpp initfini.cpp # BROKEN: llvm_map_components_to_libnames(llvm_libs all) if ($ENV{LLVMLITE_SHARED}) message("ENV LLVMLITE_SHARED") - set(llvm_libs lldCommon lldCore lldDriver lldELF lldReaderWriter lldWasm lldYAML LLVM) + set(llvm_libs lldCommon lldCore lldDriver lldELF lldMachO lldReaderWriter lldWasm lldYAML LLVM) else() message("NOT ENV LLVMLITE_SHARED") message(${LLVM_AVAILABLE_LIBS}) message("------------------------") - set(llvm_libs ${LLVM_AVAILABLE_LIBS} lldCommon lldCore lldDriver lldELF lldReaderWriter lldWasm lldYAML) + set(llvm_libs ${LLVM_AVAILABLE_LIBS} lldCommon lldCore lldDriver lldELF lldMachO lldReaderWriter lldWasm lldYAML) endif() # Since LLVM 8 "OptRemarks" is built as a shared library only and also appears diff --git a/ffi/build.py b/ffi/build.py index 9f6dbe67d..7a8e92512 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -175,7 +175,7 @@ def main_posix(kind, library_ext): libs = run_llvm_config(llvm_config, "--system-libs --libs all".split()) # Normalize whitespace (trim newlines) os.environ['LLVM_LIBS'] = \ - '-llldCommon -llldCore -llldDriver -llldELF -llldReaderWriter -llldWasm -llldYAML ' + \ + '-llldCommon -llldCore -llldDriver -llldELF -llldMachO -llldReaderWriter -llldWasm -llldYAML ' + \ ' '.join(libs.split()) cxxflags = run_llvm_config(llvm_config, ["--cxxflags"]) diff --git a/ffi/lld.cpp b/ffi/lld.cpp index 476d33dd2..e272ea3bb 100644 --- a/ffi/lld.cpp +++ b/ffi/lld.cpp @@ -11,8 +11,13 @@ LLVMPY_lld_main(int Argc, const char **Argv, const char **outstr) { llvm::raw_string_ostream command_stream(command_output); std::vector Args(Argv, Argv + Argc); +#ifdef __APPLE__ || __MACH__ + bool linker_output = + !lld::macho::link(Args, false, command_stream, command_stream); +#else bool linker_output = !lld::elf::link(Args, false, command_stream, command_stream); +#endif *outstr = LLVMPY_CreateString(command_output.c_str()); return linker_output; From 861180f297dc6d6cceff73f10283a1d87e7e7e2f Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 16 Jan 2023 20:09:48 -0500 Subject: [PATCH 079/158] actually use ifdef properly --- ffi/lld.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/lld.cpp b/ffi/lld.cpp index e272ea3bb..61ab2661b 100644 --- a/ffi/lld.cpp +++ b/ffi/lld.cpp @@ -11,7 +11,7 @@ LLVMPY_lld_main(int Argc, const char **Argv, const char **outstr) { llvm::raw_string_ostream command_stream(command_output); std::vector Args(Argv, Argv + Argc); -#ifdef __APPLE__ || __MACH__ +#ifdef defined(__APPLE__) || defined(__MACH__) bool linker_output = !lld::macho::link(Args, false, command_stream, command_stream); #else From ff1d02ca82e7211c0c042fc2c6c34812b915ffbc Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 16 Jan 2023 20:19:55 -0500 Subject: [PATCH 080/158] print statement for debugging --- llvmlite/tests/test_binding.py | 1 + 1 file changed, 1 insertion(+) diff --git a/llvmlite/tests/test_binding.py b/llvmlite/tests/test_binding.py index 1f2393cab..77816fd4c 100644 --- a/llvmlite/tests/test_binding.py +++ b/llvmlite/tests/test_binding.py @@ -2023,6 +2023,7 @@ def test_standalone_executable(self): mod = self.module(test_ir) mod.verify() with open(objfile, "wb") as o: + print(target_machine.emit_object(mod)) o.write(target_machine.emit_object(mod)) print(llvm.lld.lld_auto(binfile, [objfile])) system = platform.system() From 63859523d7262924d9ad069f581e772098265e94 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 16 Jan 2023 20:28:05 -0500 Subject: [PATCH 081/158] Another test print --- llvmlite/tests/test_binding.py | 1 + 1 file changed, 1 insertion(+) diff --git a/llvmlite/tests/test_binding.py b/llvmlite/tests/test_binding.py index 77816fd4c..4dd320b45 100644 --- a/llvmlite/tests/test_binding.py +++ b/llvmlite/tests/test_binding.py @@ -2022,6 +2022,7 @@ def test_standalone_executable(self): target_machine = self.target_machine(jit=False) mod = self.module(test_ir) mod.verify() + print(target_machine.triple) with open(objfile, "wb") as o: print(target_machine.emit_object(mod)) o.write(target_machine.emit_object(mod)) From 95557dd9a7014a40c6e4f65bac386843c2de4ea4 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 16 Jan 2023 20:43:16 -0500 Subject: [PATCH 082/158] add new "force_elf" option when creating target machine. --- ffi/CMakeLists.txt | 4 ++-- ffi/build.py | 2 +- ffi/lld.cpp | 5 ----- llvmlite/binding/targets.py | 8 ++++++-- llvmlite/tests/test_binding.py | 6 ++++-- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index fec0400af..19dd0d13b 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -53,12 +53,12 @@ add_library(llvmlite SHARED assembly.cpp bitcode.cpp core.cpp initfini.cpp # BROKEN: llvm_map_components_to_libnames(llvm_libs all) if ($ENV{LLVMLITE_SHARED}) message("ENV LLVMLITE_SHARED") - set(llvm_libs lldCommon lldCore lldDriver lldELF lldMachO lldReaderWriter lldWasm lldYAML LLVM) + set(llvm_libs lldCommon lldCore lldDriver lldELF lldReaderWriter lldWasm lldYAML LLVM) else() message("NOT ENV LLVMLITE_SHARED") message(${LLVM_AVAILABLE_LIBS}) message("------------------------") - set(llvm_libs ${LLVM_AVAILABLE_LIBS} lldCommon lldCore lldDriver lldELF lldMachO lldReaderWriter lldWasm lldYAML) + set(llvm_libs ${LLVM_AVAILABLE_LIBS} lldCommon lldCore lldELF lldMachO lldReaderWriter lldWasm lldYAML) endif() # Since LLVM 8 "OptRemarks" is built as a shared library only and also appears diff --git a/ffi/build.py b/ffi/build.py index 7a8e92512..9f6dbe67d 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -175,7 +175,7 @@ def main_posix(kind, library_ext): libs = run_llvm_config(llvm_config, "--system-libs --libs all".split()) # Normalize whitespace (trim newlines) os.environ['LLVM_LIBS'] = \ - '-llldCommon -llldCore -llldDriver -llldELF -llldMachO -llldReaderWriter -llldWasm -llldYAML ' + \ + '-llldCommon -llldCore -llldDriver -llldELF -llldReaderWriter -llldWasm -llldYAML ' + \ ' '.join(libs.split()) cxxflags = run_llvm_config(llvm_config, ["--cxxflags"]) diff --git a/ffi/lld.cpp b/ffi/lld.cpp index 61ab2661b..476d33dd2 100644 --- a/ffi/lld.cpp +++ b/ffi/lld.cpp @@ -11,13 +11,8 @@ LLVMPY_lld_main(int Argc, const char **Argv, const char **outstr) { llvm::raw_string_ostream command_stream(command_output); std::vector Args(Argv, Argv + Argc); -#ifdef defined(__APPLE__) || defined(__MACH__) - bool linker_output = - !lld::macho::link(Args, false, command_stream, command_stream); -#else bool linker_output = !lld::elf::link(Args, false, command_stream, command_stream); -#endif *outstr = LLVMPY_CreateString(command_output.c_str()); return linker_output; diff --git a/llvmlite/binding/targets.py b/llvmlite/binding/targets.py index a7e6ffdc3..72014ac52 100644 --- a/llvmlite/binding/targets.py +++ b/llvmlite/binding/targets.py @@ -218,7 +218,8 @@ def __str__(self): def create_target_machine(self, cpu='', features='', opt=2, reloc='default', codemodel='jitdefault', - printmc=False, jit=False, abiname=''): + printmc=False, jit=False, abiname='', + force_elf=False): """ Create a new TargetMachine for this target and the given options. @@ -231,6 +232,9 @@ def create_target_machine(self, cpu='', features='', The `jit` option should be set when the target-machine is to be used in a JIT engine. + The `force_elf` option will force the target object format to be ELF + instead of the target-machine's default. + The `abiname` option specifies the ABI. RISC-V targets with hard-float needs to pass the ABI name to LLVM. """ @@ -241,7 +245,7 @@ def create_target_machine(self, cpu='', features='', # MCJIT under Windows only supports ELF objects, see # http://lists.llvm.org/pipermail/llvm-dev/2013-December/068341.html # Note we still want to produce regular COFF files in AOT mode. - if os.name == 'nt' and codemodel == 'jitdefault': + if (os.name == 'nt' and codemodel == 'jitdefault') or force_elf: triple += '-elf' tm = ffi.lib.LLVMPY_CreateTargetMachine(self, _encode_string(triple), diff --git a/llvmlite/tests/test_binding.py b/llvmlite/tests/test_binding.py index 4dd320b45..893b574ef 100644 --- a/llvmlite/tests/test_binding.py +++ b/llvmlite/tests/test_binding.py @@ -2007,6 +2007,10 @@ def rel(): class TestLLD(BaseTest): + def target_machine(self, *, jit): + target = llvm.Target.from_default_triple() + return target.create_target_machine(jit=jit, force_elf=True) + def test_standalone_executable(self): test_ir = """ ;ModuleID = @@ -2022,9 +2026,7 @@ def test_standalone_executable(self): target_machine = self.target_machine(jit=False) mod = self.module(test_ir) mod.verify() - print(target_machine.triple) with open(objfile, "wb") as o: - print(target_machine.emit_object(mod)) o.write(target_machine.emit_object(mod)) print(llvm.lld.lld_auto(binfile, [objfile])) system = platform.system() From 31c633481b5b866d8b93ff5a9818a42b7e6ba5cf Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Thu, 19 Jan 2023 22:08:39 -0500 Subject: [PATCH 083/158] General fixes --- buildscripts/incremental/test.cmd | 2 +- docs/source/user-guide/binding/target-information.rst | 3 ++- ffi/CMakeLists.txt | 11 ++++------- ffi/build.py | 6 ++++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/buildscripts/incremental/test.cmd b/buildscripts/incremental/test.cmd index 9adf8122e..4a1f251ce 100644 --- a/buildscripts/incremental/test.cmd +++ b/buildscripts/incremental/test.cmd @@ -1,4 +1,4 @@ call activate %CONDA_ENV% -python runtests.py +python runtests.py -v \ No newline at end of file diff --git a/docs/source/user-guide/binding/target-information.rst b/docs/source/user-guide/binding/target-information.rst index 96376a072..02f8bb853 100644 --- a/docs/source/user-guide/binding/target-information.rst +++ b/docs/source/user-guide/binding/target-information.rst @@ -136,7 +136,7 @@ Classes * .. method:: create_target_machine(cpu='', features='', \ opt=2, reloc='default', codemodel='jitdefault', \ - abiname='') + abiname='', force_elf=False) Create a new :class:`TargetMachine` instance for this target and with the given options: @@ -148,6 +148,7 @@ Classes * *reloc* is the relocation model. * *codemodel* is the code model. * *abiname* is the name of the ABI. + * *force_elf* is an option that forces the target to use the ELF object format instead of the target's object format. The defaults for reloc and codemodel are appropriate for JIT compilation. diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index 19dd0d13b..77c5e3760 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -33,11 +33,12 @@ set(CMAKE_REQUIRED_INCLUDES ${LLVM_INCLUDE_DIRS}) CHECK_INCLUDE_FILES("llvm/IR/SVML.inc" HAVE_SVML) if(HAVE_SVML) message(STATUS "SVML found") - add_definitions(-DHAVE_SVML -DLLVM_ENABLE_PROJECTS="lld") + add_definitions(-DHAVE_SVML) else() message(STATUS "SVML not found") endif() +add_definitions(-DLLVM_ENABLE_PROJECTS="lld") # Define our shared library add_library(llvmlite SHARED assembly.cpp bitcode.cpp core.cpp initfini.cpp @@ -52,13 +53,9 @@ add_library(llvmlite SHARED assembly.cpp bitcode.cpp core.cpp initfini.cpp # Bug reported upstream at: https://bugs.llvm.org/show_bug.cgi?id=47003 # BROKEN: llvm_map_components_to_libnames(llvm_libs all) if ($ENV{LLVMLITE_SHARED}) - message("ENV LLVMLITE_SHARED") - set(llvm_libs lldCommon lldCore lldDriver lldELF lldReaderWriter lldWasm lldYAML LLVM) + set(llvm_libs ${lld_libs} LLVM) else() - message("NOT ENV LLVMLITE_SHARED") - message(${LLVM_AVAILABLE_LIBS}) - message("------------------------") - set(llvm_libs ${LLVM_AVAILABLE_LIBS} lldCommon lldCore lldELF lldMachO lldReaderWriter lldWasm lldYAML) + set(llvm_libs ${LLVM_AVAILABLE_LIBS} ${lld_libs}) endif() # Since LLVM 8 "OptRemarks" is built as a shared library only and also appears diff --git a/ffi/build.py b/ffi/build.py index 9f6dbe67d..5650112e7 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -20,6 +20,7 @@ target_dir = os.path.join(os.path.dirname(here_dir), 'llvmlite', 'binding') is_64bit = sys.maxsize >= 2**32 +lld_libs = "-llldCommon -llldCore -llldELF -llldReaderWriter -llldWasm -llldYAML" def try_cmake(cmake_dir, build_dir, generator, arch=None, toolkit=None): @@ -98,6 +99,7 @@ def main_windows(): if not os.path.exists(build_dir): os.mkdir(build_dir) try_cmake(here_dir, build_dir, *generator) + os.environ['lld_libs'] = lld_libs.strip('-l') subprocess.check_call(['cmake', '--build', build_dir, '--config', config]) shutil.copy(os.path.join(build_dir, config, 'llvmlite.dll'), target_dir) @@ -108,6 +110,7 @@ def main_posix_cmake(kind, library_ext): if not os.path.exists(build_dir): os.mkdir(build_dir) try_cmake(here_dir, build_dir, generator) + os.environ['lld_libs'] = lld_libs.strip('-l') subprocess.check_call(['cmake', '--build', build_dir, '--config', config]) shutil.copy(os.path.join(build_dir, 'libllvmlite' + library_ext), target_dir) @@ -175,8 +178,7 @@ def main_posix(kind, library_ext): libs = run_llvm_config(llvm_config, "--system-libs --libs all".split()) # Normalize whitespace (trim newlines) os.environ['LLVM_LIBS'] = \ - '-llldCommon -llldCore -llldDriver -llldELF -llldReaderWriter -llldWasm -llldYAML ' + \ - ' '.join(libs.split()) + lld_libs + ' '.join(libs.split()) cxxflags = run_llvm_config(llvm_config, ["--cxxflags"]) # on OSX cxxflags has null bytes at the end of the string, remove them From b4721deff62ac5ce03dd3e425412a4648b7ac43e Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Thu, 19 Jan 2023 22:14:09 -0500 Subject: [PATCH 084/158] add additional space in LLVM_LIBS --- ffi/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/build.py b/ffi/build.py index 5650112e7..a398410ad 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -178,7 +178,7 @@ def main_posix(kind, library_ext): libs = run_llvm_config(llvm_config, "--system-libs --libs all".split()) # Normalize whitespace (trim newlines) os.environ['LLVM_LIBS'] = \ - lld_libs + ' '.join(libs.split()) + f'{lld_libs} ' + ' '.join(libs.split()) cxxflags = run_llvm_config(llvm_config, ["--cxxflags"]) # on OSX cxxflags has null bytes at the end of the string, remove them From 090b478210185cda2b5961db6b01767709c64957 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Thu, 19 Jan 2023 22:18:29 -0500 Subject: [PATCH 085/158] use ENV{lld_libs} --- ffi/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ffi/CMakeLists.txt b/ffi/CMakeLists.txt index 77c5e3760..b757d6ee4 100755 --- a/ffi/CMakeLists.txt +++ b/ffi/CMakeLists.txt @@ -53,9 +53,9 @@ add_library(llvmlite SHARED assembly.cpp bitcode.cpp core.cpp initfini.cpp # Bug reported upstream at: https://bugs.llvm.org/show_bug.cgi?id=47003 # BROKEN: llvm_map_components_to_libnames(llvm_libs all) if ($ENV{LLVMLITE_SHARED}) - set(llvm_libs ${lld_libs} LLVM) + set(llvm_libs $ENV{lld_libs} LLVM) else() - set(llvm_libs ${LLVM_AVAILABLE_LIBS} ${lld_libs}) + set(llvm_libs ${LLVM_AVAILABLE_LIBS} $ENV{lld_libs}) endif() # Since LLVM 8 "OptRemarks" is built as a shared library only and also appears From 1f98b6019edb783babb8a340179d0330bdae018a Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Thu, 19 Jan 2023 22:27:12 -0500 Subject: [PATCH 086/158] set the environment variable BEFORE doing cmake --- ffi/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ffi/build.py b/ffi/build.py index a398410ad..80d3a65ed 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -98,8 +98,8 @@ def main_windows(): config = 'Release' if not os.path.exists(build_dir): os.mkdir(build_dir) - try_cmake(here_dir, build_dir, *generator) os.environ['lld_libs'] = lld_libs.strip('-l') + try_cmake(here_dir, build_dir, *generator) subprocess.check_call(['cmake', '--build', build_dir, '--config', config]) shutil.copy(os.path.join(build_dir, config, 'llvmlite.dll'), target_dir) @@ -109,8 +109,8 @@ def main_posix_cmake(kind, library_ext): config = 'Release' if not os.path.exists(build_dir): os.mkdir(build_dir) - try_cmake(here_dir, build_dir, generator) os.environ['lld_libs'] = lld_libs.strip('-l') + try_cmake(here_dir, build_dir, generator) subprocess.check_call(['cmake', '--build', build_dir, '--config', config]) shutil.copy(os.path.join(build_dir, 'libllvmlite' + library_ext), target_dir) From 678006d613861944ff2e40bba4b17e5a42565b44 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Thu, 19 Jan 2023 22:36:47 -0500 Subject: [PATCH 087/158] fix the removal of `-l` --- ffi/build.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ffi/build.py b/ffi/build.py index 80d3a65ed..9d47b53fc 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -33,6 +33,7 @@ def try_cmake(cmake_dir, build_dir, generator, arch=None, toolkit=None): args.append(cmake_dir) try: os.chdir(build_dir) + os.environ['lld_libs'] = lld_libs.replace("-l", "") print('Running:', ' '.join(args)) subprocess.check_call(args) finally: @@ -98,7 +99,6 @@ def main_windows(): config = 'Release' if not os.path.exists(build_dir): os.mkdir(build_dir) - os.environ['lld_libs'] = lld_libs.strip('-l') try_cmake(here_dir, build_dir, *generator) subprocess.check_call(['cmake', '--build', build_dir, '--config', config]) shutil.copy(os.path.join(build_dir, config, 'llvmlite.dll'), target_dir) @@ -109,7 +109,6 @@ def main_posix_cmake(kind, library_ext): config = 'Release' if not os.path.exists(build_dir): os.mkdir(build_dir) - os.environ['lld_libs'] = lld_libs.strip('-l') try_cmake(here_dir, build_dir, generator) subprocess.check_call(['cmake', '--build', build_dir, '--config', config]) shutil.copy(os.path.join(build_dir, 'libllvmlite' + library_ext), target_dir) From 509585aca914b3929fce5c10fa6bc9b51660d0d5 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Thu, 19 Jan 2023 22:45:19 -0500 Subject: [PATCH 088/158] put semicolons --- ffi/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/build.py b/ffi/build.py index 9d47b53fc..a94b3b755 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -33,7 +33,7 @@ def try_cmake(cmake_dir, build_dir, generator, arch=None, toolkit=None): args.append(cmake_dir) try: os.chdir(build_dir) - os.environ['lld_libs'] = lld_libs.replace("-l", "") + os.environ['lld_libs'] = lld_libs.replace("-l", "").replace(' ',';') print('Running:', ' '.join(args)) subprocess.check_call(args) finally: From a9da5ce64f75acf9d4bdbc7cc9b61749e7993e5c Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 10 Jul 2023 01:41:49 -0400 Subject: [PATCH 089/158] Update LLD binding --- ffi/lld.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/lld.cpp b/ffi/lld.cpp index 476d33dd2..92deaf155 100644 --- a/ffi/lld.cpp +++ b/ffi/lld.cpp @@ -12,7 +12,7 @@ LLVMPY_lld_main(int Argc, const char **Argv, const char **outstr) { std::vector Args(Argv, Argv + Argc); bool linker_output = - !lld::elf::link(Args, false, command_stream, command_stream); + !lld::elf::link(Args, command_stream, command_stream, false, false); *outstr = LLVMPY_CreateString(command_output.c_str()); return linker_output; From d88a118bcd5d3f3e4a1b4d6133f48542cc211108 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 10 Jul 2023 14:39:13 -0400 Subject: [PATCH 090/158] possibly a fix, some libs don't exist anymore? --- ffi/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/build.py b/ffi/build.py index 38e9fce1e..1708c5c16 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -20,7 +20,7 @@ target_dir = os.path.join(os.path.dirname(here_dir), 'llvmlite', 'binding') is_64bit = sys.maxsize >= 2**32 -lld_libs = "-llldCommon -llldCore -llldELF -llldReaderWriter -llldWasm -llldYAML" +lld_libs = "-llldCommon -llldELF -llldWasm" def try_cmake(cmake_dir, build_dir, generator, arch=None, toolkit=None): From 5afb44c1c058d68e00e3daa519017a5d8d3cf849 Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Thu, 21 Sep 2023 08:06:54 -0400 Subject: [PATCH 091/158] Update Makefile.freebsd because I suck at merging --- ffi/Makefile.freebsd | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ffi/Makefile.freebsd b/ffi/Makefile.freebsd index b96d0bac6..85f43f7b9 100644 --- a/ffi/Makefile.freebsd +++ b/ffi/Makefile.freebsd @@ -11,8 +11,7 @@ LIBS = $(LLVM_LIBS) INCLUDE = core.h SRC = assembly.cpp bitcode.cpp core.cpp initfini.cpp module.cpp value.cpp \ executionengine.cpp transforms.cpp passmanagers.cpp targets.cpp dylib.cpp \ - -linker.cpp object_file.cpp orcjit.cpp lld.cpp + linker.cpp object_file.cpp orcjit.cpp lld.cpp OUTPUT = libllvmlite.so all: $(OUTPUT) From 1f4065cbdf24c8d25b47bafb634774203569c5aa Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 25 Sep 2023 16:21:45 -0400 Subject: [PATCH 092/158] trying to update the conda setup to fix the build --- buildscripts/incremental/setup_conda_environment.sh | 2 +- ffi/build.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index 20008b264..f555e7fea 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -32,7 +32,7 @@ $CONDA_INSTALL -c numba/label/dev llvmdev="14.*" # Install the compiler toolchain, for osx, bootstrapping needed # which happens in build.sh if [[ $(uname) == Linux ]]; then -$CONDA_INSTALL gcc_linux-64 gxx_linux-64 +$CONDA_INSTALL gcc_linux-64 gxx_linux-64 libstdcxx-ng=12 fi # Install dependencies for code coverage (codecov.io) diff --git a/ffi/build.py b/ffi/build.py index 1708c5c16..95aca6c7a 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -72,7 +72,6 @@ def find_windows_generator(): generators.append( (env_generator, env_arch, env_toolkit) ) - generators.extend([ # use VS2019 to match how llvmdev is built ('Visual Studio 16 2019', ('x64' if is_64bit else 'Win32'), 'v142'), From dbdb038c9b0654cbc8fa6144720412531b6f292e Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 25 Sep 2023 16:29:26 -0400 Subject: [PATCH 093/158] hopefully correct channel this time? --- buildscripts/incremental/setup_conda_environment.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index f555e7fea..1a120d137 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -32,7 +32,8 @@ $CONDA_INSTALL -c numba/label/dev llvmdev="14.*" # Install the compiler toolchain, for osx, bootstrapping needed # which happens in build.sh if [[ $(uname) == Linux ]]; then -$CONDA_INSTALL gcc_linux-64 gxx_linux-64 libstdcxx-ng=12 +$CONDA_INSTALL gcc_linux-64 gxx_linux-64 +$CONDA_INSTALL -c conda-forge libstdcxx-ng=12 fi # Install dependencies for code coverage (codecov.io) From a0ef848eab4d7ad6180d5c4d2d54e6304ad44da3 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 25 Sep 2023 17:04:52 -0400 Subject: [PATCH 094/158] honestly just trying things to see if they help. --- buildscripts/incremental/setup_conda_environment.sh | 2 +- ffi/build.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index 1a120d137..65908da32 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -32,7 +32,7 @@ $CONDA_INSTALL -c numba/label/dev llvmdev="14.*" # Install the compiler toolchain, for osx, bootstrapping needed # which happens in build.sh if [[ $(uname) == Linux ]]; then -$CONDA_INSTALL gcc_linux-64 gxx_linux-64 +# $CONDA_INSTALL gcc_linux-64 gxx_linux-64 $CONDA_INSTALL -c conda-forge libstdcxx-ng=12 fi diff --git a/ffi/build.py b/ffi/build.py index 95aca6c7a..86989fed2 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -112,6 +112,7 @@ def main_posix_cmake(kind, library_ext): subprocess.check_call(['cmake', '--build', build_dir, '--config', config]) shutil.copy(os.path.join(build_dir, 'libllvmlite' + library_ext), target_dir) + def main_posix(kind, library_ext): if os.environ.get("LLVMLITE_USE_CMAKE", "0") == "1": return main_posix_cmake(kind, library_ext) From 927a044f628115371b03096f9f34c4df17d5c109 Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Fri, 1 Dec 2023 08:09:30 -0500 Subject: [PATCH 095/158] Change build to hopefully show stdout on error (temporarily) --- ffi/build.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ffi/build.py b/ffi/build.py index 86989fed2..6cc5f6ef4 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -208,8 +208,11 @@ def main_posix(kind, library_ext): except NotImplementedError: default_makeopts = "" makeopts = os.environ.get('LLVMLITE_MAKEOPTS', default_makeopts).split() - subprocess.check_call(['make', '-f', makefile] + makeopts) - shutil.copy('libllvmlite' + library_ext, target_dir) + try: + subprocess.check_call(['make', '-f', makefile] + makeopts) + except subprocess.CalledProcessError as e: + print(e.stdout) + shutil.copy('sulibllvmlite' + library_ext, target_dir) def main(): From b4ef76036e13884cf06dccccd7b64af42f635401 Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Fri, 1 Dec 2023 08:14:35 -0500 Subject: [PATCH 096/158] more info for more build testing --- ffi/build.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ffi/build.py b/ffi/build.py index 6cc5f6ef4..98b44c747 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -179,6 +179,7 @@ def main_posix(kind, library_ext): # Normalize whitespace (trim newlines) os.environ['LLVM_LIBS'] = \ f'{lld_libs} ' + ' '.join(libs.split()) + print("LLVM LIBS USED: ", os.environ['LLVM_LIBS']) cxxflags = run_llvm_config(llvm_config, ["--cxxflags"]) # on OSX cxxflags has null bytes at the end of the string, remove them @@ -210,9 +211,13 @@ def main_posix(kind, library_ext): makeopts = os.environ.get('LLVMLITE_MAKEOPTS', default_makeopts).split() try: subprocess.check_call(['make', '-f', makefile] + makeopts) + shutil.copy('sulibllvmlite' + library_ext, target_dir) except subprocess.CalledProcessError as e: + print('\n\n\n\n PROBLEM HERE:') + print(e.stdout) - shutil.copy('sulibllvmlite' + library_ext, target_dir) + + print('\n\n\nEND OF PROBLEM\n\n\n') def main(): From 62076bdff0056aa4eb1e5b85627b6743904b4130 Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Fri, 1 Dec 2023 08:22:25 -0500 Subject: [PATCH 097/158] more test info --- ffi/build.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ffi/build.py b/ffi/build.py index 98b44c747..ca2d0e40a 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -211,13 +211,16 @@ def main_posix(kind, library_ext): makeopts = os.environ.get('LLVMLITE_MAKEOPTS', default_makeopts).split() try: subprocess.check_call(['make', '-f', makefile] + makeopts) - shutil.copy('sulibllvmlite' + library_ext, target_dir) + shutil.copy('libllvmlite' + library_ext, target_dir) except subprocess.CalledProcessError as e: print('\n\n\n\n PROBLEM HERE:') print(e.stdout) + print('\n\n') + print(e.stderr) print('\n\n\nEND OF PROBLEM\n\n\n') + raise e def main(): From 99773063a7d8735f81bd71e58541f3c5ffb2f21b Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Fri, 1 Dec 2023 08:27:08 -0500 Subject: [PATCH 098/158] use subprocess.run instead so I can get stdout --- ffi/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/build.py b/ffi/build.py index ca2d0e40a..d4f62ce5e 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -210,7 +210,7 @@ def main_posix(kind, library_ext): default_makeopts = "" makeopts = os.environ.get('LLVMLITE_MAKEOPTS', default_makeopts).split() try: - subprocess.check_call(['make', '-f', makefile] + makeopts) + print(subprocess.run(['make', '-f', makefile] + makeopts, check=True)) shutil.copy('libllvmlite' + library_ext, target_dir) except subprocess.CalledProcessError as e: print('\n\n\n\n PROBLEM HERE:') From df963e5ce3c5577b4b80bc73b16c4c52af3a19a6 Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Fri, 1 Dec 2023 08:30:14 -0500 Subject: [PATCH 099/158] again, trying to get stdout... I thought this would be easier lol. --- ffi/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/build.py b/ffi/build.py index d4f62ce5e..60c61e7ad 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -210,7 +210,7 @@ def main_posix(kind, library_ext): default_makeopts = "" makeopts = os.environ.get('LLVMLITE_MAKEOPTS', default_makeopts).split() try: - print(subprocess.run(['make', '-f', makefile] + makeopts, check=True)) + print(subprocess.run(['make', '-f', makefile] + makeopts, check=True, capture_output=True)) shutil.copy('libllvmlite' + library_ext, target_dir) except subprocess.CalledProcessError as e: print('\n\n\n\n PROBLEM HERE:') From 5ece647543322a195e8aee685aefdcee88f535bb Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Fri, 1 Dec 2023 08:33:53 -0500 Subject: [PATCH 100/158] decode output so I can actually read it. --- ffi/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ffi/build.py b/ffi/build.py index 60c61e7ad..b78c6fbc5 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -215,9 +215,9 @@ def main_posix(kind, library_ext): except subprocess.CalledProcessError as e: print('\n\n\n\n PROBLEM HERE:') - print(e.stdout) + print(e.stdout.decode('latin1')) print('\n\n') - print(e.stderr) + print(e.stderr.decode('latin1')) print('\n\n\nEND OF PROBLEM\n\n\n') raise e From 1125c9ebf8f7de458c41273a565533593be57455 Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Fri, 1 Dec 2023 08:47:54 -0500 Subject: [PATCH 101/158] print cxx flags to see those --- ffi/build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ffi/build.py b/ffi/build.py index b78c6fbc5..78e8bc111 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -185,6 +185,7 @@ def main_posix(kind, library_ext): # on OSX cxxflags has null bytes at the end of the string, remove them cxxflags = cxxflags.replace('\0', '') cxxflags = cxxflags.split() + ['-fno-rtti', '-g'] + print('CXX FLAGS USED:', cxxflags) # look for SVML include_dir = run_llvm_config(llvm_config, ['--includedir']).strip() From dadf3be5bd8d96ed4df89f2cbe12b3333c53c94f Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Fri, 1 Dec 2023 09:14:01 -0500 Subject: [PATCH 102/158] try to find lld? --- ffi/build.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ffi/build.py b/ffi/build.py index 78e8bc111..166702666 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -200,6 +200,11 @@ def main_posix(kind, library_ext): ldflags = run_llvm_config(llvm_config, ["--ldflags"]) os.environ['LLVM_LDFLAGS'] = ldflags.strip() + + print("SEEING IF I CAN FIND LLD:") + print(subprocess.check_output(["dpkg", "-L", "lld"], shell=True)) + + # static link libstdc++ for portability if int(os.environ.get('LLVMLITE_CXX_STATIC_LINK', 0)): os.environ['CXX_STATIC_LINK'] = "-static-libstdc++" From fe85acb5354933619b1fb9657a0e4174e4ace6d2 Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Fri, 1 Dec 2023 09:31:10 -0500 Subject: [PATCH 103/158] Update build.py --- ffi/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/build.py b/ffi/build.py index 166702666..e82455fd2 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -202,7 +202,7 @@ def main_posix(kind, library_ext): os.environ['LLVM_LDFLAGS'] = ldflags.strip() print("SEEING IF I CAN FIND LLD:") - print(subprocess.check_output(["dpkg", "-L", "lld"], shell=True)) + print(subprocess.check_output(["dpkg", "-L", "lld"])) # static link libstdc++ for portability From e017e8af42e59ce89c457fc5af97c02b64577b4a Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Fri, 1 Dec 2023 09:35:25 -0500 Subject: [PATCH 104/158] another attempt to find lld --- ffi/build.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ffi/build.py b/ffi/build.py index e82455fd2..ebbd8c4cc 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -202,7 +202,14 @@ def main_posix(kind, library_ext): os.environ['LLVM_LDFLAGS'] = ldflags.strip() print("SEEING IF I CAN FIND LLD:") - print(subprocess.check_output(["dpkg", "-L", "lld"])) + try: + print(subprocess.run(["dpkg", "-L", "lld"], capture_stdout=True)) + except subprocess.CalledProcessError as e: + print("stdout:") + print(e.stdout) + print("\nstderr:") + print(e.stderr) + print("END OF ERROR") # static link libstdc++ for portability From cebf1955d9a6783c4a75d96804c20eed64519adc Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Fri, 1 Dec 2023 09:37:13 -0500 Subject: [PATCH 105/158] Update build.py --- ffi/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/build.py b/ffi/build.py index ebbd8c4cc..cfcb9901c 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -203,7 +203,7 @@ def main_posix(kind, library_ext): print("SEEING IF I CAN FIND LLD:") try: - print(subprocess.run(["dpkg", "-L", "lld"], capture_stdout=True)) + print(subprocess.run(["dpkg", "-L", "lld"], capture_output=True)) except subprocess.CalledProcessError as e: print("stdout:") print(e.stdout) From 4471e74ce6e735ba48b5d2e3fae35c92e0082296 Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Fri, 1 Dec 2023 09:39:55 -0500 Subject: [PATCH 106/158] try something slightly different --- ffi/build.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ffi/build.py b/ffi/build.py index cfcb9901c..1296dc4c3 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -203,12 +203,12 @@ def main_posix(kind, library_ext): print("SEEING IF I CAN FIND LLD:") try: - print(subprocess.run(["dpkg", "-L", "lld"], capture_output=True)) + print(subprocess.run(["dpkg", "-L", "llvm"], capture_output=True, check=True).stdout.decode('latin1')) except subprocess.CalledProcessError as e: print("stdout:") - print(e.stdout) + print(e.stdout.decode('latin1')) print("\nstderr:") - print(e.stderr) + print(e.stderr.decode('latin1')) print("END OF ERROR") From 98f9543442d94481dca0052bd8bd4667745b5135 Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Fri, 1 Dec 2023 09:49:22 -0500 Subject: [PATCH 107/158] Update build.py --- ffi/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/build.py b/ffi/build.py index 1296dc4c3..4ddbd0c8b 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -203,7 +203,7 @@ def main_posix(kind, library_ext): print("SEEING IF I CAN FIND LLD:") try: - print(subprocess.run(["dpkg", "-L", "llvm"], capture_output=True, check=True).stdout.decode('latin1')) + print(subprocess.run(["dpkg", "-L", "llvm-14"], capture_output=True, check=True).stdout.decode('latin1')) except subprocess.CalledProcessError as e: print("stdout:") print(e.stdout.decode('latin1')) From ff5a6a0968d93608d52d735f16569845b14659dd Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Fri, 1 Dec 2023 09:54:41 -0500 Subject: [PATCH 108/158] Update build.py --- ffi/build.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ffi/build.py b/ffi/build.py index 4ddbd0c8b..98f357591 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -203,6 +203,8 @@ def main_posix(kind, library_ext): print("SEEING IF I CAN FIND LLD:") try: + print(os.walk("-I/usr/lib/llvm-14/")) + print("\n\n") print(subprocess.run(["dpkg", "-L", "llvm-14"], capture_output=True, check=True).stdout.decode('latin1')) except subprocess.CalledProcessError as e: print("stdout:") From ea1c1871f081d918a43078cfde6939078fbd8482 Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Fri, 1 Dec 2023 10:06:43 -0500 Subject: [PATCH 109/158] Update build.py --- ffi/build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ffi/build.py b/ffi/build.py index 98f357591..36cdd46b8 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -203,7 +203,8 @@ def main_posix(kind, library_ext): print("SEEING IF I CAN FIND LLD:") try: - print(os.walk("-I/usr/lib/llvm-14/")) + for file in os.walk("-I/usr/lib/llvm-14/"): + print(file) print("\n\n") print(subprocess.run(["dpkg", "-L", "llvm-14"], capture_output=True, check=True).stdout.decode('latin1')) except subprocess.CalledProcessError as e: From 36cf85b4876a7cde472bba66ad809161b4c96349 Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Fri, 1 Dec 2023 10:25:18 -0500 Subject: [PATCH 110/158] Update .readthedocs.yaml --- .readthedocs.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 43152be07..cc785476e 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -8,6 +8,8 @@ build: os: ubuntu-22.04 tools: python: "3.11" + apt_packages: + - lld-14 sphinx: configuration: docs/source/conf.py From 9b98bbc44cd6c7aca2f71a85b33b95506f2aa8c0 Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Fri, 1 Dec 2023 10:34:38 -0500 Subject: [PATCH 111/158] Update build.py --- ffi/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ffi/build.py b/ffi/build.py index 36cdd46b8..c996a5d4e 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -203,10 +203,10 @@ def main_posix(kind, library_ext): print("SEEING IF I CAN FIND LLD:") try: - for file in os.walk("-I/usr/lib/llvm-14/"): + for file in os.walk("-I/usr/lib/llvm-14"): print(file) print("\n\n") - print(subprocess.run(["dpkg", "-L", "llvm-14"], capture_output=True, check=True).stdout.decode('latin1')) + print(subprocess.run(["dpkg", "-L", "lld-14"], capture_output=True, check=True).stdout.decode('latin1')) except subprocess.CalledProcessError as e: print("stdout:") print(e.stdout.decode('latin1')) From 4d7d57487d9b2d27290e17f7c9ac7383e54c802e Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Fri, 1 Dec 2023 10:46:34 -0500 Subject: [PATCH 112/158] include liblld-14-dev on docs build --- .readthedocs.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index cc785476e..b4480b193 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -10,6 +10,7 @@ build: python: "3.11" apt_packages: - lld-14 + - liblld-14-dev sphinx: configuration: docs/source/conf.py From 0165131bca0c1fd8e9fa23f27d945f91551590ae Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Fri, 1 Dec 2023 10:55:58 -0500 Subject: [PATCH 113/158] revert some of those build.py changes --- ffi/build.py | 30 ++---------------------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/ffi/build.py b/ffi/build.py index c996a5d4e..5ee6a2c4b 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -185,7 +185,6 @@ def main_posix(kind, library_ext): # on OSX cxxflags has null bytes at the end of the string, remove them cxxflags = cxxflags.replace('\0', '') cxxflags = cxxflags.split() + ['-fno-rtti', '-g'] - print('CXX FLAGS USED:', cxxflags) # look for SVML include_dir = run_llvm_config(llvm_config, ['--includedir']).strip() @@ -200,20 +199,6 @@ def main_posix(kind, library_ext): ldflags = run_llvm_config(llvm_config, ["--ldflags"]) os.environ['LLVM_LDFLAGS'] = ldflags.strip() - - print("SEEING IF I CAN FIND LLD:") - try: - for file in os.walk("-I/usr/lib/llvm-14"): - print(file) - print("\n\n") - print(subprocess.run(["dpkg", "-L", "lld-14"], capture_output=True, check=True).stdout.decode('latin1')) - except subprocess.CalledProcessError as e: - print("stdout:") - print(e.stdout.decode('latin1')) - print("\nstderr:") - print(e.stderr.decode('latin1')) - print("END OF ERROR") - # static link libstdc++ for portability if int(os.environ.get('LLVMLITE_CXX_STATIC_LINK', 0)): @@ -225,19 +210,8 @@ def main_posix(kind, library_ext): except NotImplementedError: default_makeopts = "" makeopts = os.environ.get('LLVMLITE_MAKEOPTS', default_makeopts).split() - try: - print(subprocess.run(['make', '-f', makefile] + makeopts, check=True, capture_output=True)) - shutil.copy('libllvmlite' + library_ext, target_dir) - except subprocess.CalledProcessError as e: - print('\n\n\n\n PROBLEM HERE:') - - print(e.stdout.decode('latin1')) - print('\n\n') - print(e.stderr.decode('latin1')) - - print('\n\n\nEND OF PROBLEM\n\n\n') - raise e - + print(subprocess.check_call(['make', '-f', makefile] + makeopts)) + shutil.copy('libllvmlite' + library_ext, target_dir) def main(): if sys.platform == 'win32': From 2453b4f15ec414fbbddaafb5f85e9dbae5a57f2a Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Fri, 1 Dec 2023 14:19:10 -0500 Subject: [PATCH 114/158] revert last 2 things I messed up here. --- ffi/build.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ffi/build.py b/ffi/build.py index 5ee6a2c4b..2ee3562e6 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -179,7 +179,6 @@ def main_posix(kind, library_ext): # Normalize whitespace (trim newlines) os.environ['LLVM_LIBS'] = \ f'{lld_libs} ' + ' '.join(libs.split()) - print("LLVM LIBS USED: ", os.environ['LLVM_LIBS']) cxxflags = run_llvm_config(llvm_config, ["--cxxflags"]) # on OSX cxxflags has null bytes at the end of the string, remove them @@ -210,7 +209,7 @@ def main_posix(kind, library_ext): except NotImplementedError: default_makeopts = "" makeopts = os.environ.get('LLVMLITE_MAKEOPTS', default_makeopts).split() - print(subprocess.check_call(['make', '-f', makefile] + makeopts)) + subprocess.check_call(['make', '-f', makefile] + makeopts) shutil.copy('libllvmlite' + library_ext, target_dir) def main(): From a2f05545420b0c3ca4c739c57aa9ff4eb397b1cc Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Mon, 4 Dec 2023 11:14:01 -0500 Subject: [PATCH 115/158] store lld_libs as a list --- ffi/build.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ffi/build.py b/ffi/build.py index 2ee3562e6..e9bbf6d04 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -20,7 +20,10 @@ target_dir = os.path.join(os.path.dirname(here_dir), 'llvmlite', 'binding') is_64bit = sys.maxsize >= 2**32 -lld_libs = "-llldCommon -llldELF -llldWasm" + +lld_libs = ["lldCommon", "lldELF", "lldWasm"] +cxx_lld_libs = ' '.join([f"-l{lib}" for lib in lld_libs]) +cmake_lld_libs = ';'.join(lld_libs) def try_cmake(cmake_dir, build_dir, generator, arch=None, toolkit=None): @@ -33,7 +36,7 @@ def try_cmake(cmake_dir, build_dir, generator, arch=None, toolkit=None): args.append(cmake_dir) try: os.chdir(build_dir) - os.environ['lld_libs'] = lld_libs.replace("-l", "").replace(' ',';') + os.environ['lld_libs'] = cmake_lld_libs print('Running:', ' '.join(args)) subprocess.check_call(args) finally: @@ -178,7 +181,7 @@ def main_posix(kind, library_ext): libs = run_llvm_config(llvm_config, "--system-libs --libs all".split()) # Normalize whitespace (trim newlines) os.environ['LLVM_LIBS'] = \ - f'{lld_libs} ' + ' '.join(libs.split()) + f'{cxx_lld_libs} ' + ' '.join(libs.split()) cxxflags = run_llvm_config(llvm_config, ["--cxxflags"]) # on OSX cxxflags has null bytes at the end of the string, remove them From 59c86b5f46edf9969de38cb4146510ff69e69889 Mon Sep 17 00:00:00 2001 From: spidertyler2005 <38941820+spidertyler2005@users.noreply.github.com> Date: Mon, 18 Dec 2023 10:10:51 -0500 Subject: [PATCH 116/158] make underlines smaller on lld docs --- docs/source/user-guide/binding/lld.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/user-guide/binding/lld.rst b/docs/source/user-guide/binding/lld.rst index 6fba2959b..4c7ee17ad 100644 --- a/docs/source/user-guide/binding/lld.rst +++ b/docs/source/user-guide/binding/lld.rst @@ -1,13 +1,13 @@ -================= +================ Linking with LLD -================= +================ .. currentmodule:: llvmlite.binding The LLD linker is automatically built into llvmlite to provide easy-to-use, cross-platform linking. Known limitations -================== +================= Currently, only ``lld::elf::link`` is used and thus COFF and MACHO object files will **not** link. @@ -19,7 +19,7 @@ The following llvm drivers are usable from :func:`lld_main` * wasm-ld (Web Assembly) Functions -========== +========= * .. function:: lld_main(args) @@ -60,4 +60,4 @@ Functions Automatically determines which function to use given the host operating system. This will only use the ``lld_linux``, ``lld_windows``, and ``lld_macos`` functions. - Throws :exc:`Exception` if the host operating system isn't ``Darwin``, ``Windows``, or ``Linux`` \ No newline at end of file + Throws :exc:`Exception` if the host operating system isn't ``Darwin``, ``Windows``, or ``Linux`` From b4aadb4ff81c535eb759c2d792929331a397d0e8 Mon Sep 17 00:00:00 2001 From: ArachnidAbby <38941820+ArachnidAbby@users.noreply.github.com> Date: Wed, 21 Aug 2024 22:16:54 -0400 Subject: [PATCH 117/158] Update setup_conda_environment.sh --- buildscripts/incremental/setup_conda_environment.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index 511533f88..ee901053e 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -28,7 +28,9 @@ set -v # Install llvmdev (separate channel, for now) if [ "$LLVM" == "16" ]; then - $CONDA_INSTALL -c conda-forge llvmdev="16" + # also install lld, not required when using the numba channel because the numba + # channel includes lld. + $CONDA_INSTALL -c conda-forge llvmdev="16" lld="16" else $CONDA_INSTALL -c numba llvmdev="15.*" fi From 6a8e8b4083a3d88f53f66f97cbd6c08cc525ebd3 Mon Sep 17 00:00:00 2001 From: ArachnidAbby <38941820+ArachnidAbby@users.noreply.github.com> Date: Tue, 27 Aug 2024 15:24:38 -0400 Subject: [PATCH 118/158] Attempting to update libstdcxx-ng to fix llvm16 problem --- buildscripts/incremental/setup_conda_environment.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index ee901053e..90fd51c8a 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -39,7 +39,7 @@ fi # which happens in build.sh if [[ $(uname) == Linux ]]; then # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 -$CONDA_INSTALL -c conda-forge libstdcxx-ng=12 +$CONDA_INSTALL -c conda-forge libstdcxx-ng=14 fi # Install dependencies for code coverage (codecov.io) From d2d3105d68784b311e055802db7649ef242c6715 Mon Sep 17 00:00:00 2001 From: ArachnidAbby <38941820+ArachnidAbby@users.noreply.github.com> Date: Tue, 27 Aug 2024 15:35:36 -0400 Subject: [PATCH 119/158] Update setup_conda_environment.sh --- buildscripts/incremental/setup_conda_environment.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index 90fd51c8a..ffd9918fe 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -38,8 +38,8 @@ fi # Install the compiler toolchain, for osx, bootstrapping needed # which happens in build.sh if [[ $(uname) == Linux ]]; then -# $CONDA_INSTALL gcc_linux-64 gxx_linux-64 -$CONDA_INSTALL -c conda-forge libstdcxx-ng=14 +$CONDA_INSTALL gcc_linux-64 gxx_linux-64 +# $CONDA_INSTALL -c conda-forge libstdcxx-ng=14 fi # Install dependencies for code coverage (codecov.io) From 96b6878f27a6bc13c14cd63398a7045f26b2f25e Mon Sep 17 00:00:00 2001 From: ArachnidAbby <38941820+ArachnidAbby@users.noreply.github.com> Date: Tue, 27 Aug 2024 15:43:02 -0400 Subject: [PATCH 120/158] Update setup_conda_environment.cmd --- buildscripts/incremental/setup_conda_environment.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/incremental/setup_conda_environment.cmd b/buildscripts/incremental/setup_conda_environment.cmd index ac572f3a7..bc842e73a 100644 --- a/buildscripts/incremental/setup_conda_environment.cmd +++ b/buildscripts/incremental/setup_conda_environment.cmd @@ -15,7 +15,7 @@ if %errorlevel% neq 0 exit /b %errorlevel% @rem Install llvmdev if "%LLVM%"=="16" ( - set LLVMDEV_CHANNEL="conda-forge" + set LLVMDEV_CHANNEL="conda-forge" lld="16" ) else ( set LLVMDEV_CHANNEL="numba" ) From 3cdb40ac32bb267e7b6baf601ce8fd3d230edfe0 Mon Sep 17 00:00:00 2001 From: ArachnidAbby <38941820+ArachnidAbby@users.noreply.github.com> Date: Tue, 27 Aug 2024 15:55:19 -0400 Subject: [PATCH 121/158] Update setup_conda_environment.sh --- buildscripts/incremental/setup_conda_environment.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index ffd9918fe..ee901053e 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -38,8 +38,8 @@ fi # Install the compiler toolchain, for osx, bootstrapping needed # which happens in build.sh if [[ $(uname) == Linux ]]; then -$CONDA_INSTALL gcc_linux-64 gxx_linux-64 -# $CONDA_INSTALL -c conda-forge libstdcxx-ng=14 +# $CONDA_INSTALL gcc_linux-64 gxx_linux-64 +$CONDA_INSTALL -c conda-forge libstdcxx-ng=12 fi # Install dependencies for code coverage (codecov.io) From ee683a3e4354489b5f505b83cd1f8d56eefee63d Mon Sep 17 00:00:00 2001 From: ArachnidAbby <38941820+ArachnidAbby@users.noreply.github.com> Date: Wed, 28 Aug 2024 11:28:57 -0400 Subject: [PATCH 122/158] trying using clangxx instead --- buildscripts/incremental/setup_conda_environment.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index ee901053e..98cff0238 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -39,7 +39,8 @@ fi # which happens in build.sh if [[ $(uname) == Linux ]]; then # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 -$CONDA_INSTALL -c conda-forge libstdcxx-ng=12 +# $CONDA_INSTALL -c conda-forge libstdcxx-ng=12 +$CONDA_INSTALL -c conda-forge clangxx=16 fi # Install dependencies for code coverage (codecov.io) From e4d9e5fb95e3b9970b87207ac14228969abc6dfb Mon Sep 17 00:00:00 2001 From: ArachnidAbby <38941820+ArachnidAbby@users.noreply.github.com> Date: Wed, 28 Aug 2024 13:43:27 -0400 Subject: [PATCH 123/158] Update setup_conda_environment.sh --- buildscripts/incremental/setup_conda_environment.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index 98cff0238..ee901053e 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -39,8 +39,7 @@ fi # which happens in build.sh if [[ $(uname) == Linux ]]; then # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 -# $CONDA_INSTALL -c conda-forge libstdcxx-ng=12 -$CONDA_INSTALL -c conda-forge clangxx=16 +$CONDA_INSTALL -c conda-forge libstdcxx-ng=12 fi # Install dependencies for code coverage (codecov.io) From e6d8cb91b830a5a84dfb21097c10e831b13d2c60 Mon Sep 17 00:00:00 2001 From: ArachnidAbby <38941820+ArachnidAbby@users.noreply.github.com> Date: Sun, 8 Sep 2024 12:28:43 -0400 Subject: [PATCH 124/158] undo this mistake --- buildscripts/incremental/setup_conda_environment.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/incremental/setup_conda_environment.cmd b/buildscripts/incremental/setup_conda_environment.cmd index bc842e73a..ac572f3a7 100644 --- a/buildscripts/incremental/setup_conda_environment.cmd +++ b/buildscripts/incremental/setup_conda_environment.cmd @@ -15,7 +15,7 @@ if %errorlevel% neq 0 exit /b %errorlevel% @rem Install llvmdev if "%LLVM%"=="16" ( - set LLVMDEV_CHANNEL="conda-forge" lld="16" + set LLVMDEV_CHANNEL="conda-forge" ) else ( set LLVMDEV_CHANNEL="numba" ) From c03f7ceeca1cc4e878e25797c4aff4eeff3edff4 Mon Sep 17 00:00:00 2001 From: ArachnidAbby <38941820+ArachnidAbby@users.noreply.github.com> Date: Sun, 8 Sep 2024 12:30:49 -0400 Subject: [PATCH 125/158] try adding libxml2 conda install I believe this may have worked for me in the past, altho I'm not sure if its still necessary. --- buildscripts/incremental/setup_conda_environment.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index ee901053e..5c7b49be8 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -35,6 +35,8 @@ else $CONDA_INSTALL -c numba llvmdev="15.*" fi +$CONDA_INSTALL -c numba libxml2 + # Install the compiler toolchain, for osx, bootstrapping needed # which happens in build.sh if [[ $(uname) == Linux ]]; then From 9a141d89e24563d5ba9acc3de649d68a399b9655 Mon Sep 17 00:00:00 2001 From: ArachnidAbby <38941820+ArachnidAbby@users.noreply.github.com> Date: Sun, 8 Sep 2024 12:54:42 -0400 Subject: [PATCH 126/158] Update setup_conda_environment.cmd --- buildscripts/incremental/setup_conda_environment.cmd | 3 +++ 1 file changed, 3 insertions(+) diff --git a/buildscripts/incremental/setup_conda_environment.cmd b/buildscripts/incremental/setup_conda_environment.cmd index ac572f3a7..48cc25ffb 100644 --- a/buildscripts/incremental/setup_conda_environment.cmd +++ b/buildscripts/incremental/setup_conda_environment.cmd @@ -21,4 +21,7 @@ if "%LLVM%"=="16" ( ) call conda install -y -q -c %LLVMDEV_CHANNEL% llvmdev="%LLVM%" libxml2 +if "%LLVM%"=="16" ( + call conda install -y -q -c %LLVMDEV_CHANNEL% lld="16" +) if %errorlevel% neq 0 exit /b %errorlevel% From fc869f91f46956df7961f89af1eefb017b105828 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 18 Jan 2025 18:14:31 -0500 Subject: [PATCH 127/158] add missing conda dep for lld16 --- buildscripts/incremental/setup_conda_environment.cmd | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/buildscripts/incremental/setup_conda_environment.cmd b/buildscripts/incremental/setup_conda_environment.cmd index 48cc25ffb..0bfb1156a 100644 --- a/buildscripts/incremental/setup_conda_environment.cmd +++ b/buildscripts/incremental/setup_conda_environment.cmd @@ -20,8 +20,10 @@ if "%LLVM%"=="16" ( set LLVMDEV_CHANNEL="numba" ) -call conda install -y -q -c %LLVMDEV_CHANNEL% llvmdev="%LLVM%" libxml2 if "%LLVM%"=="16" ( + call conda install libstdcxx-ng call conda install -y -q -c %LLVMDEV_CHANNEL% lld="16" ) +call conda install -y -q -c %LLVMDEV_CHANNEL% llvmdev="%LLVM%" libxml2 + if %errorlevel% neq 0 exit /b %errorlevel% From 1cbb6b65ef69146b631d4e1da6f1d7fb6b62f434 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 18 Jan 2025 18:20:12 -0500 Subject: [PATCH 128/158] add dep install in the other setup script --- buildscripts/incremental/setup_conda_environment.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index 5c7b49be8..0eaf81b84 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -30,6 +30,7 @@ set -v if [ "$LLVM" == "16" ]; then # also install lld, not required when using the numba channel because the numba # channel includes lld. + $CONDA_INSTALL libstdcxx-ng $CONDA_INSTALL -c conda-forge llvmdev="16" lld="16" else $CONDA_INSTALL -c numba llvmdev="15.*" From 91d95cec57ca92bcc91417fcc04f4834641109bd Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 18 Jan 2025 18:34:13 -0500 Subject: [PATCH 129/158] trying libstdcxx-ng on normal channel --- .../incremental/setup_conda_environment.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index 0eaf81b84..9059cbe31 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -30,20 +30,21 @@ set -v if [ "$LLVM" == "16" ]; then # also install lld, not required when using the numba channel because the numba # channel includes lld. - $CONDA_INSTALL libstdcxx-ng + if [[ $(uname) == Linux ]]; then + $CONDA_INSTALL libstdcxx-ng + fi $CONDA_INSTALL -c conda-forge llvmdev="16" lld="16" else + # Install the compiler toolchain, for osx, bootstrapping needed + # which happens in build.sh + if [[ $(uname) == Linux ]]; then + # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 + $CONDA_INSTALL -c conda-forge libstdcxx-ng=12 + fi $CONDA_INSTALL -c numba llvmdev="15.*" fi $CONDA_INSTALL -c numba libxml2 -# Install the compiler toolchain, for osx, bootstrapping needed -# which happens in build.sh -if [[ $(uname) == Linux ]]; then -# $CONDA_INSTALL gcc_linux-64 gxx_linux-64 -$CONDA_INSTALL -c conda-forge libstdcxx-ng=12 -fi - # Install dependencies for code coverage (codecov.io) if [ "$RUN_COVERAGE" == "yes" ]; then $PIP_INSTALL codecov coveralls; fi From aa2bd95bee44802a1e6156cd513791c0c91e7e8d Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 18 Jan 2025 18:41:22 -0500 Subject: [PATCH 130/158] try updating pkg instead --- buildscripts/incremental/setup_conda_environment.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index 9059cbe31..6276e34d2 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -31,7 +31,7 @@ if [ "$LLVM" == "16" ]; then # also install lld, not required when using the numba channel because the numba # channel includes lld. if [[ $(uname) == Linux ]]; then - $CONDA_INSTALL libstdcxx-ng + $CONDA_INSTALL update libstdcxx-ng fi $CONDA_INSTALL -c conda-forge llvmdev="16" lld="16" else From fd05995ed373b5206d9fc520e6129d10722e3088 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 18 Jan 2025 18:50:19 -0500 Subject: [PATCH 131/158] try installing libcxx on mac --- buildscripts/incremental/setup_conda_environment.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index 6276e34d2..4450512cc 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -31,7 +31,9 @@ if [ "$LLVM" == "16" ]; then # also install lld, not required when using the numba channel because the numba # channel includes lld. if [[ $(uname) == Linux ]]; then - $CONDA_INSTALL update libstdcxx-ng + conda update libstdcxx-ng + else + $CONDA_INSTALL -c conda-forge libcxx fi $CONDA_INSTALL -c conda-forge llvmdev="16" lld="16" else @@ -40,6 +42,8 @@ else if [[ $(uname) == Linux ]]; then # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 $CONDA_INSTALL -c conda-forge libstdcxx-ng=12 + else + $CONDA_INSTALL -c conda-forge libcxx fi $CONDA_INSTALL -c numba llvmdev="15.*" fi From 332e14347369a5ad511b0c02e968dd0d2fb9f6ca Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 18 Jan 2025 19:00:44 -0500 Subject: [PATCH 132/158] try libstdcxx-ng=11 --- .../incremental/setup_conda_environment.sh | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index 4450512cc..ea6a93a04 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -30,25 +30,21 @@ set -v if [ "$LLVM" == "16" ]; then # also install lld, not required when using the numba channel because the numba # channel includes lld. - if [[ $(uname) == Linux ]]; then - conda update libstdcxx-ng - else - $CONDA_INSTALL -c conda-forge libcxx - fi $CONDA_INSTALL -c conda-forge llvmdev="16" lld="16" else # Install the compiler toolchain, for osx, bootstrapping needed # which happens in build.sh - if [[ $(uname) == Linux ]]; then - # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 - $CONDA_INSTALL -c conda-forge libstdcxx-ng=12 - else - $CONDA_INSTALL -c conda-forge libcxx - fi $CONDA_INSTALL -c numba llvmdev="15.*" fi $CONDA_INSTALL -c numba libxml2 +if [[ $(uname) == Linux ]]; then + # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 + $CONDA_INSTALL -c conda-forge libstdcxx-ng=11 +else + $CONDA_INSTALL -c conda-forge libcxx +fi + # Install dependencies for code coverage (codecov.io) if [ "$RUN_COVERAGE" == "yes" ]; then $PIP_INSTALL codecov coveralls; fi From 7ad3015ba6fd06b5b910a8f429beaec4f0144a6d Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 18 Jan 2025 19:19:12 -0500 Subject: [PATCH 133/158] listing packages to see in azure pipeline --- buildscripts/incremental/setup_conda_environment.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index ea6a93a04..ba6288609 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -40,11 +40,13 @@ fi $CONDA_INSTALL -c numba libxml2 if [[ $(uname) == Linux ]]; then - # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 - $CONDA_INSTALL -c conda-forge libstdcxx-ng=11 + $CONDA_INSTALL gcc_linux-64 gxx_linux-64 + $CONDA_INSTALL -c conda-forge libstdcxx-ng=12 else $CONDA_INSTALL -c conda-forge libcxx fi +conda list + # Install dependencies for code coverage (codecov.io) if [ "$RUN_COVERAGE" == "yes" ]; then $PIP_INSTALL codecov coveralls; fi From e2ca6ead8f255585601dff0f841211fc67d5d2b1 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 18 Jan 2025 19:43:13 -0500 Subject: [PATCH 134/158] trying to manually add libxml2 --- buildscripts/incremental/setup_conda_environment.sh | 4 ++-- ffi/build.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index ba6288609..a731a7f6d 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -30,7 +30,7 @@ set -v if [ "$LLVM" == "16" ]; then # also install lld, not required when using the numba channel because the numba # channel includes lld. - $CONDA_INSTALL -c conda-forge llvmdev="16" lld="16" + $CONDA_INSTALL -c conda-forge lld="16" llvmdev="16" libxml2 else # Install the compiler toolchain, for osx, bootstrapping needed # which happens in build.sh @@ -40,7 +40,7 @@ fi $CONDA_INSTALL -c numba libxml2 if [[ $(uname) == Linux ]]; then - $CONDA_INSTALL gcc_linux-64 gxx_linux-64 + # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 $CONDA_INSTALL -c conda-forge libstdcxx-ng=12 else $CONDA_INSTALL -c conda-forge libcxx diff --git a/ffi/build.py b/ffi/build.py index b3987dac2..cfb5dd209 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -208,7 +208,7 @@ def main_posix(kind, library_ext): ldflags = run_llvm_config(llvm_config, ["--ldflags"]) os.environ['LLVM_LDFLAGS'] = ldflags.strip() - + # static link libstdc++ for portability if int(os.environ.get('LLVMLITE_CXX_STATIC_LINK', 0)): os.environ['CXX_STATIC_LINK'] = "-static-libstdc++" @@ -222,6 +222,7 @@ def main_posix(kind, library_ext): subprocess.check_call(['make', '-f', makefile] + makeopts) shutil.copy('libllvmlite' + library_ext, target_dir) + def main(): if sys.platform == 'win32': main_windows() From 04d2492ff25be28bba88a3db609604c4c20fe09a Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 18 Jan 2025 20:09:25 -0500 Subject: [PATCH 135/158] trying to use lld as the linker instead --- ffi/Makefile.linux | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/Makefile.linux b/ffi/Makefile.linux index 9f9fe627c..79512e766 100644 --- a/ffi/Makefile.linux +++ b/ffi/Makefile.linux @@ -1,5 +1,5 @@ # Using g++ is recommended for linking -CXX ?= g++ +CXX ?= ld.lld # -flto and --exclude-libs allow us to remove those parts of LLVM we don't use CXX_FLTO_FLAGS ?= -flto From b43e23d783c2a3d69f7d9a650f40d64b2e488488 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 18 Jan 2025 20:24:42 -0500 Subject: [PATCH 136/158] try using clang++ --- .../incremental/setup_conda_environment.sh | 16 ++++++++-------- ffi/Makefile.linux | 2 +- ffi/Makefile.osx | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index a731a7f6d..f3ce2e933 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -30,22 +30,22 @@ set -v if [ "$LLVM" == "16" ]; then # also install lld, not required when using the numba channel because the numba # channel includes lld. - $CONDA_INSTALL -c conda-forge lld="16" llvmdev="16" libxml2 + $CONDA_INSTALL -c conda-forge lld="16" llvmdev="16" + $CONDA_INSTALL -c conda-forge libclang-cpp else # Install the compiler toolchain, for osx, bootstrapping needed # which happens in build.sh $CONDA_INSTALL -c numba llvmdev="15.*" + if [[ $(uname) == Linux ]]; then + # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 + $CONDA_INSTALL -c conda-forge libstdcxx-ng=12 + else + $CONDA_INSTALL -c conda-forge libcxx + fi fi $CONDA_INSTALL -c numba libxml2 -if [[ $(uname) == Linux ]]; then - # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 - $CONDA_INSTALL -c conda-forge libstdcxx-ng=12 -else - $CONDA_INSTALL -c conda-forge libcxx -fi - conda list # Install dependencies for code coverage (codecov.io) diff --git a/ffi/Makefile.linux b/ffi/Makefile.linux index 79512e766..8c3caff42 100644 --- a/ffi/Makefile.linux +++ b/ffi/Makefile.linux @@ -1,5 +1,5 @@ # Using g++ is recommended for linking -CXX ?= ld.lld +CXX = clang++ -stdlib=libc++ # -flto and --exclude-libs allow us to remove those parts of LLVM we don't use CXX_FLTO_FLAGS ?= -flto diff --git a/ffi/Makefile.osx b/ffi/Makefile.osx index 5d14b97fc..e48025ca1 100644 --- a/ffi/Makefile.osx +++ b/ffi/Makefile.osx @@ -1,5 +1,5 @@ -CXX = clang++ +CXX = clang++ -stdlib=libc++ CXXFLAGS = $(LLVM_CXXFLAGS) # Only export the LLVMPY symbols we require and exclude everything else. EXPORT = "-Wl,-exported_symbol,_LLVMPY_*" From 49e0caec0b5fe68c4c58f2fea2b4e176cd8d58a0 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 18 Jan 2025 20:52:38 -0500 Subject: [PATCH 137/158] using clang++ without weird flags --- ffi/Makefile.linux | 2 +- ffi/Makefile.osx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ffi/Makefile.linux b/ffi/Makefile.linux index 8c3caff42..170172b93 100644 --- a/ffi/Makefile.linux +++ b/ffi/Makefile.linux @@ -1,5 +1,5 @@ # Using g++ is recommended for linking -CXX = clang++ -stdlib=libc++ +CXX = clang++ # -flto and --exclude-libs allow us to remove those parts of LLVM we don't use CXX_FLTO_FLAGS ?= -flto diff --git a/ffi/Makefile.osx b/ffi/Makefile.osx index e48025ca1..5d14b97fc 100644 --- a/ffi/Makefile.osx +++ b/ffi/Makefile.osx @@ -1,5 +1,5 @@ -CXX = clang++ -stdlib=libc++ +CXX = clang++ CXXFLAGS = $(LLVM_CXXFLAGS) # Only export the LLVMPY symbols we require and exclude everything else. EXPORT = "-Wl,-exported_symbol,_LLVMPY_*" From 0ca5d7f980bbca7bd06c324c7ef6cffa2931c5b3 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 18 Jan 2025 21:09:42 -0500 Subject: [PATCH 138/158] clang for all linux + zstd for mac --- .../incremental/setup_conda_environment.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index f3ce2e933..91a09cd0f 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -31,22 +31,23 @@ if [ "$LLVM" == "16" ]; then # also install lld, not required when using the numba channel because the numba # channel includes lld. $CONDA_INSTALL -c conda-forge lld="16" llvmdev="16" - $CONDA_INSTALL -c conda-forge libclang-cpp else # Install the compiler toolchain, for osx, bootstrapping needed # which happens in build.sh $CONDA_INSTALL -c numba llvmdev="15.*" - if [[ $(uname) == Linux ]]; then - # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 - $CONDA_INSTALL -c conda-forge libstdcxx-ng=12 - else - $CONDA_INSTALL -c conda-forge libcxx - fi fi $CONDA_INSTALL -c numba libxml2 -conda list +if [[ $(uname) == Linux ]]; then + # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 + $CONDA_INSTALL -c conda-forge libclang-cpp +fi + +if [[ $(uname) == Darwin ]]; then + # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 + $CONDA_INSTALL -c conda-forge zstd +fi # Install dependencies for code coverage (codecov.io) if [ "$RUN_COVERAGE" == "yes" ]; then $PIP_INSTALL codecov coveralls; fi From e683544fe2a934c4fb5f80dc4c3d2618ba1a3d2f Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 18 Jan 2025 21:31:23 -0500 Subject: [PATCH 139/158] go back to g++ so linux tests work properly --- buildscripts/incremental/setup_conda_environment.sh | 8 ++------ ffi/Makefile.linux | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index 91a09cd0f..997061974 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -31,6 +31,7 @@ if [ "$LLVM" == "16" ]; then # also install lld, not required when using the numba channel because the numba # channel includes lld. $CONDA_INSTALL -c conda-forge lld="16" llvmdev="16" + $CONDA_INSTALL -c conda-forge libclang-cpp else # Install the compiler toolchain, for osx, bootstrapping needed # which happens in build.sh @@ -41,12 +42,7 @@ $CONDA_INSTALL -c numba libxml2 if [[ $(uname) == Linux ]]; then # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 - $CONDA_INSTALL -c conda-forge libclang-cpp -fi - -if [[ $(uname) == Darwin ]]; then - # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 - $CONDA_INSTALL -c conda-forge zstd + $CONDA_INSTALL -c conda-forge libstdcxx-ng fi # Install dependencies for code coverage (codecov.io) diff --git a/ffi/Makefile.linux b/ffi/Makefile.linux index 170172b93..886e12999 100644 --- a/ffi/Makefile.linux +++ b/ffi/Makefile.linux @@ -1,5 +1,5 @@ # Using g++ is recommended for linking -CXX = clang++ +CXX = g++ # -flto and --exclude-libs allow us to remove those parts of LLVM we don't use CXX_FLTO_FLAGS ?= -flto From 1967d228373301880b9192d2139b45e61fea1064 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 18 Jan 2025 21:42:40 -0500 Subject: [PATCH 140/158] use libcxxas dep --- buildscripts/incremental/setup_conda_environment.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index 997061974..e8f519d0e 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -38,7 +38,7 @@ else $CONDA_INSTALL -c numba llvmdev="15.*" fi -$CONDA_INSTALL -c numba libxml2 +$CONDA_INSTALL -c numba libxml2 libcxx if [[ $(uname) == Linux ]]; then # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 From 184f92d9f18ff07005137c407110816adaaf9338 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 18 Jan 2025 22:29:05 -0500 Subject: [PATCH 141/158] trying cmake for llvm16 --- buildscripts/incremental/setup_conda_environment.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index e8f519d0e..c4b6c6dff 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -31,14 +31,14 @@ if [ "$LLVM" == "16" ]; then # also install lld, not required when using the numba channel because the numba # channel includes lld. $CONDA_INSTALL -c conda-forge lld="16" llvmdev="16" - $CONDA_INSTALL -c conda-forge libclang-cpp + export LLVMLITE_USE_CMAKE 1 else # Install the compiler toolchain, for osx, bootstrapping needed # which happens in build.sh $CONDA_INSTALL -c numba llvmdev="15.*" fi -$CONDA_INSTALL -c numba libxml2 libcxx +$CONDA_INSTALL -c numba libxml2 libcxx="12.*" if [[ $(uname) == Linux ]]; then # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 From 5da94bb41c6103f8123ee7e0340f3280e1266328 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 18 Jan 2025 22:29:57 -0500 Subject: [PATCH 142/158] use correct export syntax this time --- buildscripts/incremental/setup_conda_environment.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index c4b6c6dff..98a67a196 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -31,7 +31,7 @@ if [ "$LLVM" == "16" ]; then # also install lld, not required when using the numba channel because the numba # channel includes lld. $CONDA_INSTALL -c conda-forge lld="16" llvmdev="16" - export LLVMLITE_USE_CMAKE 1 + export LLVMLITE_USE_CMAKE=1 else # Install the compiler toolchain, for osx, bootstrapping needed # which happens in build.sh From 06a9bc778aa8907283bdbe85b36479516985314d Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 18 Jan 2025 22:33:11 -0500 Subject: [PATCH 143/158] remove libcxx --- buildscripts/incremental/setup_conda_environment.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index 98a67a196..e846310e5 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -38,7 +38,7 @@ else $CONDA_INSTALL -c numba llvmdev="15.*" fi -$CONDA_INSTALL -c numba libxml2 libcxx="12.*" +$CONDA_INSTALL -c numba libxml2 if [[ $(uname) == Linux ]]; then # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 From ea193c632326fb442c15a56aead62f802bd6b8da Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 18 Jan 2025 22:40:43 -0500 Subject: [PATCH 144/158] build uses cmake with llvm16 --- ffi/build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ffi/build.py b/ffi/build.py index cfb5dd209..2acc1d853 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -175,6 +175,7 @@ def main_posix(kind, library_ext): msg = ("Building with LLVM 16; note that LLVM 16 support is " "presently experimental") show_warning(msg) + return main_posix_cmake(kind, library_ext) elif version != 15: msg = ("Building llvmlite requires LLVM 15, got " From 976b5c07686d3c287102b630c923405b525f6ad7 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sat, 18 Jan 2025 22:48:17 -0500 Subject: [PATCH 145/158] try install libcxxabi --- buildscripts/incremental/setup_conda_environment.sh | 3 +-- ffi/build.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index e846310e5..56868e95b 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -31,14 +31,13 @@ if [ "$LLVM" == "16" ]; then # also install lld, not required when using the numba channel because the numba # channel includes lld. $CONDA_INSTALL -c conda-forge lld="16" llvmdev="16" - export LLVMLITE_USE_CMAKE=1 else # Install the compiler toolchain, for osx, bootstrapping needed # which happens in build.sh $CONDA_INSTALL -c numba llvmdev="15.*" fi -$CONDA_INSTALL -c numba libxml2 +$CONDA_INSTALL -c numba libxml2 libcxxabi if [[ $(uname) == Linux ]]; then # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 diff --git a/ffi/build.py b/ffi/build.py index 2acc1d853..cfb5dd209 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -175,7 +175,6 @@ def main_posix(kind, library_ext): msg = ("Building with LLVM 16; note that LLVM 16 support is " "presently experimental") show_warning(msg) - return main_posix_cmake(kind, library_ext) elif version != 15: msg = ("Building llvmlite requires LLVM 15, got " From c015e32136ce84b8588ac10ff3f5ccb3aafc5cfe Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sun, 19 Jan 2025 14:25:08 -0500 Subject: [PATCH 146/158] Try rebuilding conda-forge lld recipe --- .../incremental/setup_conda_environment.sh | 6 +- conda-recipes/lld/bld.bat | 17 ++++++ conda-recipes/lld/build.sh | 21 +++++++ conda-recipes/lld/conda_build_config.yaml | 39 +++++++++++++ conda-recipes/lld/meta.yaml | 55 +++++++++++++++++++ ...lusion-in-lld-MachO-CMakeLists.txt-t.patch | 25 +++++++++ conda-recipes/lld/recipe-scripts-license.txt | 27 +++++++++ ffi/Makefile.linux | 2 +- 8 files changed, 189 insertions(+), 3 deletions(-) create mode 100644 conda-recipes/lld/bld.bat create mode 100644 conda-recipes/lld/build.sh create mode 100644 conda-recipes/lld/conda_build_config.yaml create mode 100644 conda-recipes/lld/meta.yaml create mode 100644 conda-recipes/lld/patches/0001-point-header-inclusion-in-lld-MachO-CMakeLists.txt-t.patch create mode 100644 conda-recipes/lld/recipe-scripts-license.txt diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index 56868e95b..d27a814db 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -30,14 +30,16 @@ set -v if [ "$LLVM" == "16" ]; then # also install lld, not required when using the numba channel because the numba # channel includes lld. - $CONDA_INSTALL -c conda-forge lld="16" llvmdev="16" + $CONDA_INSTALL -c conda-forge conda-build llvmdev="16" + conda-build conda-recipes/lld + $CONDA_INSTALL --use-local lld else # Install the compiler toolchain, for osx, bootstrapping needed # which happens in build.sh $CONDA_INSTALL -c numba llvmdev="15.*" fi -$CONDA_INSTALL -c numba libxml2 libcxxabi +$CONDA_INSTALL -c numba libxml2 if [[ $(uname) == Linux ]]; then # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 diff --git a/conda-recipes/lld/bld.bat b/conda-recipes/lld/bld.bat new file mode 100644 index 000000000..4822dbbbb --- /dev/null +++ b/conda-recipes/lld/bld.bat @@ -0,0 +1,17 @@ +mkdir build +cd build + +set CC=cl.exe +set CXX=cl.exe + +cmake -G "Ninja" ^ + -DCMAKE_BUILD_TYPE="Release" ^ + -DCMAKE_PREFIX_PATH=%LIBRARY_PREFIX% ^ + -DCMAKE_INSTALL_PREFIX=%LIBRARY_PREFIX% ^ + -DCMAKE_INSTALL_LIBDIR=%LIBRARY_PREFIX%\lib ^ + -DLLVM_INCLUDE_TESTS=OFF ^ + %SRC_DIR%/lld +if %ERRORLEVEL% neq 0 exit 1 + +cmake --build . --target install +if %ERRORLEVEL% neq 0 exit 1 diff --git a/conda-recipes/lld/build.sh b/conda-recipes/lld/build.sh new file mode 100644 index 000000000..47944f842 --- /dev/null +++ b/conda-recipes/lld/build.sh @@ -0,0 +1,21 @@ +mkdir build +cd build + +if [[ "$CONDA_BUILD_CROSS_COMPILATION" == "1" ]]; then + CMAKE_ARGS="$CMAKE_ARGS -DLLVM_CONFIG_PATH=$BUILD_PREFIX/bin/llvm-config -DLLVM_TABLEGEN_EXE=$BUILD_PREFIX/bin/llvm-tblgen" +fi + +cmake -G Ninja \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR=$PREFIX/lib \ + -DCMAKE_PREFIX_PATH=$PREFIX \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_RTTI=ON \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DLLVM_OBJ_ROOT=${PREFIX} \ + -DLLVM_MAIN_INCLUDE_DIR=${PREFIX}/include \ + ${CMAKE_ARGS} \ + ../lld + +cmake --build . -- -j${CPU_COUNT} +cmake --install . diff --git a/conda-recipes/lld/conda_build_config.yaml b/conda-recipes/lld/conda_build_config.yaml new file mode 100644 index 000000000..75dfed3db --- /dev/null +++ b/conda-recipes/lld/conda_build_config.yaml @@ -0,0 +1,39 @@ +CI: azure +c_compiler: gcc +c_compiler_version: '11' +cdt_name: cos7 +channel_sources: conda-forge +channel_targets: conda-forge main +cpu_optimization_target: nocona +cran_mirror: https://cran.r-project.org +cxx_compiler: gxx +cxx_compiler_version: '11' +docker_image: quay.io/condaforge/linux-anvil-ppc64le +extend_keys: +- pin_run_as_build +- ignore_version +- extend_keys +- ignore_build_only_deps +fortran_compiler: gfortran +ignore_build_only_deps: +- numpy +- python +libxml2: '2.10' +lua: '5' +numpy: '1.21' +perl: 5.26.2 +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x + r-base: + min_pin: x.x + max_pin: x.x +python: '3.10' +r_base: '3.5' +target_platform: linux-ppc64le +zip_keys: +- - c_compiler_version + - cxx_compiler_version +zlib: '1.2' +zstd: '1.5' diff --git a/conda-recipes/lld/meta.yaml b/conda-recipes/lld/meta.yaml new file mode 100644 index 000000000..9f6729902 --- /dev/null +++ b/conda-recipes/lld/meta.yaml @@ -0,0 +1,55 @@ +{% set version = "16.0.0" %} + +package: + name: lld + version: {{ version }} + +source: + url: https://github.com/llvm/llvm-project/releases/download/llvmorg-{{ version.replace(".rc", "-rc") }}/llvm-project-{{ version.replace(".rc", "rc") }}.src.tar.xz + sha256: 9a56d906a2c81f16f06efc493a646d497c53c2f4f28f0cb1f3c8da7f74350254 + patches: + - patches/0001-point-header-inclusion-in-lld-MachO-CMakeLists.txt-t.patch + +build: + number: 0 + ignore_run_exports_from: + # not actually needed on unix + - libxml2 # [unix] + +requirements: + build: + - {{ compiler('c') }} + - {{ compiler('cxx') }} + - cmake + - ninja + - llvmdev =={{ version }} # [build_platform != target_platform] + host: + - libcxx {{ cxx_compiler_version }} # [osx] + - llvmdev =={{ version }} + - llvm + # these they are present in the host env anyway as run-deps of llvm + # and will be picked up for the build by the default settings + - libxml2 + - zlib + - zstd + run: + - {{ pin_compatible("libcxx", max_pin=None) }} # [osx] + run_constrained: + - llvm =={{ version }} + +test: + commands: + - ld.lld --version # [linux] + - lld-link /? # [win] + +about: + home: https://lld.llvm.org/ + license: Apache-2.0 WITH LLVM-exception + license_file: lld/LICENSE.TXT + summary: The LLVM Linker + dev_url: https://github.com/llvm/llvm-project/ + +extra: + recipe-maintainers: + - isuruf + - h-vetinari diff --git a/conda-recipes/lld/patches/0001-point-header-inclusion-in-lld-MachO-CMakeLists.txt-t.patch b/conda-recipes/lld/patches/0001-point-header-inclusion-in-lld-MachO-CMakeLists.txt-t.patch new file mode 100644 index 000000000..46d729f03 --- /dev/null +++ b/conda-recipes/lld/patches/0001-point-header-inclusion-in-lld-MachO-CMakeLists.txt-t.patch @@ -0,0 +1,25 @@ +From 14d6c677c1758066f1ef10635439754cc3161057 Mon Sep 17 00:00:00 2001 +From: "H. Vetinari" +Date: Wed, 14 Jul 2021 21:53:53 +0200 +Subject: [PATCH] point header inclusion in lld/MachO/CMakeLists.txt to SRC_DIR + +--- + lld/MachO/CMakeLists.txt | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/lld/MachO/CMakeLists.txt b/lld/MachO/CMakeLists.txt +index ea26889267..3e06eb78fc 100644 +--- a/lld/MachO/CMakeLists.txt ++++ b/lld/MachO/CMakeLists.txt +@@ -2,7 +2,7 @@ set(LLVM_TARGET_DEFINITIONS Options.td) + tablegen(LLVM Options.inc -gen-opt-parser-defs) + add_public_tablegen_target(MachOOptionsTableGen) + +-include_directories(${LLVM_MAIN_SRC_DIR}/../libunwind/include) ++include_directories($ENV{SRC_DIR}/libunwind/include) + + add_lld_library(lldMachO + Arch/ARM.cpp +-- +2.38.1.windows.1 + diff --git a/conda-recipes/lld/recipe-scripts-license.txt b/conda-recipes/lld/recipe-scripts-license.txt new file mode 100644 index 000000000..2ec51d75f --- /dev/null +++ b/conda-recipes/lld/recipe-scripts-license.txt @@ -0,0 +1,27 @@ +BSD-3-Clause license +Copyright (c) 2015-2022, conda-forge contributors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. diff --git a/ffi/Makefile.linux b/ffi/Makefile.linux index 886e12999..9f9fe627c 100644 --- a/ffi/Makefile.linux +++ b/ffi/Makefile.linux @@ -1,5 +1,5 @@ # Using g++ is recommended for linking -CXX = g++ +CXX ?= g++ # -flto and --exclude-libs allow us to remove those parts of LLVM we don't use CXX_FLTO_FLAGS ?= -flto From 286ef00f8bf084760214089bdc14d98e5afb0dce Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sun, 19 Jan 2025 18:28:26 -0500 Subject: [PATCH 147/158] build lld from scratch --- .gitignore | 3 + .../incremental/setup_conda_environment.sh | 17 +++--- conda-recipes/lld/bld.bat | 17 ------ conda-recipes/lld/build.sh | 21 ------- conda-recipes/lld/conda_build_config.yaml | 39 ------------- conda-recipes/lld/meta.yaml | 55 ------------------- conda-recipes/lld/recipe-scripts-license.txt | 27 --------- .../MachoLLDpatch.patch | 7 +-- ffi/build.py | 3 + ffi/build_lld.sh | 53 ++++++++++++++++++ 10 files changed, 70 insertions(+), 172 deletions(-) delete mode 100644 conda-recipes/lld/bld.bat delete mode 100644 conda-recipes/lld/build.sh delete mode 100644 conda-recipes/lld/conda_build_config.yaml delete mode 100644 conda-recipes/lld/meta.yaml delete mode 100644 conda-recipes/lld/recipe-scripts-license.txt rename conda-recipes/lld/patches/0001-point-header-inclusion-in-lld-MachO-CMakeLists.txt-t.patch => ffi/MachoLLDpatch.patch (97%) create mode 100644 ffi/build_lld.sh diff --git a/.gitignore b/.gitignore index dd191b234..02d9a3b6e 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,9 @@ *.dylib *.dll build/ +ffi/*.tar.xz +ffi/lldbuild/ +ffi/llvm-project-16.0.0.src/ dist/ *.egg-info ffi/build/ diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index d27a814db..c08f06a4a 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -26,25 +26,24 @@ set +v source activate $CONDA_ENV set -v +$CONDA_INSTALL -c numba libxml2 +$CONDA_INSTALL -c conda-forge libcrc32c + +if [[ $(uname) == Linux ]]; then + # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 + $CONDA_INSTALL -c conda-forge libstdcxx-ng +fi + # Install llvmdev (separate channel, for now) if [ "$LLVM" == "16" ]; then # also install lld, not required when using the numba channel because the numba # channel includes lld. $CONDA_INSTALL -c conda-forge conda-build llvmdev="16" - conda-build conda-recipes/lld - $CONDA_INSTALL --use-local lld else # Install the compiler toolchain, for osx, bootstrapping needed # which happens in build.sh $CONDA_INSTALL -c numba llvmdev="15.*" fi -$CONDA_INSTALL -c numba libxml2 - -if [[ $(uname) == Linux ]]; then - # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 - $CONDA_INSTALL -c conda-forge libstdcxx-ng -fi - # Install dependencies for code coverage (codecov.io) if [ "$RUN_COVERAGE" == "yes" ]; then $PIP_INSTALL codecov coveralls; fi diff --git a/conda-recipes/lld/bld.bat b/conda-recipes/lld/bld.bat deleted file mode 100644 index 4822dbbbb..000000000 --- a/conda-recipes/lld/bld.bat +++ /dev/null @@ -1,17 +0,0 @@ -mkdir build -cd build - -set CC=cl.exe -set CXX=cl.exe - -cmake -G "Ninja" ^ - -DCMAKE_BUILD_TYPE="Release" ^ - -DCMAKE_PREFIX_PATH=%LIBRARY_PREFIX% ^ - -DCMAKE_INSTALL_PREFIX=%LIBRARY_PREFIX% ^ - -DCMAKE_INSTALL_LIBDIR=%LIBRARY_PREFIX%\lib ^ - -DLLVM_INCLUDE_TESTS=OFF ^ - %SRC_DIR%/lld -if %ERRORLEVEL% neq 0 exit 1 - -cmake --build . --target install -if %ERRORLEVEL% neq 0 exit 1 diff --git a/conda-recipes/lld/build.sh b/conda-recipes/lld/build.sh deleted file mode 100644 index 47944f842..000000000 --- a/conda-recipes/lld/build.sh +++ /dev/null @@ -1,21 +0,0 @@ -mkdir build -cd build - -if [[ "$CONDA_BUILD_CROSS_COMPILATION" == "1" ]]; then - CMAKE_ARGS="$CMAKE_ARGS -DLLVM_CONFIG_PATH=$BUILD_PREFIX/bin/llvm-config -DLLVM_TABLEGEN_EXE=$BUILD_PREFIX/bin/llvm-tblgen" -fi - -cmake -G Ninja \ - -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DCMAKE_INSTALL_LIBDIR=$PREFIX/lib \ - -DCMAKE_PREFIX_PATH=$PREFIX \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_ENABLE_RTTI=ON \ - -DLLVM_INCLUDE_TESTS=OFF \ - -DLLVM_OBJ_ROOT=${PREFIX} \ - -DLLVM_MAIN_INCLUDE_DIR=${PREFIX}/include \ - ${CMAKE_ARGS} \ - ../lld - -cmake --build . -- -j${CPU_COUNT} -cmake --install . diff --git a/conda-recipes/lld/conda_build_config.yaml b/conda-recipes/lld/conda_build_config.yaml deleted file mode 100644 index 75dfed3db..000000000 --- a/conda-recipes/lld/conda_build_config.yaml +++ /dev/null @@ -1,39 +0,0 @@ -CI: azure -c_compiler: gcc -c_compiler_version: '11' -cdt_name: cos7 -channel_sources: conda-forge -channel_targets: conda-forge main -cpu_optimization_target: nocona -cran_mirror: https://cran.r-project.org -cxx_compiler: gxx -cxx_compiler_version: '11' -docker_image: quay.io/condaforge/linux-anvil-ppc64le -extend_keys: -- pin_run_as_build -- ignore_version -- extend_keys -- ignore_build_only_deps -fortran_compiler: gfortran -ignore_build_only_deps: -- numpy -- python -libxml2: '2.10' -lua: '5' -numpy: '1.21' -perl: 5.26.2 -pin_run_as_build: - python: - min_pin: x.x - max_pin: x.x - r-base: - min_pin: x.x - max_pin: x.x -python: '3.10' -r_base: '3.5' -target_platform: linux-ppc64le -zip_keys: -- - c_compiler_version - - cxx_compiler_version -zlib: '1.2' -zstd: '1.5' diff --git a/conda-recipes/lld/meta.yaml b/conda-recipes/lld/meta.yaml deleted file mode 100644 index 9f6729902..000000000 --- a/conda-recipes/lld/meta.yaml +++ /dev/null @@ -1,55 +0,0 @@ -{% set version = "16.0.0" %} - -package: - name: lld - version: {{ version }} - -source: - url: https://github.com/llvm/llvm-project/releases/download/llvmorg-{{ version.replace(".rc", "-rc") }}/llvm-project-{{ version.replace(".rc", "rc") }}.src.tar.xz - sha256: 9a56d906a2c81f16f06efc493a646d497c53c2f4f28f0cb1f3c8da7f74350254 - patches: - - patches/0001-point-header-inclusion-in-lld-MachO-CMakeLists.txt-t.patch - -build: - number: 0 - ignore_run_exports_from: - # not actually needed on unix - - libxml2 # [unix] - -requirements: - build: - - {{ compiler('c') }} - - {{ compiler('cxx') }} - - cmake - - ninja - - llvmdev =={{ version }} # [build_platform != target_platform] - host: - - libcxx {{ cxx_compiler_version }} # [osx] - - llvmdev =={{ version }} - - llvm - # these they are present in the host env anyway as run-deps of llvm - # and will be picked up for the build by the default settings - - libxml2 - - zlib - - zstd - run: - - {{ pin_compatible("libcxx", max_pin=None) }} # [osx] - run_constrained: - - llvm =={{ version }} - -test: - commands: - - ld.lld --version # [linux] - - lld-link /? # [win] - -about: - home: https://lld.llvm.org/ - license: Apache-2.0 WITH LLVM-exception - license_file: lld/LICENSE.TXT - summary: The LLVM Linker - dev_url: https://github.com/llvm/llvm-project/ - -extra: - recipe-maintainers: - - isuruf - - h-vetinari diff --git a/conda-recipes/lld/recipe-scripts-license.txt b/conda-recipes/lld/recipe-scripts-license.txt deleted file mode 100644 index 2ec51d75f..000000000 --- a/conda-recipes/lld/recipe-scripts-license.txt +++ /dev/null @@ -1,27 +0,0 @@ -BSD-3-Clause license -Copyright (c) 2015-2022, conda-forge contributors -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH -DAMAGE. diff --git a/conda-recipes/lld/patches/0001-point-header-inclusion-in-lld-MachO-CMakeLists.txt-t.patch b/ffi/MachoLLDpatch.patch similarity index 97% rename from conda-recipes/lld/patches/0001-point-header-inclusion-in-lld-MachO-CMakeLists.txt-t.patch rename to ffi/MachoLLDpatch.patch index 46d729f03..769de8d28 100644 --- a/conda-recipes/lld/patches/0001-point-header-inclusion-in-lld-MachO-CMakeLists.txt-t.patch +++ b/ffi/MachoLLDpatch.patch @@ -14,12 +14,11 @@ index ea26889267..3e06eb78fc 100644 @@ -2,7 +2,7 @@ set(LLVM_TARGET_DEFINITIONS Options.td) tablegen(LLVM Options.inc -gen-opt-parser-defs) add_public_tablegen_target(MachOOptionsTableGen) - + -include_directories(${LLVM_MAIN_SRC_DIR}/../libunwind/include) +include_directories($ENV{SRC_DIR}/libunwind/include) - + add_lld_library(lldMachO Arch/ARM.cpp -- -2.38.1.windows.1 - +2.38.1.windows.1 \ No newline at end of file diff --git a/ffi/build.py b/ffi/build.py index cfb5dd209..9827c3226 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -172,6 +172,7 @@ def main_posix(kind, library_ext): (version, _) = out.split('.', 1) version = int(version) if version == 16: + subprocess.check_call(['sh', f'{here_dir}/build_lld.sh']) msg = ("Building with LLVM 16; note that LLVM 16 support is " "presently experimental") show_warning(msg) @@ -194,6 +195,8 @@ def main_posix(kind, library_ext): # on OSX cxxflags has null bytes at the end of the string, remove them cxxflags = cxxflags.replace('\0', '') cxxflags = cxxflags.split() + ['-fno-rtti', '-g'] + if version == 16: + cxxflags += [f"-L{here_dir}/lldbuild/lib -I{here_dir}/lldbuild/include"] # look for SVML include_dir = run_llvm_config(llvm_config, ['--includedir']).strip() diff --git a/ffi/build_lld.sh b/ffi/build_lld.sh new file mode 100644 index 000000000..00a56c66e --- /dev/null +++ b/ffi/build_lld.sh @@ -0,0 +1,53 @@ +# curl -L https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.0/llvm-project-16.0.0.src.tar.xz -o llvm-project-16.0.0.tar.xz +# tar -xf llvm-project-16.0.0.tar.xz +cd . +cwd=$PWD +cd llvm-project-16.0.0.src +git apply ../MachoLLDpatch.path +cd ../ + +# cd $CONDA_PREFIX +mkdir lldbuild +cd lldbuild + + + + +# if [[ "$target_platform" == "linux-64" ]]; then +# CMAKE_ARGS="${CMAKE_ARGS} -DLLVM_USE_INTEL_JITEVENTS=ON" +# fi + +# if [[ "$CC_FOR_BUILD" != "" && "$CC_FOR_BUILD" != "$CC" ]]; then +# CMAKE_ARGS="${CMAKE_ARGS} -DCROSS_TOOLCHAIN_FLAGS_NATIVE=-DCMAKE_C_COMPILER=$CC_FOR_BUILD;-DCMAKE_CXX_COMPILER=$CXX_FOR_BUILD;-DCMAKE_C_FLAGS=-O2;-DCMAKE_CXX_FLAGS=-O2;-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath,${BUILD_PREFIX}/lib;-DCMAKE_MODULE_LINKER_FLAGS=;-DCMAKE_SHARED_LINKER_FLAGS=;-DCMAKE_STATIC_LINKER_FLAGS=;-DLLVM_INCLUDE_BENCHMARKS=OFF;" +# CMAKE_ARGS="${CMAKE_ARGS} -DLLVM_HOST_TRIPLE=$(echo $HOST | sed s/conda/unknown/g) -DLLVM_DEFAULT_TARGET_TRIPLE=$(echo $HOST | sed s/conda/unknown/g)" +# fi + +# # disable -fno-plt due to https://bugs.llvm.org/show_bug.cgi?id=51863 due to some GCC bug +# if [[ "$target_platform" == "linux-ppc64le" ]]; then +# CFLAGS="$(echo $CFLAGS | sed 's/-fno-plt //g')" +# CXXFLAGS="$(echo $CXXFLAGS | sed 's/-fno-plt //g')" +# CMAKE_ARGS="${CMAKE_ARGS} -DFFI_INCLUDE_DIR=$PREFIX/include" +# CMAKE_ARGS="${CMAKE_ARGS} -DFFI_LIBRARY_DIR=$PREFIX/lib" +# fi + +# if [[ $target_platform == osx-arm64 ]]; then +# CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_ENABLE_WERROR=FALSE" +# fi + +cmake --fresh -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_RTTI=ON \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DCMAKE_PREFIX_PATH=${CONDA_PREFIX} \ + -DCMAKE_LIBRARY_PATH=${CONDA_PREFIX}/lib \ + ${CMAKE_ARGS} \ + ${cwd}/llvm-project-16.0.0.src/lld + # -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \ + # -DCMAKE_INSTALL_LIBDIR=$CONDA_PREFIX/lib \ + # -DCMAKE_PREFIX_PATH=$CONDA_PREFIX \ + # -DLLVM_MAIN_INCLUDE_DIR=${CONDA_PREFIX}/include \ + # -DLLVM_OBJ_ROOT=${CONDA_PREFIX} \ + +make +# cmake --fresh --build . +# cmake --install . From a3e6e6e0511264e1849bade5e634c5d04301005d Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sun, 19 Jan 2025 18:30:20 -0500 Subject: [PATCH 148/158] uncomment the important stuff --- ffi/build_lld.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ffi/build_lld.sh b/ffi/build_lld.sh index 00a56c66e..5e5f7abdf 100644 --- a/ffi/build_lld.sh +++ b/ffi/build_lld.sh @@ -1,5 +1,5 @@ -# curl -L https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.0/llvm-project-16.0.0.src.tar.xz -o llvm-project-16.0.0.tar.xz -# tar -xf llvm-project-16.0.0.tar.xz +curl -L https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.0/llvm-project-16.0.0.src.tar.xz -o llvm-project-16.0.0.tar.xz +tar -xf llvm-project-16.0.0.tar.xz cd . cwd=$PWD cd llvm-project-16.0.0.src From 91d2ccc9a9180bce81bac3067bdfcb3f7881c915 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sun, 19 Jan 2025 19:23:25 -0500 Subject: [PATCH 149/158] add -whole-archive flag --- buildscripts/incremental/setup_conda_environment.sh | 12 ++++++------ ffi/Makefile.linux | 2 +- ffi/build_lld.sh | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index c08f06a4a..c5645d3df 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -29,21 +29,21 @@ set -v $CONDA_INSTALL -c numba libxml2 $CONDA_INSTALL -c conda-forge libcrc32c -if [[ $(uname) == Linux ]]; then - # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 - $CONDA_INSTALL -c conda-forge libstdcxx-ng -fi - # Install llvmdev (separate channel, for now) if [ "$LLVM" == "16" ]; then # also install lld, not required when using the numba channel because the numba # channel includes lld. - $CONDA_INSTALL -c conda-forge conda-build llvmdev="16" + $CONDA_INSTALL -c conda-forge llvmdev="16" else # Install the compiler toolchain, for osx, bootstrapping needed # which happens in build.sh $CONDA_INSTALL -c numba llvmdev="15.*" fi +if [[ $(uname) == Linux ]]; then + # $CONDA_INSTALL gcc_linux-64 gxx_linux-64 + $CONDA_INSTALL -c conda-forge libstdcxx-ng +fi + # Install dependencies for code coverage (codecov.io) if [ "$RUN_COVERAGE" == "yes" ]; then $PIP_INSTALL codecov coveralls; fi diff --git a/ffi/Makefile.linux b/ffi/Makefile.linux index 9f9fe627c..44dfde26a 100644 --- a/ffi/Makefile.linux +++ b/ffi/Makefile.linux @@ -9,7 +9,7 @@ CXX_FPIC_FLAGS ?= -fPIC CXXFLAGS := $(CPPFLAGS) $(CXXFLAGS) $(LLVM_CXXFLAGS) $(CXX_FLTO_FLAGS) $(CXX_FPIC_FLAGS) LDFLAGS := $(LDFLAGS) $(LLVM_LDFLAGS) $(LD_FLTO_FLAGS) -LIBS = $(LLVM_LIBS) +LIBS = -rdynamic -Wl,-whole-archive $(LLVM_LIBS) -Wl,-no-whole-archive INCLUDE = core.h OBJ = assembly.o bitcode.o core.o initfini.o module.o value.o \ executionengine.o transforms.o passmanagers.o targets.o type.o dylib.o \ diff --git a/ffi/build_lld.sh b/ffi/build_lld.sh index 5e5f7abdf..2dddbbe4f 100644 --- a/ffi/build_lld.sh +++ b/ffi/build_lld.sh @@ -1,5 +1,5 @@ -curl -L https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.0/llvm-project-16.0.0.src.tar.xz -o llvm-project-16.0.0.tar.xz -tar -xf llvm-project-16.0.0.tar.xz +# curl -L https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.0/llvm-project-16.0.0.src.tar.xz -o llvm-project-16.0.0.tar.xz +# tar -xf llvm-project-16.0.0.tar.xz cd . cwd=$PWD cd llvm-project-16.0.0.src @@ -40,6 +40,8 @@ cmake --fresh -G "Unix Makefiles" \ -DLLVM_INCLUDE_TESTS=OFF \ -DCMAKE_PREFIX_PATH=${CONDA_PREFIX} \ -DCMAKE_LIBRARY_PATH=${CONDA_PREFIX}/lib \ + -DCMAKE_SHARED_LINKER_FLAGS="-stdc++" \ + -DLLD_BUILT_STANDALONE=ON \ ${CMAKE_ARGS} \ ${cwd}/llvm-project-16.0.0.src/lld # -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \ From 7fab7bed21a85149e186cc78f3776649e30704ec Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sun, 19 Jan 2025 19:24:01 -0500 Subject: [PATCH 150/158] remember to uncomment first 2 lines (again) --- ffi/build_lld.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ffi/build_lld.sh b/ffi/build_lld.sh index 2dddbbe4f..f56f161a0 100644 --- a/ffi/build_lld.sh +++ b/ffi/build_lld.sh @@ -1,5 +1,5 @@ -# curl -L https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.0/llvm-project-16.0.0.src.tar.xz -o llvm-project-16.0.0.tar.xz -# tar -xf llvm-project-16.0.0.tar.xz +curl -L https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.0/llvm-project-16.0.0.src.tar.xz -o llvm-project-16.0.0.tar.xz +tar -xf llvm-project-16.0.0.tar.xz cd . cwd=$PWD cd llvm-project-16.0.0.src From 7e3590934cacbf0f0f8be0657ca31d4d6dac2d1e Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Sun, 19 Jan 2025 19:35:09 -0500 Subject: [PATCH 151/158] only add additional flags when using llvm16 --- buildscripts/incremental/setup_conda_environment.sh | 2 +- ffi/Makefile.linux | 2 +- ffi/build.py | 8 +++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index c5645d3df..85d81d61c 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -33,7 +33,7 @@ $CONDA_INSTALL -c conda-forge libcrc32c if [ "$LLVM" == "16" ]; then # also install lld, not required when using the numba channel because the numba # channel includes lld. - $CONDA_INSTALL -c conda-forge llvmdev="16" + $CONDA_INSTALL -c conda-forge llvmdev="16" lld="16" else # Install the compiler toolchain, for osx, bootstrapping needed # which happens in build.sh diff --git a/ffi/Makefile.linux b/ffi/Makefile.linux index 44dfde26a..9f9fe627c 100644 --- a/ffi/Makefile.linux +++ b/ffi/Makefile.linux @@ -9,7 +9,7 @@ CXX_FPIC_FLAGS ?= -fPIC CXXFLAGS := $(CPPFLAGS) $(CXXFLAGS) $(LLVM_CXXFLAGS) $(CXX_FLTO_FLAGS) $(CXX_FPIC_FLAGS) LDFLAGS := $(LDFLAGS) $(LLVM_LDFLAGS) $(LD_FLTO_FLAGS) -LIBS = -rdynamic -Wl,-whole-archive $(LLVM_LIBS) -Wl,-no-whole-archive +LIBS = $(LLVM_LIBS) INCLUDE = core.h OBJ = assembly.o bitcode.o core.o initfini.o module.o value.o \ executionengine.o transforms.o passmanagers.o targets.o type.o dylib.o \ diff --git a/ffi/build.py b/ffi/build.py index 9827c3226..1fdc61e25 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -172,7 +172,7 @@ def main_posix(kind, library_ext): (version, _) = out.split('.', 1) version = int(version) if version == 16: - subprocess.check_call(['sh', f'{here_dir}/build_lld.sh']) + # subprocess.check_call(['sh', f'{here_dir}/build_lld.sh']) msg = ("Building with LLVM 16; note that LLVM 16 support is " "presently experimental") show_warning(msg) @@ -190,13 +190,15 @@ def main_posix(kind, library_ext): # Normalize whitespace (trim newlines) os.environ['LLVM_LIBS'] = \ f'{cxx_lld_libs} ' + ' '.join(libs.split()) + if version == 16: + os.environ['LLVM_LIBS'] = f"-rdynamic -Wl,-whole-archive {os.environ['LLVM_LIBS']} -Wl,-no-whole-archive" cxxflags = run_llvm_config(llvm_config, ["--cxxflags"]) # on OSX cxxflags has null bytes at the end of the string, remove them cxxflags = cxxflags.replace('\0', '') cxxflags = cxxflags.split() + ['-fno-rtti', '-g'] - if version == 16: - cxxflags += [f"-L{here_dir}/lldbuild/lib -I{here_dir}/lldbuild/include"] + # if version == 16: + # cxxflags += [f"-L{here_dir}/lldbuild/lib -I{here_dir}/lldbuild/include"] # look for SVML include_dir = run_llvm_config(llvm_config, ['--includedir']).strip() From 6a6db4fe90e5c6fede9e4e44e79714b702d36ea7 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 20 Jan 2025 16:19:40 -0500 Subject: [PATCH 152/158] make sure additional linker flags only on linux --- ffi/build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ffi/build.py b/ffi/build.py index 1fdc61e25..5e4143431 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -190,7 +190,8 @@ def main_posix(kind, library_ext): # Normalize whitespace (trim newlines) os.environ['LLVM_LIBS'] = \ f'{cxx_lld_libs} ' + ' '.join(libs.split()) - if version == 16: + # A necessary fix to use the conda-forge version of lld. + if version == 16 and sys.platform.startswith(('linux', 'gnu')): os.environ['LLVM_LIBS'] = f"-rdynamic -Wl,-whole-archive {os.environ['LLVM_LIBS']} -Wl,-no-whole-archive" cxxflags = run_llvm_config(llvm_config, ["--cxxflags"]) From 253fc2fd0bbcaebaa8d6442f380fd0fb9e20e783 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 20 Jan 2025 17:30:22 -0500 Subject: [PATCH 153/158] try adding zstd as a lib when linking on osx --- ffi/Makefile.osx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/Makefile.osx b/ffi/Makefile.osx index 5d14b97fc..c4d19b14f 100644 --- a/ffi/Makefile.osx +++ b/ffi/Makefile.osx @@ -4,7 +4,7 @@ CXXFLAGS = $(LLVM_CXXFLAGS) # Only export the LLVMPY symbols we require and exclude everything else. EXPORT = "-Wl,-exported_symbol,_LLVMPY_*" LDFLAGS := $(LDFLAGS) $(EXPORT) $(LLVM_LDFLAGS) -LIBS = $(LLVM_LIBS) +LIBS = $(LLVM_LIBS) -lzstd INCLUDE = core.h SRC = assembly.cpp bitcode.cpp core.cpp initfini.cpp module.cpp value.cpp \ executionengine.cpp transforms.cpp passmanagers.cpp targets.cpp type.cpp \ From afcd3cffaae986cd95510486d043fab93ddd6639 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 20 Jan 2025 17:38:28 -0500 Subject: [PATCH 154/158] add dependency on zstd-static --- buildscripts/incremental/setup_conda_environment.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index 85d81d61c..6b1737127 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -33,7 +33,7 @@ $CONDA_INSTALL -c conda-forge libcrc32c if [ "$LLVM" == "16" ]; then # also install lld, not required when using the numba channel because the numba # channel includes lld. - $CONDA_INSTALL -c conda-forge llvmdev="16" lld="16" + $CONDA_INSTALL -c conda-forge llvmdev="16" lld="16" zstd-static else # Install the compiler toolchain, for osx, bootstrapping needed # which happens in build.sh From 637777ee56d3f6ff2445d7f36583b7c30b268a21 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 20 Jan 2025 18:03:05 -0500 Subject: [PATCH 155/158] use .a instead --- buildscripts/incremental/setup_conda_environment.sh | 1 - ffi/Makefile.osx | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/buildscripts/incremental/setup_conda_environment.sh b/buildscripts/incremental/setup_conda_environment.sh index 6b1737127..7db7d3a83 100755 --- a/buildscripts/incremental/setup_conda_environment.sh +++ b/buildscripts/incremental/setup_conda_environment.sh @@ -27,7 +27,6 @@ source activate $CONDA_ENV set -v $CONDA_INSTALL -c numba libxml2 -$CONDA_INSTALL -c conda-forge libcrc32c # Install llvmdev (separate channel, for now) if [ "$LLVM" == "16" ]; then diff --git a/ffi/Makefile.osx b/ffi/Makefile.osx index c4d19b14f..8d41a3109 100644 --- a/ffi/Makefile.osx +++ b/ffi/Makefile.osx @@ -4,7 +4,7 @@ CXXFLAGS = $(LLVM_CXXFLAGS) # Only export the LLVMPY symbols we require and exclude everything else. EXPORT = "-Wl,-exported_symbol,_LLVMPY_*" LDFLAGS := $(LDFLAGS) $(EXPORT) $(LLVM_LDFLAGS) -LIBS = $(LLVM_LIBS) -lzstd +LIBS = $(LLVM_LIBS) -l:libzstd.a -l:libz.a INCLUDE = core.h SRC = assembly.cpp bitcode.cpp core.cpp initfini.cpp module.cpp value.cpp \ executionengine.cpp transforms.cpp passmanagers.cpp targets.cpp type.cpp \ From 9ec51c299e879ad8477e70ebfd03243516f376c7 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 20 Jan 2025 18:08:58 -0500 Subject: [PATCH 156/158] revert the .a change --- ffi/Makefile.osx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/Makefile.osx b/ffi/Makefile.osx index 8d41a3109..21ea6cf44 100644 --- a/ffi/Makefile.osx +++ b/ffi/Makefile.osx @@ -4,7 +4,7 @@ CXXFLAGS = $(LLVM_CXXFLAGS) # Only export the LLVMPY symbols we require and exclude everything else. EXPORT = "-Wl,-exported_symbol,_LLVMPY_*" LDFLAGS := $(LDFLAGS) $(EXPORT) $(LLVM_LDFLAGS) -LIBS = $(LLVM_LIBS) -l:libzstd.a -l:libz.a +LIBS = $(LLVM_LIBS) -lzstd -lz INCLUDE = core.h SRC = assembly.cpp bitcode.cpp core.cpp initfini.cpp module.cpp value.cpp \ executionengine.cpp transforms.cpp passmanagers.cpp targets.cpp type.cpp \ From b3f1b8ed878134a13a4e751d1d67a41fc938db51 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 20 Jan 2025 18:15:42 -0500 Subject: [PATCH 157/158] remove lld build script --- .gitignore | 3 --- ffi/MachoLLDpatch.patch | 24 ------------------ ffi/build.py | 2 -- ffi/build_lld.sh | 55 ----------------------------------------- 4 files changed, 84 deletions(-) delete mode 100644 ffi/MachoLLDpatch.patch delete mode 100644 ffi/build_lld.sh diff --git a/.gitignore b/.gitignore index 02d9a3b6e..dd191b234 100644 --- a/.gitignore +++ b/.gitignore @@ -5,9 +5,6 @@ *.dylib *.dll build/ -ffi/*.tar.xz -ffi/lldbuild/ -ffi/llvm-project-16.0.0.src/ dist/ *.egg-info ffi/build/ diff --git a/ffi/MachoLLDpatch.patch b/ffi/MachoLLDpatch.patch deleted file mode 100644 index 769de8d28..000000000 --- a/ffi/MachoLLDpatch.patch +++ /dev/null @@ -1,24 +0,0 @@ -From 14d6c677c1758066f1ef10635439754cc3161057 Mon Sep 17 00:00:00 2001 -From: "H. Vetinari" -Date: Wed, 14 Jul 2021 21:53:53 +0200 -Subject: [PATCH] point header inclusion in lld/MachO/CMakeLists.txt to SRC_DIR - ---- - lld/MachO/CMakeLists.txt | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/lld/MachO/CMakeLists.txt b/lld/MachO/CMakeLists.txt -index ea26889267..3e06eb78fc 100644 ---- a/lld/MachO/CMakeLists.txt -+++ b/lld/MachO/CMakeLists.txt -@@ -2,7 +2,7 @@ set(LLVM_TARGET_DEFINITIONS Options.td) - tablegen(LLVM Options.inc -gen-opt-parser-defs) - add_public_tablegen_target(MachOOptionsTableGen) - --include_directories(${LLVM_MAIN_SRC_DIR}/../libunwind/include) -+include_directories($ENV{SRC_DIR}/libunwind/include) - - add_lld_library(lldMachO - Arch/ARM.cpp --- -2.38.1.windows.1 \ No newline at end of file diff --git a/ffi/build.py b/ffi/build.py index 5e4143431..7b3f409bd 100755 --- a/ffi/build.py +++ b/ffi/build.py @@ -198,8 +198,6 @@ def main_posix(kind, library_ext): # on OSX cxxflags has null bytes at the end of the string, remove them cxxflags = cxxflags.replace('\0', '') cxxflags = cxxflags.split() + ['-fno-rtti', '-g'] - # if version == 16: - # cxxflags += [f"-L{here_dir}/lldbuild/lib -I{here_dir}/lldbuild/include"] # look for SVML include_dir = run_llvm_config(llvm_config, ['--includedir']).strip() diff --git a/ffi/build_lld.sh b/ffi/build_lld.sh deleted file mode 100644 index f56f161a0..000000000 --- a/ffi/build_lld.sh +++ /dev/null @@ -1,55 +0,0 @@ -curl -L https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.0/llvm-project-16.0.0.src.tar.xz -o llvm-project-16.0.0.tar.xz -tar -xf llvm-project-16.0.0.tar.xz -cd . -cwd=$PWD -cd llvm-project-16.0.0.src -git apply ../MachoLLDpatch.path -cd ../ - -# cd $CONDA_PREFIX -mkdir lldbuild -cd lldbuild - - - - -# if [[ "$target_platform" == "linux-64" ]]; then -# CMAKE_ARGS="${CMAKE_ARGS} -DLLVM_USE_INTEL_JITEVENTS=ON" -# fi - -# if [[ "$CC_FOR_BUILD" != "" && "$CC_FOR_BUILD" != "$CC" ]]; then -# CMAKE_ARGS="${CMAKE_ARGS} -DCROSS_TOOLCHAIN_FLAGS_NATIVE=-DCMAKE_C_COMPILER=$CC_FOR_BUILD;-DCMAKE_CXX_COMPILER=$CXX_FOR_BUILD;-DCMAKE_C_FLAGS=-O2;-DCMAKE_CXX_FLAGS=-O2;-DCMAKE_EXE_LINKER_FLAGS=-Wl,-rpath,${BUILD_PREFIX}/lib;-DCMAKE_MODULE_LINKER_FLAGS=;-DCMAKE_SHARED_LINKER_FLAGS=;-DCMAKE_STATIC_LINKER_FLAGS=;-DLLVM_INCLUDE_BENCHMARKS=OFF;" -# CMAKE_ARGS="${CMAKE_ARGS} -DLLVM_HOST_TRIPLE=$(echo $HOST | sed s/conda/unknown/g) -DLLVM_DEFAULT_TARGET_TRIPLE=$(echo $HOST | sed s/conda/unknown/g)" -# fi - -# # disable -fno-plt due to https://bugs.llvm.org/show_bug.cgi?id=51863 due to some GCC bug -# if [[ "$target_platform" == "linux-ppc64le" ]]; then -# CFLAGS="$(echo $CFLAGS | sed 's/-fno-plt //g')" -# CXXFLAGS="$(echo $CXXFLAGS | sed 's/-fno-plt //g')" -# CMAKE_ARGS="${CMAKE_ARGS} -DFFI_INCLUDE_DIR=$PREFIX/include" -# CMAKE_ARGS="${CMAKE_ARGS} -DFFI_LIBRARY_DIR=$PREFIX/lib" -# fi - -# if [[ $target_platform == osx-arm64 ]]; then -# CMAKE_ARGS="${CMAKE_ARGS} -DCMAKE_ENABLE_WERROR=FALSE" -# fi - -cmake --fresh -G "Unix Makefiles" \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_ENABLE_RTTI=ON \ - -DLLVM_INCLUDE_TESTS=OFF \ - -DCMAKE_PREFIX_PATH=${CONDA_PREFIX} \ - -DCMAKE_LIBRARY_PATH=${CONDA_PREFIX}/lib \ - -DCMAKE_SHARED_LINKER_FLAGS="-stdc++" \ - -DLLD_BUILT_STANDALONE=ON \ - ${CMAKE_ARGS} \ - ${cwd}/llvm-project-16.0.0.src/lld - # -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \ - # -DCMAKE_INSTALL_LIBDIR=$CONDA_PREFIX/lib \ - # -DCMAKE_PREFIX_PATH=$CONDA_PREFIX \ - # -DLLVM_MAIN_INCLUDE_DIR=${CONDA_PREFIX}/include \ - # -DLLVM_OBJ_ROOT=${CONDA_PREFIX} \ - -make -# cmake --fresh --build . -# cmake --install . From 16e8cfdb9ed17cbdb602cd3ce3786c08c9bb34f2 Mon Sep 17 00:00:00 2001 From: Benjamin Austin Jr Date: Mon, 20 Jan 2025 18:21:38 -0500 Subject: [PATCH 158/158] remove deprecated get_html_theme_path --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 5682279a3..835884ef1 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -123,7 +123,7 @@ # a list of builtin themes. html_theme = 'sphinx_rtd_theme' -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +html_theme_path = [] # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the