diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1599492..edae08f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,139 +2,178 @@ name: CI on: push: - branches: [ master ] + branches: [master] pull_request: - branches: [ master ] + branches: [master] release: - types: [ published ] + types: [published] jobs: - - build_windows_gui: needs: [build] runs-on: windows-latest env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} strategy: matrix: node-version: [18.x] steps: - - uses: actions/checkout@v2 - - name: Download artifact - uses: actions/download-artifact@v1 - with: + - uses: actions/checkout@v2 + - name: Download artifact + uses: actions/download-artifact@v1 + with: name: wheels-3.9 path: gui - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - - uses: conda-incubator/setup-miniconda@v2 - with: - auto-activate-base: true - activate-environment: "" - - name: Build Electron app - run: | - cd gui - dir - conda create --name ETA python=3.11 - conda activate ETA - conda install conda-forge::conda-pack - pip install --find-links=.\ etabackend - conda pack -n ETA -o bundle_eta.tar.gz - mkdir eta_env - tar -xzf bundle_eta.tar.gz -C eta_env - yarn - yarn dist - cd .. - ls .\gui\dist\ - echo 7z a ETA_Install-win64.zip .\gui\dist\*.exe - mkdir artifact - move .\gui\dist\*.exe .\artifact - - name: Upload artifact - uses: actions/upload-artifact@v1 - with: + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - uses: conda-incubator/setup-miniconda@v2 + with: + auto-activate-base: true + activate-environment: "" + - name: Build Electron app + run: | + cd gui + dir + conda create --name ETA python=3.11 + conda activate ETA + conda install conda-forge::conda-pack + pip install --find-links=.\ etabackend + conda pack -n ETA -o bundle_eta.tar.gz + mkdir eta_env + tar -xzf bundle_eta.tar.gz -C eta_env + yarn + yarn dist + cd .. + ls .\gui\dist\ + echo 7z a ETA_Install-win64.zip .\gui\dist\*.exe + mkdir artifact + move .\gui\dist\*.exe .\artifact + - name: Upload artifact + uses: actions/upload-artifact@v1 + with: name: ETA_Install-win64 path: artifact + # compile-m1: + # runs-on: macos-latest + # strategy: + # matrix: + # python-version: [3.9] + # steps: + # - uses: actions/checkout@v2 + # - name: Set up Python ${{ matrix.python-version }} + # uses: actions/setup-python@v4 + # with: + # python-version: ${{ matrix.python-version }} + # - name: Install clang-12 + # run: | + # brew install llvm@12 + # - name: Install dependencies and build + # run: | + # pip3 install virtualenv + # virtualenv venv + # source venv/bin/activate + # ls + # export PATH="/usr/local/opt/llvm@12/bin:$PATH" + # export LDFLAGS="-L/opt/homebrew/opt/llvm@12/lib" + # export CPPFLAGS="-I/opt/homebrew/opt/llvm@12/include" + # export CXXFLAGS="-I/opt/homebrew/opt/llvm@12/include" + # clang etabackend/cpp/PARSE_TimeTagFileHeader.cpp -S -emit-llvm + # clang etabackend/cpp/PARSE_TimeTags.cpp -S -emit-llvm + # clang etabackend/cpp/INFRA_vchn.cpp -S -emit-llvm + # mv *.ll etabackend/ll/m1/ + # pip install -e ./ + # pytest + # - name: Upload artifact + # uses: actions/upload-artifact@v1 + # with: + # name: ll-m1 + # path: etabackend/ll/m1 build: + #needs: [compile-m1] runs-on: ubuntu-latest strategy: matrix: python-version: [3.9] steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies and build - run: | - sudo apt-get update - sudo apt-get install -y clang-11 - sudo apt-get install -y llvm-11 - pip3 install virtualenv - virtualenv venv - source venv/bin/activate - ls - clang-11 etabackend/cpp/PARSE_TimeTagFileHeader.cpp -S -emit-llvm - clang-11 etabackend/cpp/PARSE_TimeTags.cpp -S -emit-llvm - clang-11 etabackend/cpp/INFRA_vchn.cpp -S -emit-llvm - mv *.ll etabackend/ll/posix - python3 setup.py bdist_wheel - - name: Upload artifact - uses: actions/upload-artifact@v1 - with: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies and build + run: | + sudo apt-get update + sudo apt-get install -y clang-11 llvm-11 gcc-multilib + pip3 install virtualenv + virtualenv venv + source venv/bin/activate + ls + clang-11 etabackend/cpp/PARSE_TimeTagFileHeader.cpp -S -emit-llvm + clang-11 etabackend/cpp/PARSE_TimeTags.cpp -S -emit-llvm + clang-11 etabackend/cpp/INFRA_vchn.cpp -S -emit-llvm + mv *.ll etabackend/ll/posix + clang-11 -target aarch64-linux-gnu etabackend/cpp/PARSE_TimeTagFileHeader.cpp -S -emit-llvm + clang-11 -target aarch64-linux-gnu etabackend/cpp/PARSE_TimeTags.cpp -S -emit-llvm + clang-11 -target aarch64-linux-gnu etabackend/cpp/INFRA_vchn.cpp -S -emit-llvm + mv *.ll etabackend/ll/m1 + python3 setup.py bdist_wheel + - name: Upload artifact + uses: actions/upload-artifact@v1 + with: name: wheels-${{ matrix.python-version }} path: dist publish: - needs: [build,test] + needs: [build, test] if: ${{ github.event_name == 'release' }} runs-on: ubuntu-latest env: - PYPI_PASSWORD: ${{secrets.PYPI_PASSWORD}} - TWINE_PASSWORD: ${{secrets.PYPI_PASSWORD}} - TWINE_USERNAME: ${{secrets.PYPI_USERNAME}} - PYPI_USERNAME: ${{secrets.PYPI_USERNAME}} + PYPI_PASSWORD: ${{secrets.PYPI_PASSWORD}} + TWINE_PASSWORD: ${{secrets.PYPI_PASSWORD}} + TWINE_USERNAME: ${{secrets.PYPI_USERNAME}} + PYPI_USERNAME: ${{secrets.PYPI_USERNAME}} strategy: matrix: python-version: [3.9] steps: - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - pip3 install twine - - name: Download artifact - uses: actions/download-artifact@v1 - with: + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + pip3 install twine + - name: Download artifact + uses: actions/download-artifact@v1 + with: name: wheels-3.9 path: dist - - name: Twine upload - run: | - ls - twine upload dist/*.whl --skip-existing + - name: Twine upload + run: | + ls + twine upload dist/*.whl --skip-existing test: needs: [build] runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-latest, windows-latest] + os: [ubuntu-latest, windows-latest, macos-13, macos-latest] python-version: ["3.9", "3.11"] steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Download artifact - uses: actions/download-artifact@v1 - with: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Download artifact + uses: actions/download-artifact@v1 + with: name: wheels-3.9 path: dist - - name: Test with pytest - run: | - python -m pip install tox tox-gh-actions - tox + - name: Test with pytest + run: | + python -m pip install pytest + mv ./dist/*.whl ./etabackend-9.9.9-py3-none-any.whl + python -m pip install ./etabackend-9.9.9-py3-none-any.whl + pytest diff --git a/MANIFEST.in b/MANIFEST.in index a36d63b..2d92f39 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ include requirements.txt include etabackend/ll/nt/*.* include etabackend/ll/posix/*.* +include etabackend/ll/m1/*.* graft etabackend/static \ No newline at end of file diff --git a/etabackend/etalang/jit_linker.py b/etabackend/etalang/jit_linker.py index b1346f2..f6e3918 100644 --- a/etabackend/etalang/jit_linker.py +++ b/etabackend/etalang/jit_linker.py @@ -3,6 +3,7 @@ import inspect import math import os +import platform import sys from os import listdir from pathlib import Path @@ -44,7 +45,7 @@ def visit_FunctionDef(self, node): return self.generic_visit(node) -def compile_library(context, asm, libname='compiled_module'): +def compile_library(context, asm, libname="compiled_module"): library = context.codegen().create_library(libname) ll_module = ll.parse_assembly(asm) ll_module.verify() @@ -79,17 +80,16 @@ def BytesofRecords_get(typingctx): sig = nb.core.typing.signature(nb.int64) def codegen(context, builder, sig, args): - library = compile_library( - context, llvm_global_get.replace("test", name)) + library = compile_library(context, llvm_global_get.replace("test", name)) # no more weird hack to get the library linked context.active_code_library.add_linking_library(library) argtypes = [context.get_argument_type(aty) for aty in sig.args] restype = context.get_argument_type(sig.return_type) fnty = ir.FunctionType(restype, argtypes) fn = nb.core.cgutils.insert_pure_function( - builder.module, fnty, name=name + "_get") - retval = context.call_external_function( - builder, fn, sig.args, args) + builder.module, fnty, name=name + "_get" + ) + retval = context.call_external_function(builder, fn, sig.args, args) # print(fn) return retval @@ -100,17 +100,16 @@ def BytesofRecords_set(typingctx, param1): def codegen(context, builder, sig, args): code = llvm_global_set.replace("test", name) - library = compile_library( - context, code) + library = compile_library(context, code) # no more weird hack to get the library linked context.active_code_library.add_linking_library(library) argtypes = [context.get_argument_type(aty) for aty in sig.args] restype = context.get_argument_type(sig.return_type) fnty = ir.FunctionType(restype, argtypes) fn = nb.core.cgutils.insert_pure_function( - builder.module, fnty, name=name + "_set") - retval = context.call_external_function( - builder, fn, sig.args, args) + builder.module, fnty, name=name + "_set" + ) + retval = context.call_external_function(builder, fn, sig.args, args) # print(fn) return retval @@ -128,7 +127,8 @@ def link_libs(typingctx=None): def codegen(context, builder, sig, args): # print("===== linking =====") - ll_path = Path(__file__).resolve().parent.parent/"ll" / os.name + foldername = "m1" if platform.machine() == "arm64" else os.name + ll_path = Path(__file__).resolve().parent.parent / "ll" / foldername for f in listdir(ll_path): lib_path = Path(ll_path) / f if lib_path.is_file() and f.find(".ll") >= 0: @@ -136,9 +136,11 @@ def codegen(context, builder, sig, args): with open(lib_path, "r") as fio: assembly = fio.read() assembly = assembly.replace( - """!llvm.linker.options = !{!0}""", "") # hack: remove useless linker options for LLVM7 + """!llvm.linker.options = !{!0}""", "" + ) # hack: remove useless linker options for LLVM7 library = compile_library( - context, assembly, str(lib_path.resolve())) + context, assembly, str(lib_path.resolve()) + ) # no more weird hack to get the library linked context.active_code_library.add_linking_library(library) @@ -150,14 +152,20 @@ def codegen(context, builder, sig, args): def link_function(func_name="", param=1, i64ret=False): typer = "nb.int32" - if (i64ret): + if i64ret: typer = "nb.int64" para = ",".join(["a{}".format(i) for i in range(0, param)]) args = { - "ARB_PARAM": ast.parse("def ARB_PARAM(typingctx, {para}): pass".format(para=para)), + "ARB_PARAM": ast.parse( + "def ARB_PARAM(typingctx, {para}): pass".format(para=para) + ), "func_name": ast.parse("'{}'".format(func_name)), - "makesig": ast.parse("sig = nb.core.typing.signature({typer}, {para})".format(typer=typer, para=para)), + "makesig": ast.parse( + "sig = nb.core.typing.signature({typer}, {para})".format( + typer=typer, para=para + ) + ), } sig = None makesig = None @@ -168,15 +176,16 @@ def codegen(context, builder, sig, args): restype = context.get_argument_type(sig.return_type) fnty = ir.FunctionType(restype, argtypes) fn = nb.core.cgutils.insert_pure_function( - builder.module, fnty, name=func_name) - retval = context.call_external_function( - builder, fn, sig.args, args) + builder.module, fnty, name=func_name + ) + retval = context.call_external_function(builder, fn, sig.args, args) return retval @nb.extending.intrinsic def ARB_PARAM(): makesig return sig, codegen + return ARB_PARAM ret = ast.parse(dedent(inspect.getsource(ARB_PARAM_MAKER))) @@ -189,12 +198,15 @@ def ARB_PARAM(): def link_jit_code(args): glb = { - "jit": jit, "ffi": ffi, "nb": nb, "np": np, "math": math, "cmath": cmath, + "jit": jit, + "ffi": ffi, + "nb": nb, + "np": np, + "math": math, + "cmath": cmath, "link_libs": link_libs, - "FileReader_pop_event": link_function("FileReader_pop_event", 3, i64ret=True), "FileReader_init": link_function("FileReader_init", 5), - "VFILE_init": link_function("VFILE_init", 5), "POOL_update": link_function("POOL_update", 4), "VCHN_init": link_function("VCHN_init", 10), @@ -204,11 +216,24 @@ def link_jit_code(args): loc = {} # init - FileReader_pop_event, POOL_update, VCHN_next = ( - lambda *vargs: 0,)*3 - scalar_chn_next, READER, scalar_chn, scalar_fileid = (np.zeros(0),)*4 - uettp_initial, init_llvm, deinit, looping, beforeloop_code, num_rslot, global_initial, table_list, ptr_VCHN, ptr_fileid, ptr_chn, ptr_READER, ptr_chn_next, INTERRUPT = ( - 0,)*14 + FileReader_pop_event, POOL_update, VCHN_next = (lambda *vargs: 0,) * 3 + scalar_chn_next, READER, scalar_chn, scalar_fileid = (np.zeros(0),) * 4 + ( + uettp_initial, + init_llvm, + deinit, + looping, + beforeloop_code, + num_rslot, + global_initial, + table_list, + ptr_VCHN, + ptr_fileid, + ptr_chn, + ptr_READER, + ptr_chn_next, + INTERRUPT, + ) = (0,) * 14 @jit(nopython=True, nogil=True) # parallel=True, def mainloop(tables): @@ -232,12 +257,17 @@ def mainloop(tables): break if fileid < num_rslot: controller_rfile_time = FileReader_pop_event( - ptr_READER, nb.uint8(fileid), ptr_chn_next) + ptr_READER, nb.uint8(fileid), ptr_chn_next + ) if controller_rfile_time == 9223372036854775807: # early stop break else: - eta_ret += POOL_update(ptr_VCHN, nb.int64(controller_rfile_time), - nb.uint8(fileid), nb.uint8(scalar_chn_next[0])) + eta_ret += POOL_update( + ptr_VCHN, + nb.int64(controller_rfile_time), + nb.uint8(fileid), + nb.uint8(scalar_chn_next[0]), + ) deinit return eta_ret @@ -258,7 +288,7 @@ def cmp_dc(a, b): try: for k, v in a.items(): if isinstance(v, ast.AST): - if not(ast.dump(v) == ast.dump(b[k])): + if not (ast.dump(v) == ast.dump(b[k])): return False else: if v != b[k]: @@ -266,11 +296,13 @@ def cmp_dc(a, b): return True except Exception: return False - + + PARSE_TimeTagFileHeader = link_function("PARSE_TimeTagFileHeader", 2) + + @jit(nopython=True, nogil=True) def PARSE_TimeTagFileHeader_wrapper(PARSER, UniBuf): link_libs() - ret1 = PARSE_TimeTagFileHeader( - ffi.from_buffer(PARSER), ffi.from_buffer(UniBuf)) + ret1 = PARSE_TimeTagFileHeader(ffi.from_buffer(PARSER), ffi.from_buffer(UniBuf)) return ret1 diff --git a/gui/package.json b/gui/package.json index f31792b..6ec149e 100644 --- a/gui/package.json +++ b/gui/package.json @@ -48,7 +48,7 @@ "electron-updater": "^4.3.9" }, "devDependencies": { - "electron": "^18.3.7", + "electron": "^22.3.19", "electron-builder": "^23.6.0" }, "resolutions": {} diff --git a/requirements.txt b/requirements.txt index bdd2da2..e4bf844 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ cffi>=1.12.3 -numpy>=1.16.2 +numpy>=1.16.2,<2.0.0 astunparse>=1.6.2 numba>=0.50.1 llvmlite>=0.33.0 diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 22f5912..0000000 --- a/tox.ini +++ /dev/null @@ -1,14 +0,0 @@ -# tox.ini -[tox] -envlist = py39, py311 -[gh-actions] -python = - 3.11: py311 - 3.9: py39 -[testenv] -description = Unit tests -deps = - pytest - -rrequirements.txt -package_glob = 'dist/*.whl' -commands = pytest