diff --git a/.cirrus.yml b/.cirrus.yml index e34e374469..499ccc3d7e 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,6 +1,8 @@ -env: # Global defaults +### Global defaults + +env: PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y" - MAKEJOBS: "-j10" + MAKEJOBS: "-j4" TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache CCACHE_SIZE: "200M" CCACHE_DIR: "/tmp/ccache_dir" @@ -16,46 +18,56 @@ persistent_worker_template: &PERSISTENT_WORKER_TEMPLATE persistent_worker: {} # https://cirrus-ci.org/guide/persistent-workers/ # https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks -filter_template: &FILTER_TEMPLATE - skip: $CIRRUS_REPO_FULL_NAME == "bitcoin-core/gui" && $CIRRUS_PR == "" # No need to run on the read-only mirror, unless it is a PR. https://cirrus-ci.org/guide/writing-tasks/#conditional-task-execution - stateful: false # https://cirrus-ci.org/guide/writing-tasks/#stateful-tasks - base_template: &BASE_TEMPLATE - << : *FILTER_TEMPLATE + skip: $CIRRUS_REPO_FULL_NAME == "digibyte-core/gui" && $CIRRUS_PR == "" # No need to run on the read-only mirror, unless it is a PR. https://cirrus-ci.org/guide/writing-tasks/#conditional-task-execution merge_base_script: - # Unconditionally install git (used in fingerprint_script) and set the - # default git author name (used in verify-commits.py) + - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi - bash -c "$PACKAGE_MANAGER_INSTALL git" + - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH - git config --global user.email "ci@ci.ci" - git config --global user.name "ci" - - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi - - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH - git merge FETCH_HEAD # Merge base to detect silent merge conflicts + stateful: false # https://cirrus-ci.org/guide/writing-tasks/#stateful-tasks -main_template: &MAIN_TEMPLATE +global_task_template: &GLOBAL_TASK_TEMPLATE + << : *BASE_TEMPLATE timeout_in: 120m # https://cirrus-ci.org/faq/#instance-timed-out container: # https://cirrus-ci.org/faq/#are-there-any-limits # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel cpu: 2 - greedy: true memory: 8G # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers ccache_cache: folder: "/tmp/ccache_dir" depends_built_cache: folder: "depends/built" - fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-list -1 HEAD ./depends) ci_script: - ./ci/test_run_all.sh -global_task_template: &GLOBAL_TASK_TEMPLATE - << : *BASE_TEMPLATE - << : *MAIN_TEMPLATE +depends_sdk_cache_template: &DEPENDS_SDK_CACHE_TEMPLATE + depends_sdk_cache: + folder: "depends/sdk-sources" compute_credits_template: &CREDITS_TEMPLATE # https://cirrus-ci.org/pricing/#compute-credits # Only use credits for pull requests to the main repo - use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'bitcoin/bitcoin' && $CIRRUS_PR != "" + use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'digibyte-core/digibyte' && $CIRRUS_PR != "" + +#task: +# name: "Windows" +# windows_container: +# image: cirrusci/windowsservercore:2019 +# env: +# CIRRUS_SHELL: powershell +# PATH: 'C:\Python37;C:\Python37\Scripts;%PATH%' +# PYTHONUTF8: 1 +# QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.6/Qt5.9.8_x64_static_vs2019.zip' +# QT_DOWNLOAD_HASH: '9a8c6eb20967873785057fdcd329a657c7f922b0af08c5fde105cc597dd37e21' +# QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019' +# VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed' +# VCPKG_COMMIT_ID: 'ed0df8ecc4ed7e755ea03e18aaf285fd9b4b4a74' +# install_script: +# - choco install python --version=3.7.7 -y task: name: 'lint [bionic]' @@ -72,108 +84,13 @@ task: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV task: - name: "Win64 native [msvc]" - << : *FILTER_TEMPLATE - windows_container: - cpu: 4 - memory: 8G - image: cirrusci/windowsservercore:visualstudio2019 - timeout_in: 120m - env: - PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Current\Bin;%PATH%' - PYTHONUTF8: 1 - CI_VCPKG_TAG: '2021.05.12' - VCPKG_DOWNLOADS: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\downloads' - VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives' - QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.2/single/qt-everywhere-src-5.15.2.zip' - QT_LOCAL_PATH: 'C:\qt-everywhere-src-5.15.2.zip' - QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.2' - QTBASEDIR: 'C:\Qt_static' - x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"' - IgnoreWarnIntDirInTempDetected: 'true' - merge_script: - - git config --global user.email "ci@ci.ci" - - git config --global user.name "ci" - # Windows filesystem loses the executable bit, and all of the executable - # files are considered "modified" now. It will break the following `git merge` - # command. The next two commands make git ignore this issue. - - git config core.filemode false - - git reset --hard - - PowerShell -NoLogo -Command if ($env:CIRRUS_PR -ne $null) { git fetch $env:CIRRUS_REPO_CLONE_URL $env:CIRRUS_BASE_BRANCH; git merge FETCH_HEAD; } - msvc_qt_built_cache: - folder: "%QTBASEDIR%" - reupload_on_changes: false - fingerprint_script: - - echo %QT_DOWNLOAD_URL% - - msbuild -version - populate_script: - - curl -L -o C:\jom.zip http://download.qt.io/official_releases/jom/jom.zip - - mkdir C:\jom - - tar -xf C:\jom.zip -C C:\jom - - curl -L -o %QT_LOCAL_PATH% %QT_DOWNLOAD_URL% - - tar -xf %QT_LOCAL_PATH% -C C:\ - - '%x64_NATIVE_TOOLS%' - - cd %QT_SOURCE_DIR% - - mkdir build - - cd build - - ..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml -prefix %QTBASEDIR% - - jom - - jom install - vcpkg_tools_cache: - folder: '%VCPKG_DOWNLOADS%\tools' - reupload_on_changes: false - fingerprint_script: - - echo %CI_VCPKG_TAG% - - msbuild -version - vcpkg_binary_cache: - folder: '%VCPKG_DEFAULT_BINARY_CACHE%' - reupload_on_changes: true - fingerprint_script: - - echo %CI_VCPKG_TAG% - - msbuild -version - populate_script: - - mkdir %VCPKG_DEFAULT_BINARY_CACHE% - install_python_script: - - choco install --yes --no-progress python3 --version=3.9.6 - - pip install zmq - - python -VV - install_vcpkg_script: - - cd .. - - git clone --quiet https://github.com/microsoft/vcpkg.git - - cd vcpkg - - git -c advice.detachedHead=false checkout %CI_VCPKG_TAG% - - .\bootstrap-vcpkg -disableMetrics - - echo set(VCPKG_BUILD_TYPE release) >> triplets\x64-windows-static.cmake - - .\vcpkg integrate install - - .\vcpkg version - build_script: - - cd %CIRRUS_WORKING_DIR% - - python build_msvc\msvc-autogen.py - - msbuild build_msvc\bitcoin.sln -property:Configuration=Release -maxCpuCount -verbosity:minimal -noLogo - unit_tests_script: - - src\test_bitcoin.exe -l test_suite - - src\bench_bitcoin.exe > NUL - - python test\util\test_runner.py - - python test\util\rpcauth-test.py - functional_tests_script: - # Increase the dynamic port range to the maximum allowed value to mitigate "OSError: [WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted". - # See: https://docs.microsoft.com/en-us/biztalk/technical-guides/settings-that-can-be-modified-to-improve-network-performance - - netsh int ipv4 set dynamicport tcp start=1025 num=64511 - - netsh int ipv6 set dynamicport tcp start=1025 num=64511 - # Exclude feature_dbcrash for now due to timeout - - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=4000 --jobs=4 --timeout-factor=8 --extended --exclude feature_dbcrash - -task: - name: 'ARM [unit tests, no functional tests] [bullseye]' + name: 'ARM [unit tests, no functional tests] [buster]' << : *GLOBAL_TASK_TEMPLATE - arm_container: - image: debian:bullseye - cpu: 2 - memory: 8G + container: + image: debian:buster env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_arm.sh" - QEMU_USER_CMD: "" # Disable qemu and run the test natively task: name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [focal]' @@ -205,39 +122,37 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh" task: - name: '[TSan, depends, gui] [jammy]' + name: '[depends, sanitizers: thread (TSan), no gui] [hirsute]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:jammy + image: ubuntu:hirsute cpu: 6 # Increase CPU and Memory to avoid timeout memory: 24G env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + MAKEJOBS: "-j8" FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh" task: - name: '[MSan, depends] [focal]' + name: '[depends, sanitizers: memory (MSan)] [focal]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_native_msan.sh" - MAKEJOBS: "-j4" # Avoid excessive memory use due to MSan task: - name: '[ASan + LSan + UBSan + integer, no depends] [jammy]' + name: '[no depends, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer] [hirsute]' << : *GLOBAL_TASK_TEMPLATE container: - image: ubuntu:jammy + image: ubuntu:hirsute env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV FILE_ENV: "./ci/test/00_setup_env_native_asan.sh" - MAKEJOBS: "-j4" # Avoid excessive memory use task: - name: '[fuzzer,address,undefined,integer, no depends] [focal]' - only_if: $CIRRUS_BRANCH == $CIRRUS_DEFAULT_BRANCH || $CIRRUS_BASE_BRANCH == $CIRRUS_DEFAULT_BRANCH + name: '[no depends, sanitizers: fuzzer,address,undefined,integer] [focal]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal @@ -245,10 +160,11 @@ task: memory: 16G env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV + MAKEJOBS: "-j8" FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh" task: - name: '[multiprocess, i686, DEBUG] [focal]' + name: '[multiprocess, DEBUG] [focal]' << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal @@ -256,7 +172,8 @@ task: memory: 16G # The default memory is sometimes just a bit too small, so double everything env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh" + MAKEJOBS: "-j8" + FILE_ENV: "./ci/test/00_setup_env_native_multiprocess.sh" task: name: '[no wallet] [bionic]' @@ -297,16 +214,12 @@ task: task: name: 'ARM64 Android APK [focal]' - << : *BASE_TEMPLATE - android_sdk_cache: - folder: "depends/SDKs/android" - fingerprint_key: "ANDROID_API_LEVEL=28 ANDROID_BUILD_TOOLS_VERSION=28.0.3 ANDROID_NDK_VERSION=23.1.7779620" + << : *DEPENDS_SDK_CACHE_TEMPLATE depends_sources_cache: folder: "depends/sources" - fingerprint_script: git rev-list -1 HEAD ./depends - << : *MAIN_TEMPLATE + << : *GLOBAL_TASK_TEMPLATE container: image: ubuntu:focal env: << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV - FILE_ENV: "./ci/test/00_setup_env_android.sh" \ No newline at end of file + FILE_ENV: "./ci/test/00_setup_env_android.sh" diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index d7de3c5f89..c8d0cd8d89 100755 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -15,4 +15,4 @@ export XCODE_BUILD_ID=12B45b export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" -export BITCOIN_CONFIG="--with-gui --enable-reduce-exports" \ No newline at end of file +export DIGIBYTE_CONFIG="--with-gui --enable-reduce-exports" \ No newline at end of file diff --git a/contrib/devtools/security-check.py b/contrib/devtools/security-check.py index d8a6ce1aa3..1fd20ff5d8 100755 --- a/contrib/devtools/security-check.py +++ b/contrib/devtools/security-check.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2021 The Bitcoin Core developers +# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2020 The DigiByte Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' @@ -8,258 +9,248 @@ Otherwise the exit status will be 1 and it will log which executables failed which checks. ''' import sys -from typing import List +from typing import List, Optional -import lief #type:ignore +import lief +import pixie -# temporary constant, to be replaced with lief.ELF.ARCH.RISCV -# https://github.com/lief-project/LIEF/pull/562 -LIEF_ELF_ARCH_RISCV = lief.ELF.ARCH(243) +def check_ELF_PIE(executable) -> bool: + ''' + Check for position independent executable (PIE), allowing for address space randomization. + ''' + elf = pixie.load(executable) + return elf.hdr.e_type == pixie.ET_DYN + +def check_ELF_NX(executable) -> bool: + ''' + Check that no sections are writable and executable (including the stack) + ''' + elf = pixie.load(executable) + have_wx = False + have_gnu_stack = False + for ph in elf.program_headers: + if ph.p_type == pixie.PT_GNU_STACK: + have_gnu_stack = True + if (ph.p_flags & pixie.PF_W) != 0 and (ph.p_flags & pixie.PF_X) != 0: # section is both writable and executable + have_wx = True + return have_gnu_stack and not have_wx -def check_ELF_RELRO(binary) -> bool: +def check_ELF_RELRO(executable) -> bool: ''' Check for read-only relocations. GNU_RELRO program header must exist Dynamic section must have BIND_NOW flag ''' + elf = pixie.load(executable) have_gnu_relro = False - for segment in binary.segments: + for ph in elf.program_headers: # Note: not checking p_flags == PF_R: here as linkers set the permission differently # This does not affect security: the permission flags of the GNU_RELRO program # header are ignored, the PT_LOAD header determines the effective permissions. # However, the dynamic linker need to write to this area so these are RW. # Glibc itself takes care of mprotecting this area R after relocations are finished. # See also https://marc.info/?l=binutils&m=1498883354122353 - if segment.type == lief.ELF.SEGMENT_TYPES.GNU_RELRO: + if ph.p_type == pixie.PT_GNU_RELRO: have_gnu_relro = True have_bindnow = False - try: - flags = binary.get(lief.ELF.DYNAMIC_TAGS.FLAGS) - if flags.value & lief.ELF.DYNAMIC_FLAGS.BIND_NOW: + for flags in elf.query_dyn_tags(pixie.DT_FLAGS): + assert isinstance(flags, int) + if flags & pixie.DF_BIND_NOW: have_bindnow = True - except: - have_bindnow = False return have_gnu_relro and have_bindnow -def check_ELF_Canary(binary) -> bool: +def check_ELF_Canary(executable) -> bool: ''' Check for use of stack canary ''' - return binary.has_symbol('__stack_chk_fail') + elf = pixie.load(executable) + ok = False + for symbol in elf.dyn_symbols: + if symbol.name == b'__stack_chk_fail': + ok = True + return ok -def check_ELF_separate_code(binary): +def check_ELF_separate_code(executable): ''' Check that sections are appropriately separated in virtual memory, based on their permissions. This checks for missing -Wl,-z,separate-code and potentially other problems. ''' - R = lief.ELF.SEGMENT_FLAGS.R - W = lief.ELF.SEGMENT_FLAGS.W - E = lief.ELF.SEGMENT_FLAGS.X + elf = pixie.load(executable) + R = pixie.PF_R + W = pixie.PF_W + E = pixie.PF_X EXPECTED_FLAGS = { # Read + execute - '.init': R | E, - '.plt': R | E, - '.plt.got': R | E, - '.plt.sec': R | E, - '.text': R | E, - '.fini': R | E, + b'.init': R | E, + b'.plt': R | E, + b'.plt.got': R | E, + b'.plt.sec': R | E, + b'.text': R | E, + b'.fini': R | E, # Read-only data - '.interp': R, - '.note.gnu.property': R, - '.note.gnu.build-id': R, - '.note.ABI-tag': R, - '.gnu.hash': R, - '.dynsym': R, - '.dynstr': R, - '.gnu.version': R, - '.gnu.version_r': R, - '.rela.dyn': R, - '.rela.plt': R, - '.rodata': R, - '.eh_frame_hdr': R, - '.eh_frame': R, - '.qtmetadata': R, - '.gcc_except_table': R, - '.stapsdt.base': R, + b'.interp': R, + b'.note.gnu.property': R, + b'.note.gnu.build-id': R, + b'.note.ABI-tag': R, + b'.gnu.hash': R, + b'.dynsym': R, + b'.dynstr': R, + b'.gnu.version': R, + b'.gnu.version_r': R, + b'.rela.dyn': R, + b'.rela.plt': R, + b'.rodata': R, + b'.eh_frame_hdr': R, + b'.eh_frame': R, + b'.qtmetadata': R, + b'.gcc_except_table': R, + b'.stapsdt.base': R, # Writable data - '.init_array': R | W, - '.fini_array': R | W, - '.dynamic': R | W, - '.got': R | W, - '.data': R | W, - '.bss': R | W, + b'.init_array': R | W, + b'.fini_array': R | W, + b'.dynamic': R | W, + b'.got': R | W, + b'.data': R | W, + b'.bss': R | W, } - if binary.header.machine_type == lief.ELF.ARCH.PPC64: + if elf.hdr.e_machine == pixie.EM_PPC64: # .plt is RW on ppc64 even with separate-code - EXPECTED_FLAGS['.plt'] = R | W + EXPECTED_FLAGS[b'.plt'] = R | W # For all LOAD program headers get mapping to the list of sections, # and for each section, remember the flags of the associated program header. flags_per_section = {} - for segment in binary.segments: - if segment.type == lief.ELF.SEGMENT_TYPES.LOAD: - for section in segment.sections: + for ph in elf.program_headers: + if ph.p_type == pixie.PT_LOAD: + for section in ph.sections: assert(section.name not in flags_per_section) - flags_per_section[section.name] = segment.flags + flags_per_section[section.name] = ph.p_flags # Spot-check ELF LOAD program header flags per section # If these sections exist, check them against the expected R/W/E flags for (section, flags) in flags_per_section.items(): if section in EXPECTED_FLAGS: - if int(EXPECTED_FLAGS[section]) != int(flags): + if EXPECTED_FLAGS[section] != flags: return False return True -def check_ELF_control_flow(binary) -> bool: - ''' - Check for control flow instrumentation - ''' - main = binary.get_function_address('main') - content = binary.get_content_from_virtual_address(main, 4, lief.Binary.VA_TYPES.AUTO) - - if content == [243, 15, 30, 250]: # endbr64 - return True - return False - -def check_PE_DYNAMIC_BASE(binary) -> bool: +def check_PE_DYNAMIC_BASE(executable) -> bool: '''PIE: DllCharacteristics bit 0x40 signifies dynamicbase (ASLR)''' + binary = lief.parse(executable) return lief.PE.DLL_CHARACTERISTICS.DYNAMIC_BASE in binary.optional_header.dll_characteristics_lists # Must support high-entropy 64-bit address space layout randomization # in addition to DYNAMIC_BASE to have secure ASLR. -def check_PE_HIGH_ENTROPY_VA(binary) -> bool: +def check_PE_HIGH_ENTROPY_VA(executable) -> bool: '''PIE: DllCharacteristics bit 0x20 signifies high-entropy ASLR''' + binary = lief.parse(executable) return lief.PE.DLL_CHARACTERISTICS.HIGH_ENTROPY_VA in binary.optional_header.dll_characteristics_lists -def check_PE_RELOC_SECTION(binary) -> bool: +def check_PE_RELOC_SECTION(executable) -> bool: '''Check for a reloc section. This is required for functional ASLR.''' + binary = lief.parse(executable) return binary.has_relocations -def check_PE_control_flow(binary) -> bool: - ''' - Check for control flow instrumentation - ''' - main = binary.get_symbol('main').value - - section_addr = binary.section_from_rva(main).virtual_address - virtual_address = binary.optional_header.imagebase + section_addr + main - - content = binary.get_content_from_virtual_address(virtual_address, 4, lief.Binary.VA_TYPES.VA) - - if content == [243, 15, 30, 250]: # endbr64 - return True - return False - -def check_MACHO_NOUNDEFS(binary) -> bool: +def check_MACHO_NOUNDEFS(executable) -> bool: ''' Check for no undefined references. ''' + binary = lief.parse(executable) return binary.header.has(lief.MachO.HEADER_FLAGS.NOUNDEFS) -def check_MACHO_LAZY_BINDINGS(binary) -> bool: +def check_MACHO_LAZY_BINDINGS(executable) -> bool: ''' Check for no lazy bindings. We don't use or check for MH_BINDATLOAD. See #18295. ''' + binary = lief.parse(executable) return binary.dyld_info.lazy_bind == (0,0) -def check_MACHO_Canary(binary) -> bool: +def check_MACHO_Canary(executable) -> bool: ''' Check for use of stack canary ''' + binary = lief.parse(executable) return binary.has_symbol('___stack_chk_fail') -def check_PIE(binary) -> bool: +def check_PIE(executable) -> bool: ''' Check for position independent executable (PIE), allowing for address space randomization. ''' + binary = lief.parse(executable) return binary.is_pie -def check_NX(binary) -> bool: +def check_NX(executable) -> bool: ''' Check for no stack execution ''' + binary = lief.parse(executable) return binary.has_nx -def check_MACHO_control_flow(binary) -> bool: +def check_control_flow(executable) -> bool: ''' Check for control flow instrumentation ''' + binary = lief.parse(executable) + content = binary.get_content_from_virtual_address(binary.entrypoint, 4, lief.Binary.VA_TYPES.AUTO) if content == [243, 15, 30, 250]: # endbr64 return True return False -BASE_ELF = [ - ('PIE', check_PIE), - ('NX', check_NX), + +CHECKS = { +'ELF': [ + ('PIE', check_ELF_PIE), + ('NX', check_ELF_NX), ('RELRO', check_ELF_RELRO), ('Canary', check_ELF_Canary), ('separate_code', check_ELF_separate_code), -] - -BASE_PE = [ +], +'PE': [ ('PIE', check_PIE), ('DYNAMIC_BASE', check_PE_DYNAMIC_BASE), ('HIGH_ENTROPY_VA', check_PE_HIGH_ENTROPY_VA), ('NX', check_NX), - ('RELOC_SECTION', check_PE_RELOC_SECTION), - ('CONTROL_FLOW', check_PE_control_flow), -] - -BASE_MACHO = [ + ('RELOC_SECTION', check_PE_RELOC_SECTION) +], +'MACHO': [ + ('PIE', check_PIE), ('NOUNDEFS', check_MACHO_NOUNDEFS), + ('NX', check_NX), ('LAZY_BINDINGS', check_MACHO_LAZY_BINDINGS), ('Canary', check_MACHO_Canary), + ('CONTROL_FLOW', check_control_flow), ] - -CHECKS = { - lief.EXE_FORMATS.ELF: { - lief.ARCHITECTURES.X86: BASE_ELF + [('CONTROL_FLOW', check_ELF_control_flow)], - lief.ARCHITECTURES.ARM: BASE_ELF, - lief.ARCHITECTURES.ARM64: BASE_ELF, - lief.ARCHITECTURES.PPC: BASE_ELF, - LIEF_ELF_ARCH_RISCV: BASE_ELF, - }, - lief.EXE_FORMATS.PE: { - lief.ARCHITECTURES.X86: BASE_PE, - }, - lief.EXE_FORMATS.MACHO: { - lief.ARCHITECTURES.X86: BASE_MACHO + [('PIE', check_PIE), - ('NX', check_NX), - ('CONTROL_FLOW', check_MACHO_control_flow)], - lief.ARCHITECTURES.ARM64: BASE_MACHO, - } } +def identify_executable(executable) -> Optional[str]: + with open(filename, 'rb') as f: + magic = f.read(4) + if magic.startswith(b'MZ'): + return 'PE' + elif magic.startswith(b'\x7fELF'): + return 'ELF' + elif magic.startswith(b'\xcf\xfa'): + return 'MACHO' + return None + if __name__ == '__main__': retval: int = 0 for filename in sys.argv[1:]: try: - binary = lief.parse(filename) - etype = binary.format - arch = binary.abstract.header.architecture - binary.concrete - - if etype == lief.EXE_FORMATS.UNKNOWN: - print(f'{filename}: unknown executable format') + etype = identify_executable(filename) + if etype is None: + print(f'{filename}: unknown format') retval = 1 continue - if arch == lief.ARCHITECTURES.NONE: - if binary.header.machine_type == LIEF_ELF_ARCH_RISCV: - arch = LIEF_ELF_ARCH_RISCV - else: - print(f'{filename}: unknown architecture') - retval = 1 - continue - failed: List[str] = [] - for (name, func) in CHECKS[etype][arch]: - if not func(binary): + for (name, func) in CHECKS[etype]: + if not func(filename): failed.append(name) if failed: print(f'{filename}: failed {" ".join(failed)}') @@ -268,3 +259,4 @@ def check_MACHO_control_flow(binary) -> bool: print(f'{filename}: cannot open') retval = 1 sys.exit(retval) + diff --git a/contrib/devtools/symbol-check.py b/contrib/devtools/symbol-check.py index 461132ae63..8193a3bb7b 100755 --- a/contrib/devtools/symbol-check.py +++ b/contrib/devtools/symbol-check.py @@ -10,82 +10,61 @@ find ../path/to/binaries -type f -executable | xargs python3 contrib/devtools/symbol-check.py ''' +import subprocess import sys -from typing import List, Dict +from typing import List, Optional -import lief #type:ignore +import lief +import pixie -# temporary constant, to be replaced with lief.ELF.ARCH.RISCV -# https://github.com/lief-project/LIEF/pull/562 -LIEF_ELF_ARCH_RISCV = lief.ELF.ARCH(243) +from utils import determine_wellknown_cmd -# Debian 9 (Stretch) EOL: 2022. https://wiki.debian.org/DebianReleases#Production_Releases +# Debian 8 (Jessie) EOL: 2020. https://wiki.debian.org/DebianReleases#Production_Releases # -# - g++ version 6.3.0 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=g%2B%2B) -# - libc version 2.24 (https://packages.debian.org/search?suite=stretch&arch=any&searchon=names&keywords=libc6) +# - g++ version 4.9.2 (https://packages.debian.org/search?suite=jessie&arch=any&searchon=names&keywords=g%2B%2B) +# - libc version 2.19 (https://packages.debian.org/search?suite=jessie&arch=any&searchon=names&keywords=libc6) # -# Ubuntu 16.04 (Xenial) EOL: 2026. https://wiki.ubuntu.com/Releases +# Ubuntu 16.04 (Xenial) EOL: 2024. https://wiki.ubuntu.com/Releases # -# - g++ version 5.3.1 -# - libc version 2.23 +# - g++ version 5.3.1 (https://packages.ubuntu.com/search?keywords=g%2B%2B&searchon=names&suite=xenial§ion=all) +# - libc version 2.23.0 (https://packages.ubuntu.com/search?keywords=libc6&searchon=names&suite=xenial§ion=all) # -# CentOS Stream 8 EOL: 2024. https://wiki.centos.org/About/Product +# CentOS 7 EOL: 2024. https://wiki.centos.org/FAQ/General # -# - g++ version 8.5.0 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) -# - libc version 2.28 (http://mirror.centos.org/centos/8-stream/AppStream/x86_64/os/Packages/) +# - g++ version 4.8.5 (http://mirror.centos.org/centos/7/os/x86_64/Packages/) +# - libc version 2.17 (http://mirror.centos.org/centos/7/os/x86_64/Packages/) +# +# Taking the minimum of these as our target. +# +# According to GNU ABI document (https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html) this corresponds to: +# GCC 4.8.5: GCC_4.8.0 +# (glibc) GLIBC_2_17 # -# See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html for more info. - MAX_VERSIONS = { 'GCC': (4,8,0), 'GLIBC': { - lief.ELF.ARCH.i386: (2,18), - lief.ELF.ARCH.x86_64: (2,18), - lief.ELF.ARCH.ARM: (2,18), - lief.ELF.ARCH.AARCH64:(2,18), - lief.ELF.ARCH.PPC64: (2,18), - LIEF_ELF_ARCH_RISCV: (2,27), + pixie.EM_386: (2,18), + pixie.EM_X86_64: (2,18), + pixie.EM_ARM: (2,18), + pixie.EM_AARCH64:(2,18), + pixie.EM_PPC64: (2,18), + pixie.EM_RISCV: (2,27), }, 'LIBATOMIC': (1,0), -'V': (0,5,0), # xkb (bitcoin-qt only) +'V': (0,5,0), # xkb (digibyte-qt only) } # See here for a description of _IO_stdin_used: # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=634261#109 # Ignore symbols that are exported as part of every executable IGNORE_EXPORTS = { -'_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', -'__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr', +'_edata', '_end', '__end__', '_init', '__bss_start', '__bss_start__', '_bss_end__', '__bss_end__', '_fini', '_IO_stdin_used', 'stdin', 'stdout', 'stderr', 'environ', '_environ', '__environ', } -# Expected linker-loader names can be found here: -# https://sourceware.org/glibc/wiki/ABIList?action=recall&rev=16 -ELF_INTERPRETER_NAMES: Dict[lief.ELF.ARCH, Dict[lief.ENDIANNESS, str]] = { - lief.ELF.ARCH.i386: { - lief.ENDIANNESS.LITTLE: "/lib/ld-linux.so.2", - }, - lief.ELF.ARCH.x86_64: { - lief.ENDIANNESS.LITTLE: "/lib64/ld-linux-x86-64.so.2", - }, - lief.ELF.ARCH.ARM: { - lief.ENDIANNESS.LITTLE: "/lib/ld-linux-armhf.so.3", - }, - lief.ELF.ARCH.AARCH64: { - lief.ENDIANNESS.LITTLE: "/lib/ld-linux-aarch64.so.1", - }, - lief.ELF.ARCH.PPC64: { - lief.ENDIANNESS.BIG: "/lib64/ld64.so.1", - lief.ENDIANNESS.LITTLE: "/lib64/ld64.so.2", - }, - LIEF_ELF_ARCH_RISCV: { - lief.ENDIANNESS.LITTLE: "/lib/ld-linux-riscv64-lp64d.so.1", - }, -} - # Allowed NEEDED libraries ELF_ALLOWED_LIBRARIES = { -# bitcoind and bitcoin-qt +# digibyted and digibyte-qt 'libgcc_s.so.1', # GCC base support 'libc.so.6', # C library 'libpthread.so.0', # threading @@ -99,7 +78,7 @@ 'ld64.so.1', # POWER64 ABIv1 dynamic linker 'ld64.so.2', # POWER64 ABIv2 dynamic linker 'ld-linux-riscv64-lp64d.so.1', # 64-bit RISC-V dynamic linker -# bitcoin-qt only +# digibyte-qt only 'libxcb.so.1', # part of X11 'libxkbcommon.so.0', # keyboard keymapping 'libxkbcommon-x11.so.0', # keyboard keymapping @@ -121,10 +100,10 @@ } MACHO_ALLOWED_LIBRARIES = { -# bitcoind and bitcoin-qt +# digibyted and digibyte-qt 'libc++.1.dylib', # C++ Standard Library 'libSystem.B.dylib', # libc, libm, libpthread, libinfo -# bitcoin-qt only +# digibyte-qt only 'AppKit', # user interface 'ApplicationServices', # common application tasks. 'Carbon', # deprecated c back-compat API @@ -152,7 +131,7 @@ 'SHELL32.dll', # shell API 'USER32.dll', # user interface 'WS2_32.dll', # sockets -# bitcoin-qt only +# digibyte-qt only 'dwmapi.dll', # desktop window manager 'GDI32.dll', # graphics device interface 'IMM32.dll', # input method editor @@ -167,8 +146,31 @@ 'WTSAPI32.dll', } +class CPPFilt(object): + ''' + Demangle C++ symbol names. + + Use a pipe to the 'c++filt' command. + ''' + def __init__(self): + self.proc = subprocess.Popen(determine_wellknown_cmd('CPPFILT', 'c++filt'), stdin=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True) + + def __call__(self, mangled): + self.proc.stdin.write(mangled + '\n') + self.proc.stdin.flush() + return self.proc.stdout.readline().rstrip() + + def close(self): + self.proc.stdin.close() + self.proc.stdout.close() + self.proc.wait() + def check_version(max_versions, version, arch) -> bool: - (lib, _, ver) = version.rpartition('_') + if '_' in version: + (lib, _, ver) = version.rpartition('_') + else: + lib = version + ver = '0' ver = tuple([int(x) for x in ver.split('.')]) if not lib in max_versions: return False @@ -177,45 +179,48 @@ def check_version(max_versions, version, arch) -> bool: else: return ver <= max_versions[lib][arch] -def check_imported_symbols(binary) -> bool: +def check_imported_symbols(filename) -> bool: + elf = pixie.load(filename) + cppfilt = CPPFilt() ok: bool = True - for symbol in binary.imported_symbols: - if not symbol.imported: + for symbol in elf.dyn_symbols: + if not symbol.is_import: continue - - version = symbol.symbol_version if symbol.has_version else None - - if version: - aux_version = version.symbol_version_auxiliary.name if version.has_auxiliary_version else None - if aux_version and not check_version(MAX_VERSIONS, aux_version, binary.header.machine_type): - print(f'{filename}: symbol {symbol.name} from unsupported version {version}') - ok = False + sym = symbol.name.decode() + version = symbol.version.decode() if symbol.version is not None else None + if version and not check_version(MAX_VERSIONS, version, elf.hdr.e_machine): + print('{}: symbol {} from unsupported version {}'.format(filename, cppfilt(sym), version)) + ok = False return ok -def check_exported_symbols(binary) -> bool: +def check_exported_symbols(filename) -> bool: + elf = pixie.load(filename) + cppfilt = CPPFilt() ok: bool = True - - for symbol in binary.dynamic_symbols: - if not symbol.exported: + for symbol in elf.dyn_symbols: + if not symbol.is_export: continue - name = symbol.name - if binary.header.machine_type == LIEF_ELF_ARCH_RISCV or name in IGNORE_EXPORTS: + sym = symbol.name.decode() + if elf.hdr.e_machine == pixie.EM_RISCV or sym in IGNORE_EXPORTS: continue - print(f'{binary.name}: export of symbol {name} not allowed!') + print('{}: export of symbol {} not allowed'.format(filename, cppfilt(sym))) ok = False return ok -def check_ELF_libraries(binary) -> bool: +def check_ELF_libraries(filename) -> bool: ok: bool = True - for library in binary.libraries: - if library not in ELF_ALLOWED_LIBRARIES: - print(f'{filename}: {library} is not in ALLOWED_LIBRARIES!') + elf = pixie.load(filename) + for library_name in elf.query_dyn_tags(pixie.DT_NEEDED): + assert(isinstance(library_name, bytes)) + if library_name.decode() not in ELF_ALLOWED_LIBRARIES: + print('{}: NEEDED library {} is not allowed'.format(filename, library_name.decode())) ok = False return ok -def check_MACHO_libraries(binary) -> bool: +def check_MACHO_libraries(filename) -> bool: ok: bool = True + binary = lief.parse(filename) for dylib in binary.libraries: split = dylib.name.split('/') if split[-1] not in MACHO_ALLOWED_LIBRARIES: @@ -223,68 +228,76 @@ def check_MACHO_libraries(binary) -> bool: ok = False return ok -def check_MACHO_min_os(binary) -> bool: +def check_MACHO_min_os(filename) -> bool: + binary = lief.parse(filename) if binary.build_version.minos == [10,15,0]: return True return False -def check_MACHO_sdk(binary) -> bool: +def check_MACHO_sdk(filename) -> bool: + binary = lief.parse(filename) if binary.build_version.sdk == [11, 0, 0]: return True return False -def check_PE_libraries(binary) -> bool: +def check_PE_libraries(filename) -> bool: ok: bool = True + binary = lief.parse(filename) for dylib in binary.libraries: if dylib not in PE_ALLOWED_LIBRARIES: print(f'{dylib} is not in ALLOWED_LIBRARIES!') ok = False return ok -def check_PE_subsystem_version(binary) -> bool: +def check_PE_subsystem_version(filename) -> bool: + binary = lief.parse(filename) major: int = binary.optional_header.major_subsystem_version minor: int = binary.optional_header.minor_subsystem_version if major == 6 and minor == 1: return True return False -def check_ELF_interpreter(binary) -> bool: - expected_interpreter = ELF_INTERPRETER_NAMES[binary.header.machine_type][binary.abstract.header.endianness] - - return binary.concrete.interpreter == expected_interpreter - CHECKS = { -lief.EXE_FORMATS.ELF: [ +'ELF': [ ('IMPORTED_SYMBOLS', check_imported_symbols), ('EXPORTED_SYMBOLS', check_exported_symbols), - ('LIBRARY_DEPENDENCIES', check_ELF_libraries), - ('INTERPRETER_NAME', check_ELF_interpreter), + ('LIBRARY_DEPENDENCIES', check_ELF_libraries) ], -lief.EXE_FORMATS.MACHO: [ +'MACHO': [ ('DYNAMIC_LIBRARIES', check_MACHO_libraries), ('MIN_OS', check_MACHO_min_os), ('SDK', check_MACHO_sdk), ], -lief.EXE_FORMATS.PE: [ +'PE' : [ ('DYNAMIC_LIBRARIES', check_PE_libraries), ('SUBSYSTEM_VERSION', check_PE_subsystem_version), ] } +def identify_executable(executable) -> Optional[str]: + with open(filename, 'rb') as f: + magic = f.read(4) + if magic.startswith(b'MZ'): + return 'PE' + elif magic.startswith(b'\x7fELF'): + return 'ELF' + elif magic.startswith(b'\xcf\xfa'): + return 'MACHO' + return None + if __name__ == '__main__': retval: int = 0 for filename in sys.argv[1:]: try: - binary = lief.parse(filename) - etype = binary.format - if etype == lief.EXE_FORMATS.UNKNOWN: - print(f'{filename}: unknown executable format') + etype = identify_executable(filename) + if etype is None: + print(f'{filename}: unknown format') retval = 1 continue failed: List[str] = [] for (name, func) in CHECKS[etype]: - if not func(binary): + if not func(filename): failed.append(name) if failed: print(f'{filename}: failed {" ".join(failed)}') diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index d3d225f3ab..6dee90819e 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -1,14 +1,13 @@ #!/usr/bin/env python3 -# Copyright (c) 2015-2021 The Bitcoin Core developers +# Copyright (c) 2015-2018 The Bitcoin Core developers +# Copyright (c) 2015-2020 The DigiByte Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Test script for security-check.py ''' -import lief #type:ignore import os import subprocess -from typing import List import unittest from utils import determine_wellknown_cmd @@ -29,62 +28,29 @@ def clean_files(source, executable): os.remove(executable) def call_security_check(cc, source, executable, options): - # This should behave the same as AC_TRY_LINK, so arrange well-known flags - # in the same order as autoconf would. - # - # See the definitions for ac_link in autoconf's lib/autoconf/c.m4 file for - # reference. - env_flags: List[str] = [] - for var in ['CFLAGS', 'CPPFLAGS', 'LDFLAGS']: - env_flags += filter(None, os.environ.get(var, '').split(' ')) - - subprocess.run([*cc,source,'-o',executable] + env_flags + options, check=True) + subprocess.run([*cc,source,'-o',executable] + options, check=True) p = subprocess.run(['./contrib/devtools/security-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True) return (p.returncode, p.stdout.rstrip()) -def get_arch(cc, source, executable): - subprocess.run([*cc, source, '-o', executable], check=True) - binary = lief.parse(executable) - arch = binary.abstract.header.architecture - os.remove(executable) - return arch - class TestSecurityChecks(unittest.TestCase): def test_ELF(self): source = 'test1.c' executable = 'test1' cc = determine_wellknown_cmd('CC', 'gcc') write_testcode(source) - arch = get_arch(cc, source, executable) - if arch == lief.ARCHITECTURES.X86: - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE NX RELRO Canary CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE RELRO Canary CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE RELRO CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), - (1, executable+': failed RELRO CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), - (1, executable+': failed separate_code CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), - (1, executable+': failed CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code', '-fcf-protection=full']), - (0, '')) - else: - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE NX RELRO Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE RELRO Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), - (1, executable+': failed PIE RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), - (1, executable+': failed RELRO')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), - (1, executable+': failed separate_code')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), - (0, '')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-zexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE NX RELRO Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fno-stack-protector','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO Canary')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-no-pie','-fno-PIE', '-Wl,-z,separate-code']), + (1, executable+': failed PIE RELRO')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-znorelro','-pie','-fPIE', '-Wl,-z,separate-code']), + (1, executable+': failed RELRO')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,noseparate-code']), + (1, executable+': failed separate_code')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-znoexecstack','-fstack-protector-all','-Wl,-zrelro','-Wl,-z,now','-pie','-fPIE', '-Wl,-z,separate-code']), + (0, '')) clean_files(source, executable) @@ -94,19 +60,17 @@ def test_PE(self): cc = determine_wellknown_cmd('CC', 'x86_64-w64-mingw32-gcc') write_testcode(source) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--disable-nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-no-pie','-fno-PIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--disable-dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE']), - (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA CONTROL_FLOW')) # -pie -fPIE does nothing unless --dynamicbase is also supplied - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--disable-high-entropy-va','-pie','-fPIE']), - (1, executable+': failed HIGH_ENTROPY_VA CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--no-nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA NX RELOC_SECTION')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--disable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA RELOC_SECTION')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-no-pie','-fno-PIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--no-dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']), + (1, executable+': failed PIE DYNAMIC_BASE HIGH_ENTROPY_VA')) # -pie -fPIE does nothing unless --dynamicbase is also supplied + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--no-high-entropy-va','-pie','-fPIE']), + (1, executable+': failed HIGH_ENTROPY_VA')) self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE']), - (1, executable+': failed CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,--nxcompat','-Wl,--enable-reloc-section','-Wl,--dynamicbase','-Wl,--high-entropy-va','-pie','-fPIE', '-fcf-protection=full']), (0, '')) clean_files(source, executable) @@ -116,34 +80,21 @@ def test_MACHO(self): executable = 'test1' cc = determine_wellknown_cmd('CC', 'clang') write_testcode(source) - arch = get_arch(cc, source, executable) - - if arch == lief.ARCHITECTURES.X86: - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']), - (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary PIE NX CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all']), - (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE NX CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all']), - (1, executable+': failed NOUNDEFS LAZY_BINDINGS PIE CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all']), - (1, executable+': failed LAZY_BINDINGS PIE CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all']), - (1, executable+': failed PIE CONTROL_FLOW')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), - (1, executable+': failed PIE')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), - (0, '')) - else: - # arm64 darwin doesn't support non-PIE binaries, control flow or executable stacks - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector']), - (1, executable+': failed NOUNDEFS LAZY_BINDINGS Canary')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all']), - (1, executable+': failed NOUNDEFS LAZY_BINDINGS')) - self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all']), - (1, executable+': failed LAZY_BINDINGS')) - self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-bind_at_load','-fstack-protector-all']), - (0, '')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fno-stack-protector']), + (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS Canary CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-Wl,-allow_stack_execute','-fstack-protector-all']), + (1, executable+': failed PIE NOUNDEFS NX LAZY_BINDINGS CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-flat_namespace','-fstack-protector-all']), + (1, executable+': failed PIE NOUNDEFS LAZY_BINDINGS CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-fstack-protector-all']), + (1, executable+': failed PIE LAZY_BINDINGS CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all']), + (1, executable+': failed PIE CONTROL_FLOW')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-no_pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), + (1, executable+': failed PIE')) + self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-pie','-Wl,-bind_at_load','-fstack-protector-all', '-fcf-protection=full']), + (0, '')) clean_files(source, executable) diff --git a/contrib/guix/README.md b/contrib/guix/README.md index becf84aa71..432d8e530c 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -159,7 +159,7 @@ which case you can override the default list by setting the space-separated `HOSTS` environment variable: ```sh -env HOSTS='x86_64-w64-mingw32 x86_64-apple-darwin18' ./contrib/guix/guix-build +env HOSTS='x86_64-w64-mingw32 x86_64-apple-darwin' ./contrib/guix/guix-build ``` See the [recognized environment variables][env-vars-list] section for more diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 4bd4e33f9a..dc1baa3aee 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -13,13 +13,10 @@ When complete, it will have produced `DigiByte-Core.dmg`. ### Step 1: Obtaining `Xcode.app` Our current macOS SDK -(`Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`) can be +(`Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz`) can be extracted from -[Xcode_12.2.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.2/Xcode_12.2.xip). -Alternatively, after logging in to your account go to 'Downloads', then 'More' -and look for [`Xcode_12.2`](https://download.developer.apple.com/Developer_Tools/Xcode_12.2/Xcode_12.2.xip). -An Apple ID and cookies enabled for the hostname are needed to download this. -The `sha256sum` of the archive should be `28d352f8c14a43d9b8a082ac6338dc173cb153f964c6e8fb6ba389e5be528bd0`. +[Xcode_12.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.1/Xcode_12.1.xip). +An Apple ID is needed to download this. After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip` archive. This makes the SDK less-trivial to extract on non-macOS machines. One @@ -30,25 +27,25 @@ approach (tested on Debian Buster) is outlined below: apt install cpio git clone https://github.com/digibyte-core/apple-sdk-tools.git -# Unpack Xcode_12.2.xip and place the resulting Xcode.app in your current +# Unpack Xcode_12.1.xip and place the resulting Xcode.app in your current # working directory -python3 apple-sdk-tools/extract_xcode.py -f Xcode_12.2.xip | cpio -d -i +python3 apple-sdk-tools/extract_xcode.py -f Xcode_12.1.xip | cpio -d -i ``` On macOS the process is more straightforward: ```bash -xip -x Xcode_12.2.xip +xip -x Xcode_12.1.xip ``` -### Step 2: Generating `Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app` +### Step 2: Generating `Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app` -To generate `Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`, run +To generate `Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz`, run the script [`gen-sdk`](./gen-sdk) with the path to `Xcode.app` (extracted in the previous stage) as the first argument. ```bash -# Generate a Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz from +# Generate a Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz from # the supplied Xcode.app ./contrib/macdeploy/gen-sdk '/path/to/Xcode.app' ``` @@ -56,13 +53,15 @@ previous stage) as the first argument. ## Deterministic macOS DMG Notes Working macOS DMGs are created in Linux by combining a recent `clang`, the Apple `binutils` (`ld`, `ar`, etc) and DMG authoring tools. + Apple uses `clang` extensively for development and has upstreamed the necessary functionality so that a vanilla clang can take advantage. It supports the use of `-F`, `-target`, `-mmacosx-version-min`, and `-isysroot`, which are all necessary when building for macOS. Apple's version of `binutils` (called `cctools`) contains lots of functionality missing in the -FSF's `binutils`. In addition to extra linker options for frameworks and sysroots, severalother tools are needed as well such as `install_name_tool`, `lipo`, and `nmedit`. These +FSF's `binutils`. In addition to extra linker options for frameworks and sysroots, several +other tools are needed as well such as `install_name_tool`, `lipo`, and `nmedit`. These do not build under Linux, so they have been patched to do so. The work here was used as a starting point: [mingwandroid/toolchain4](https://github.com/mingwandroid/toolchain4). @@ -77,7 +76,7 @@ and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done To complicate things further, all builds must target an Apple SDK. These SDKs are free to download, but not redistributable. To obtain it, register for an Apple Developer Account, -then download [Xcode_12.1](https://download.developer.apple.com/Developer_Tools/Xcode_12.1/Xcode_12.1.xip). +then download [Xcode_11.3.1](https://download.developer.apple.com/Developer_Tools/Xcode_11.3.1/Xcode_11.3.1.xip). This file is many gigabytes in size, but most (but not all) of what we need is contained only in a single directory: @@ -88,35 +87,25 @@ Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk See the SDK Extraction notes above for how to obtain it. -The Guix process build 2 sets of files: Linux tools, then Apple binaries which are +The Gitian descriptors build 2 sets of files: Linux tools, then Apple binaries which are created using these tools. The build process has been designed to avoid including the -SDK's files in Guix's outputs. All interim tarballs are fully deterministic and may be freely +SDK's files in Gitian's outputs. All interim tarballs are fully deterministic and may be freely redistributed. [`xorrisofs`](https://www.gnu.org/software/xorriso/) is used to create the DMG. -`xorrisofs` cannot compress DMGs, so afterwards, the DMG tool from the -`libdmg-hfsplus` project is used to compress it. There are several bugs in this -tool and its maintainer has seemingly abandoned the project. - -The DMG tool has the ability to create DMGs from scratch as well, but this functionality is -broken. Only the compression feature is currently used. Ideally, the creation could be fixed -and `xorrisofs` would no longer be necessary. - -Background images and other features can be added to DMG files by inserting a -`.DS_Store` during creation. +A background image is added to DMG files by inserting a `.DS_Store` during creation. As of OS X 10.9 Mavericks, using an Apple-blessed key to sign binaries is a requirement in order to satisfy the new Gatekeeper requirements. Because this private key cannot be shared, we'll have to be a bit creative in order for the build process to remain somewhat deterministic. Here's how it works: -- Builders use Guix to create an unsigned release. This outputs an unsigned DMG which +- Builders use Gitian to create an unsigned release. This outputs an unsigned DMG which users may choose to bless and run. It also outputs an unsigned app structure in the form of a tarball, which also contains all of the tools that have been previously (deterministically) built in order to create a final DMG. - The Apple keyholder uses this unsigned app to create a detached signature, using the script that is also included there. Detached signatures are available from this [repository](https://github.com/digibyte-core/digibyte-detached-sigs). -- Builders feed the unsigned app + detached signature back into Guix. It uses the +- Builders feed the unsigned app + detached signature back into Gitian. It uses the pre-built tools to recombine the pieces into a deterministic DMG. - \ No newline at end of file