Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add remill and gap portfiles #950

Open
wants to merge 25 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
a60d907
ports: Add remill port.
xlauko May 18, 2022
c585288
ports: Add gap library.
xlauko May 18, 2022
2371c3c
ports: Add remill port features.
xlauko May 20, 2022
8796ce4
ports: Fix remill features.
xlauko May 20, 2022
79cbb7d
ports: Fix redundent remill debug paths.
xlauko May 24, 2022
c0daa4a
ports: Update remill config paths.
xlauko May 24, 2022
86d7c5c
ports: Update gap port file configs.
xlauko May 24, 2022
adce86a
ports: Don't build gap tests.
xlauko May 24, 2022
9e3add9
ports: Remove empty lib folder from gap port.
xlauko May 24, 2022
ca5dbec
ports: Add zlib to remill dependencies.
xlauko May 24, 2022
d30a83e
ports: Fix config install paths.
xlauko May 25, 2022
839ba4e
ports: Bump up remill port.
xlauko May 26, 2022
44e45e8
Various fixes
ekilmer Jun 8, 2022
c7b613e
ports: bump up gap version
xlauko Jul 14, 2022
e2c32b1
Fix usage file
ekilmer Jul 29, 2022
a3d9751
ports: bump up gap port
xlauko Aug 3, 2022
7c8412e
ports: bump up remill
xlauko Aug 8, 2022
6814704
ports: Bump remill.
lkorenc Jun 23, 2023
8a6507f
ports: bump up default llvm version to 16 for remill
xlauko Jun 24, 2023
18adef4
vcpkg: bump up remill portfile hash
xlauko Jun 24, 2023
dbc72e0
ports: bump up remill version decription
xlauko Jun 27, 2023
1965d3a
Improve port upgrading experience (#1027)
ekilmer Jul 11, 2023
2984768
Combine unknown attrs annotate and attr type attrs (#1028)
PappasBrent Jul 18, 2023
fe68818
Drop llvm-15 + ubuntu 20.04 and more (#1029)
Ninja3047 Aug 2, 2023
6db35b8
gap: bump up portfile
xlauko Sep 15, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 19 additions & 20 deletions .github/workflows/vcpkg_ci_amd64.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,8 @@ jobs:
matrix:
image:
# 'name' is Docker image name whereas 'os' is more generic
- { os: 'ubuntu', name: 'ubuntu-v2', tag: '20.04' }
- { os: 'ubuntu', name: 'ubuntu-v2', tag: '22.04' }
llvm: [ 'llvm-15', 'llvm-16[pasta]', 'llvm-16' ]
llvm: [ 'llvm-16[pasta]', 'llvm-16' ]
target_arch: [ 'x64', 'arm64' ]

container:
Expand Down Expand Up @@ -119,6 +118,12 @@ jobs:
echo "VCPKG_ROOT=$(pwd)/${{ env.ARTIFACT_NAME }}" >> $GITHUB_ENV
echo "TARGET_TRIPLET=${{ matrix.target_arch }}-linux-rel" >> $GITHUB_ENV

- name: Cleanup NuGet
shell: 'bash'
run: |
du -sh ~/.nuget || true
rm -rf ~/.nuget || true

- name: Upload CMake logs on error
if: failure()
uses: actions/upload-artifact@v3
Expand Down Expand Up @@ -175,7 +180,7 @@ jobs:
- name: 'Rellic build'
shell: 'bash'
working-directory: rellic
if: contains(matrix.llvm, 'llvm-15') && matrix.target_arch == 'x64'
if: matrix.target_arch == 'x64'
run: |
# Does not compile with gcc
export CC="$(which clang)"
Expand All @@ -191,10 +196,10 @@ jobs:
cmake --build build --target install
- name: 'Rellic test'
shell: 'bash'
working-directory: rellic/build
if: contains(matrix.llvm, 'llvm-15') && matrix.target_arch == 'x64'
working-directory: rellic
if: matrix.target_arch == 'x64'
run: |
../scripts/roundtrip.py ./tools/rellic-decomp ../tests/tools/decomp "${VCPKG_ROOT}/installed/${TARGET_TRIPLET}/tools/llvm/clang"
cmake --build build --target test

- name: 'Remill dependencies'
shell: 'bash'
Expand All @@ -211,11 +216,6 @@ jobs:
if: matrix.target_arch == 'x64'
working-directory: remill
run: |
if [[ '${{ matrix.image.tag }}' == '20.04' ]]; then
# Remill uses C++20 concepts that aren't supported by gcc-9 in 20.04
export CC="$(which clang)"
export CXX="$(which clang++)"
fi
cmake -G Ninja \
-DCMAKE_VERBOSE_MAKEFILE=ON \
"-DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake" \
Expand All @@ -237,7 +237,7 @@ jobs:
- name: 'Anvill build'
shell: 'bash'
working-directory: anvill
if: contains(matrix.llvm, 'llvm-15') && matrix.target_arch == 'x64'
if: matrix.target_arch == 'x64'
run: |
# TODO: Does not support compilation with gcc
export CC="$(which clang)"
Expand All @@ -257,17 +257,16 @@ jobs:
-S . -B build
cmake --build build
cmake --install build
# NOTE: This is an old test that doesn't make sense anymore
# Need to find some other way to run a smoketest
#- name: 'Anvill test'
# shell: 'bash'
# working-directory: anvill
# if: contains(matrix.llvm, 'llvm-15') && matrix.target_arch == 'x64'
# run: |
# ./install/bin/anvill-decompile-spec --spec ../bin/Decompile/tests/specs/ret0.json --bc_out ./ret0.bc --ir_out ret0.ir
- name: 'Anvill test'
shell: 'bash'
working-directory: anvill
if: matrix.target_arch == 'x64'
run: |
cmake --build build --target test

- name: Cache cleanup and reporting
shell: 'bash'
run: |
rm -rf vcpkg/{buildtrees,installed,packages}
ccache --show-stats
df -h
29 changes: 12 additions & 17 deletions .github/workflows/vcpkg_ci_mac.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ jobs:
matrix:
os:
- { runner: 'macos-12', xcode: '14.2' }
llvm: [ 'llvm-15', 'llvm-16[pasta]', 'llvm-16' ]
llvm: [ 'llvm-16[pasta]', 'llvm-16' ]
target_arch: [ 'x64', 'arm64' ]

runs-on: ${{ matrix.os.runner }}
Expand Down Expand Up @@ -177,7 +177,7 @@ jobs:
- name: 'Rellic build'
shell: 'bash'
working-directory: rellic
if: contains(matrix.llvm, 'llvm-15') && matrix.target_arch == 'x64'
if: matrix.target_arch == 'x64'
run: |
cmake -G Ninja \
-DCMAKE_VERBOSE_MAKEFILE=ON \
Expand All @@ -191,11 +191,10 @@ jobs:
cmake --install build
- name: 'Rellic test'
shell: 'bash'
working-directory: rellic/build
if: contains(matrix.llvm, 'llvm-15') && matrix.target_arch == 'x64'
working-directory: rellic
if: matrix.target_arch == 'x64'
run: |
# Test only should run when we're not cross compiling
../scripts/roundtrip.py ./tools/rellic-decomp ../tests/tools/decomp "${VCPKG_ROOT}/installed/${TARGET_TRIPLET}/tools/llvm/clang"
cmake --build build --target test

- name: 'Remill dependencies'
shell: 'bash'
Expand Down Expand Up @@ -234,7 +233,7 @@ jobs:
- name: 'Anvill build'
shell: 'bash'
working-directory: anvill
if: contains(matrix.llvm, 'llvm-15') && matrix.target_arch == 'x64'
if: matrix.target_arch == 'x64'
run: |
cmake -G Ninja \
-DCMAKE_VERBOSE_MAKEFILE=ON \
Expand All @@ -251,16 +250,12 @@ jobs:
-S . -B build
cmake --build build
cmake --install build

# NOTE: This is an old test that doesn't make sense anymore
# Need to find some other way to run a smoketest
#- name: 'Anvill test'
# shell: 'bash'
# working-directory: anvill
# # if: contains(matrix.llvm, 'llvm-15') && matrix.target_arch == 'x64'
# if: matrix.target_arch == 'x64'
# run: |
# ./install/bin/anvill-decompile-spec -spec ../bin/Decompile/tests/specs/ret0.json -bc_out ./ret0.bc -ir_out ret0.ir
- name: 'Anvill test'
shell: 'bash'
working-directory: anvill
if: matrix.target_arch == 'x64'
run: |
cmake --build build --target test

- name: Cache cleanup and reporting
shell: 'bash'
Expand Down
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,8 @@ See [the vcpkg docs](https://github.com/microsoft/vcpkg/blob/master/docs/example

Installing additional dependencies will not update any existing dependencies by default. We do not update/upgrade by default because this could cause unexpected rebuilds that could potentially take hours (in the case of LLVM). To update dependencies, pass the `--upgrade-ports` option to the build script along with the respective options affecting vcpkg triplet selection (like `--release`).

You must specify the exact package/ports you want to upgrade. If the port does not exist, this will fail.

## Useful manual vcpkg commands

Sometimes it is useful to run vcpkg commands manually for testing a single package. Ideally, someone who wants to do this would read the [vcpkg documentation](https://github.com/microsoft/vcpkg/tree/master/docs), but below we list some commonly used commands. Inspecting the output of the build script will also show all of the vcpkg commands executed.
Expand Down
2 changes: 1 addition & 1 deletion anvill
Submodule anvill updated 94 files
+96 −77 .github/workflows/build.yml
+1 −1 CMakeLists.txt
+6 −6 Dockerfile
+2 −2 README.md
+15 −4 bin/Decompile/Main.cpp
+0 −176 bin/Decompile/tests/scripts/roundtrip.py
+103 −18 ci/challenge_bins_test_settings.json
+168 −10 data_specifications/specification.proto
+14 −0 include/anvill/ABI.h
+4 −0 include/anvill/CrossReferenceFolder.h
+247 −13 include/anvill/Declarations.h
+38 −22 include/anvill/Lifters.h
+7 −2 include/anvill/Optimize.h
+53 −0 include/anvill/Passes/BasicBlockPass.h
+3 −3 include/anvill/Passes/CodeQualityStatCollector.h
+50 −0 include/anvill/Passes/ConvertPointerArithmeticToGEP.h
+29 −0 include/anvill/Passes/InlineBasicBlockFunctions.h
+0 −50 include/anvill/Passes/LowerSwitchIntrinsics.h
+0 −24 include/anvill/Passes/LowerTypeHintIntrinsics.h
+0 −40 include/anvill/Passes/RecoverBasicStackFrame.h
+16 −0 include/anvill/Passes/RemoveAnvillReturns.h
+44 −0 include/anvill/Passes/RemoveCallIntrinsics.h
+0 −63 include/anvill/Passes/RemoveRemillFunctionReturns.h
+34 −0 include/anvill/Passes/ReplaceStackReferences.h
+3 −2 include/anvill/Result.h
+34 −4 include/anvill/Specification.h
+1 −16 include/anvill/Transforms.h
+143 −0 include/anvill/Type.h
+52 −4 include/anvill/Utils.h
+15 −6 lib/ABI.cpp
+1 −1 lib/Arch/AArch64_C.cpp
+10 −9 lib/Arch/Arch.cpp
+10 −7 lib/Arch/Arch.h
+324 −0 lib/Arch/PPC_SysV.cpp
+28 −0 lib/Arch/StubABI.cpp
+40 −32 lib/CMakeLists.txt
+25 −5 lib/CrossReferenceFolder.cpp
+364 −53 lib/Declarations.cpp
+790 −0 lib/Lifters/BasicBlockLifter.cpp
+172 −0 lib/Lifters/BasicBlockLifter.h
+298 −0 lib/Lifters/CodeLifter.cpp
+87 −0 lib/Lifters/CodeLifter.h
+8 −7 lib/Lifters/DataLifter.cpp
+4 −4 lib/Lifters/EntityLifter.cpp
+189 −1,299 lib/Lifters/FunctionLifter.cpp
+73 −225 lib/Lifters/FunctionLifter.h
+34 −19 lib/Lifters/Options.cpp
+18 −20 lib/Lifters/ValueLifter.cpp
+7 −7 lib/Lifters/ValueLifter.h
+136 −22 lib/Optimize.cpp
+58 −27 lib/Passes/CodeQualityStatCollector.cpp
+579 −0 lib/Passes/ConvertPointerArithmeticToGEP.cpp
+38 −0 lib/Passes/InlineBasicBlockFunctions.cpp
+0 −214 lib/Passes/LowerSwitchIntrinsics.cpp
+0 −66 lib/Passes/LowerTypeHintIntrinsics.cpp
+0 −550 lib/Passes/RecoverBasicStackFrame.cpp
+85 −0 lib/Passes/RemoveCallIntrinsics.cpp
+0 −252 lib/Passes/RemoveRemillFunctionReturns.cpp
+9 −10 lib/Passes/RemoveStackPointerCExprs.cpp
+378 −0 lib/Passes/ReplaceStackReferences.cpp
+13 −13 lib/Passes/SliceManager.cpp
+76 −97 lib/Passes/SplitStackFrameAtReturnAddress.cpp
+3 −5 lib/Passes/TransformRemillJumpIntrinsics.cpp
+38 −6 lib/Passes/Utils.cpp
+8 −4 lib/Passes/Utils.h
+247 −55 lib/Protobuf.cpp
+24 −2 lib/Protobuf.h
+61 −9 lib/Specification.cpp
+11 −1 lib/Specification.h
+260 −50 lib/Type.cpp
+365 −112 lib/Utils.cpp
+1 −1 libraries/lifting-tools-ci
+1 −1 remill
+13 −14 scripts/build.sh
+1 −1 scripts/run-on-anghabench.sh
+22 −10 scripts/test-amp-challenge-bins.sh
+6 −12 scripts/test-angha-1k.sh
+1 −2 tests/anvill_passes/CMakeLists.txt
+5,036 −0 tests/anvill_passes/data/MainBasicBlocks.ll
+7 −7 tests/anvill_passes/src/BranchRecoveryPass.cpp
+44 −46 tests/anvill_passes/src/BrightenPointers.cpp
+3 −4 tests/anvill_passes/src/InstructionFolderPass.cpp
+30 −32 tests/anvill_passes/src/RecoverEntityUses.cpp
+0 −317 tests/anvill_passes/src/RecoverStackFrameInformation.cpp
+4 −4 tests/anvill_passes/src/RemoveStackPointerCExprs.cpp
+19 −16 tests/anvill_passes/src/SinkSelectionsIntoBranchTargets.cpp
+4 −6 tests/anvill_passes/src/SplitStackFrameAtReturnAddress.cpp
+0 −319 tests/anvill_passes/src/SwitchLoweringPass.cpp
+51 −0 tests/anvill_passes/src/TestAbstractStackBB.cpp
+47 −47 tests/anvill_passes/src/TransformRemillJump.cpp
+0 −8 tests/anvill_passes/src/Utils.cpp
+0 −2 tests/anvill_passes/src/Utils.h
+4 −5 tests/anvill_passes/src/XorConversionPass.cpp
+0 −10 tests/tools/src/TypeSpecification.cpp
17 changes: 13 additions & 4 deletions build_dependencies.sh
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ while [[ $# -gt 0 ]] ; do
esac
shift
done
msg "Passing extra args to 'vcpkg install':"
msg "Passing extra args to vcpkg:"
msg " " "${VCPKG_ARGS[@]}"

function die_if_not_installed {
Expand Down Expand Up @@ -237,10 +237,19 @@ if [[ ${UPGRADE_PORTS} == "true" ]]; then
cd "${repo_dir}"
(
set -x
# shellcheck disable=SC2046
"${vcpkg_dir}/vcpkg" upgrade "${extra_vcpkg_args[@]}" "${overlays[@]}" --no-dry-run --allow-unsupported
"${vcpkg_dir}/vcpkg" upgrade "${extra_vcpkg_args[@]}" "${overlays[@]}" --allow-unsupported "${VCPKG_ARGS[@]}" || true

set +x
read -p "Are you sure? If so, enter 'y' " -n 1 -r
echo ""
if [[ $REPLY =~ ^[Yy]$ ]]
then
set -x
"${vcpkg_dir}/vcpkg" upgrade "${extra_vcpkg_args[@]}" "${overlays[@]}" --no-dry-run --allow-unsupported "${VCPKG_ARGS[@]}" || exit 1
fi
)
) || exit 1
)
exit 0
fi

deps=()
Expand Down
39 changes: 39 additions & 0 deletions ports/gap/portfile.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
vcpkg_from_github(
OUT_SOURCE_PATH SOURCE_PATH
REPO lifting-bits/gap
REF ad8fefaf7235a9cd6670e272ca4487807ed81f8a
SHA512 9e9259fd4c3c96e31965613092f7dec9df5e236aa4ef9ac122378fe1708d4efd2fb6bd9837530e8474646dc3b0aa409781fd6f45fc28929d07bb54a95a072566
HEAD_REF main
)

vcpkg_cmake_configure(
SOURCE_PATH "${SOURCE_PATH}"
OPTIONS
-DGAP_ENABLE_COROUTINES=ON
-DGAP_ENABLE_TESTING=OFF
-DGAP_ENABLE_EXAMPLES=OFF
-DGAP_INSTALL=ON
-DUSE_SYSTEM_DEPENDENCIES=ON
)

vcpkg_cmake_install()
vcpkg_cmake_config_fixup(
PACKAGE_NAME "gap"
CONFIG_PATH lib/cmake/gap
)

file( REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug" )

# we do not populate lib folder yet
file( REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/lib" )

file(
INSTALL "${SOURCE_PATH}/LICENSE"
DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}"
RENAME copyright
)

file(
INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage"
DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}"
)
4 changes: 4 additions & 0 deletions ports/gap/usage
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
The package gap provides CMake targets:

find_package(gap CONFIG REQUIRED)
target_link_libraries(main PRIVATE gap::gap gap::gap-core gap::gap-settings)
17 changes: 17 additions & 0 deletions ports/gap/vcpkg.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{
"name": "gap",
"version": "0.0.0",
"description": "A utility library to bridge llvm and mlir gaps",
"homepage": "https://github.com/lifting-bits/gap",
"license": "Apache-2.0",
"dependencies": [
{
"name": "vcpkg-cmake",
"host": true
},
{
"name": "vcpkg-cmake-config",
"host": true
}
]
}
Loading