From 7979bd2bba8ee02effc1655d532f5f73a968a8f3 Mon Sep 17 00:00:00 2001 From: qstokkink Date: Fri, 2 Aug 2024 11:39:49 +0200 Subject: [PATCH] WIP: Updated builders to use GitHub Actions --- .github/workflows/build.yml | 105 ++++++++++++++++++++++++++++++++ build/debian/makedist_debian.sh | 28 ++------- build/debian/update_metainfo.py | 49 ++++----------- build/update_version.py | 47 -------------- build/win/build.py | 1 - build/win/clean.bat | 2 - build/win/makedist_win.bat | 22 +------ build/win/replace_nsi.py | 32 ---------- build/win/requirements.txt | 4 -- build/win/resources/tribler.nsi | 17 +++--- requirements-build.txt | 20 ++---- setup.py | 5 +- 12 files changed, 137 insertions(+), 195 deletions(-) create mode 100644 .github/workflows/build.yml delete mode 100644 build/update_version.py delete mode 100644 build/win/clean.bat delete mode 100644 build/win/replace_nsi.py delete mode 100644 build/win/requirements.txt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000..d66c7b9098 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,105 @@ +name: Build +on: pull_request # TODO: Only run on workflow_dispatch +jobs: + build: + strategy: + matrix: + os: [windows-latest] # TODO: Run all [macos-12, ubuntu-latest, windows-latest] COMPLETED: [ubuntu-latest] + + runs-on: ${{ matrix.os }} + + steps: + - name: Check out SimpleFC repository (Windows) + if: matrix.os == 'windows-latest' + uses: actions/checkout@v4 + with: + repository: mazinsw/SimpleFC + - name: Build SimpleFC NSIS Plugin (Windows) + if: matrix.os == 'windows-latest' + run: | + copy "bin\x86-unicode\SimpleFC.dll" "C:\Program Files (x86)\NSIS\Plugins\x86-unicode\SimpleFC.dll" + - name: Check out nsProcess repository (Windows) + if: matrix.os == 'windows-latest' + uses: actions/checkout@v4 + with: + repository: simdsoft/nsProcess + submodules: 'true' + - name: Add msbuild to PATH (Windows) + if: matrix.os == 'windows-latest' + uses: microsoft/setup-msbuild@v2 + - name: Build nsProcess NSIS Plugin (Windows) + if: matrix.os == 'windows-latest' + run: | + cd nsProcess + msbuild nsProcess.vcxproj /p:PlatformToolset=v143 /p:Configuration="Release UNICODE" + copy "Release UNICODE\nsProcess.dll" "C:\Program Files (x86)\NSIS\Plugins\x86-unicode\nsProcess.dll" + cd ../Include + copy nsProcess.nsh "C:\Program Files (x86)\NSIS\Include\nsProcess.nsh" + - name: Check-out repository + uses: actions/checkout@v4 + with: + submodules: 'true' + fetch-tags: 'true' + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'pip' + cache-dependency-path: | + **/requirements-build.txt + - name: Install Dependencies + run: | + pip install --upgrade -r requirements-build.txt + - name: Setup npm + uses: actions/setup-node@v4 + - name: Build npm + run: | + cd src/tribler/ui/ + npm install + npm run build + - name: Determine tag + run: | + git fetch --tags + git for-each-ref --count=1 --sort=-creatordate --format '%(refname)' refs/tags > raw_tag.txt + echo "GITHUB_TAG=$(git name-rev --tags --name-only $(cat raw_tag.txt))" >> $GITHUB_ENV + - name: Prepare version + shell: python + env: + GITHUB_REF: ${{ vars.GITHUB_SHA }} + GITHUB_TAG: ${{ env.GITHUB_TAG }} + run: | + # TODO: The environment variables should be used instead of version.py, this "Prepare version" step can be deleted then. + from os import getenv + from time import ctime + print(f'Writing version ({getenv("GITHUB_SHA")}, {getenv("GITHUB_TAG")})') + with open("src/tribler/core/version.py", "w") as version_py: + version_py.write(f'version_id = "{getenv("GITHUB_TAG")}"\n') + version_py.write(f'build_date = "{ctime()}"\n') + version_py.write(f'commit_id = "{getenv("GITHUB_SHA")}"\n') + - name: Build Executables (Ubuntu) + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt-get install -y --allow-downgrades alien cpio=2.13+dfsg-7 devscripts fakeroot gir1.2-gtk-4.0 rpm + ./build/debian/makedist_debian.sh + - name: Build Executables (MacOS-12) + if: matrix.os == 'macos-12' + run: | + ./build/mac/makedist_macos.sh + - uses: actions/cache/restore@v4 + if: matrix.os == 'windows-latest' + id: restore_cache + with: + path: src/libsodium.dll + key: cache_libsodium_dll + - name: Build Executables (Windows) + if: matrix.os == 'windows-latest' + run: | + ./build/win/makedist_win.bat + - name: Upload Artifacts + uses: actions/upload-artifact@v3 + with: + name: ${{ runner.os }} Build + path: | + build/* + dist/* + #build/debian/*.deb diff --git a/build/debian/makedist_debian.sh b/build/debian/makedist_debian.sh index 70e7f884a0..c55980fa5c 100755 --- a/build/debian/makedist_debian.sh +++ b/build/debian/makedist_debian.sh @@ -2,32 +2,15 @@ set -x # print all commands set -e # exit when any command fails -LOG_LEVEL=${LOG_LEVEL:-"DEBUG"} - -if [[ ! -d build/debian ]]; then - echo "Please run this script from project root as:\n./build/debian/makedist_debian.sh" -fi - rm -rf build/tribler rm -rf dist/tribler rm -rf build/debian/tribler/usr/share/tribler -if [ ! -z "$VENV" ]; then - echo "Setting venv to $VENV" - source $VENV/bin/activate -else - echo "Creating a new venv" - python3 -m venv build-env - . ./build-env/bin/activate -fi - # ----- Install dependencies before the build -python3 -m pip install --upgrade pip -python3 -m pip install --upgrade -r requirements-build.txt +python3 -m pip install --upgrade PyGObject # ----- Update version -python3 ./build/update_version.py -r . -python3 ./build/debian/update_metainfo.py -r . +python3 ./build/debian/update_metainfo.py # ----- Build UI pushd . @@ -37,18 +20,15 @@ npm run build popd # ----- Build binaries -echo Building Tribler using Cx_Freeze python3 setup.py build # ----- Build dpkg cp -r ./dist/tribler ./build/debian/tribler/usr/share/tribler -TRIBLER_VERSION=$(head -n 1 .TriblerVersion) # read the first line only - # Compose the changelog cd ./build/debian/tribler -dch -v $TRIBLER_VERSION "New release" -dch -v $TRIBLER_VERSION "See https://github.com/Tribler/tribler/releases/tag/$TRIBLER_VERSION for more info" +dch -v $GITHUB_TAG "New release" +dch -v $GITHUB_TAG "See https://github.com/Tribler/tribler/releases/tag/$GITHUB_TAG for more info" dpkg-buildpackage -b -rfakeroot -us -uc diff --git a/build/debian/update_metainfo.py b/build/debian/update_metainfo.py index 114623e154..b48733c80d 100644 --- a/build/debian/update_metainfo.py +++ b/build/debian/update_metainfo.py @@ -1,37 +1,12 @@ -import logging -import time -import xml.etree.ElementTree as xml -from argparse import ArgumentParser -from pathlib import Path - -import defusedxml.ElementTree as defxml - -logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO) - - -def parse_arguments(): - parser = ArgumentParser(description='Update Tribler metainfo.xml') - parser.add_argument('-r', '--repo', type=str, help='path to a repository folder', default='.') - return parser.parse_args() - - -if __name__ == '__main__': - arguments = parse_arguments() - - version = Path('.TriblerVersion').read_text().lstrip('v').rstrip('\n') - - release_info = { - 'version': version, - 'date': time.strftime("%Y-%m-%d", time.localtime()) - } - - logger.info(f'Release info: {release_info}') - metainfo_xml = Path(arguments.repo) / 'build/debian/tribler/usr/share/metainfo/org.tribler.Tribler.metainfo.xml' - - xml_dom = defxml.parse(metainfo_xml) - releases = xml_dom.getroot().find('releases') - release = xml.SubElement(releases, 'release', release_info) - - xml_dom.write(metainfo_xml, encoding='utf-8', xml_declaration=True) - logger.info(f'Content of metainfo.xml: {metainfo_xml.read_text()}') +from os import getenv +from time import localtime, strftime +from xml.etree.ElementTree import SubElement, parse + +if __name__ == "__main__": + metainfo_xml = "build/debian/tribler/usr/share/metainfo/org.tribler.Tribler.metainfo.xml" + tree = parse(metainfo_xml) + root = tree.getroot() + releases_tag = root.find("releases") + releases_tag.append(SubElement(releases_tag, "release", {"version": getenv("GITHUB_TAG"), + "date": strftime("%Y-%m-%d", localtime())})) + tree.write(metainfo_xml, encoding="utf-8", xml_declaration=True) diff --git a/build/update_version.py b/build/update_version.py deleted file mode 100644 index 198cca051f..0000000000 --- a/build/update_version.py +++ /dev/null @@ -1,47 +0,0 @@ -import logging -import os -from argparse import ArgumentParser -from pathlib import Path -from time import ctime - -logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO) - - -def parse_arguments(): - parser = ArgumentParser(description='Update Tribler Version') - parser.add_argument('-r', '--repo', type=str, help='path to a repository folder', default='.') - return parser.parse_args() - - -if __name__ == '__main__': - arguments = parse_arguments() - logger.info(f'Arguments: {arguments}') - - ref_name = Path('.TriblerVersion').read_text().rstrip('\n') - logger.info(f'Tribler tag: {ref_name}') - - commit = Path('.TriblerCommit').read_text().rstrip('\n') - logger.info(f'Git Commit: {commit}') - - build_time = ctime() - logger.info(f'Build time: {build_time}') - - sentry_url = os.environ.get('SENTRY_URL', None) - logger.info(f'Sentry URL (hash): {hash(sentry_url)}') - if sentry_url is None: - logger.critical('Sentry url is not defined. To define sentry url use:' - 'EXPORT SENTRY_URL=\n' - 'If you want to disable sentry, then define the following:' - 'EXPORT SENTRY_URL=') - exit(1) - - version_py = Path(arguments.repo) / 'src/tribler/core/version.py' - logger.info(f'Write info to: {version_py}') - version_py.write_text( - f'version_id = "{ref_name}"\n' - f'build_date = "{build_time}"\n' - f'commit_id = "{commit}"\n' - f'sentry_url = "{sentry_url}"\n' - ) - diff --git a/build/win/build.py b/build/win/build.py index d4b5c80a65..3dec6e2ac2 100644 --- a/build/win/build.py +++ b/build/win/build.py @@ -57,7 +57,6 @@ def get_freeze_build_options(): included_files = [ ("src/tribler/ui/public", "lib/tribler/ui/public"), ("src/tribler/ui/dist", "lib/tribler/ui/dist"), - ("src/tribler/core", "tribler_source/tribler/core"), ("src/tribler/ui/public", "tribler_source/tribler/ui/public"), ("src/tribler/ui/dist", "tribler_source/tribler/ui/dist"), diff --git a/build/win/clean.bat b/build/win/clean.bat deleted file mode 100644 index 4291d61f15..0000000000 --- a/build/win/clean.bat +++ /dev/null @@ -1,2 +0,0 @@ -rmdir /S /Q dist -del /S /Q *.pyc diff --git a/build/win/makedist_win.bat b/build/win/makedist_win.bat index 4cc22b965e..2c56f98dda 100644 --- a/build/win/makedist_win.bat +++ b/build/win/makedist_win.bat @@ -26,13 +26,6 @@ REM ----- Clean up call build\win\clean.bat -REM ----- Prepare venv & install dependencies before the build -if defined VENV ( - call "%VENV%\Scripts\activate.bat" -) else ( - echo VENV environment variable is not set. Skipping. -) - call python3 -m pip install --upgrade pip call python3 -m pip install --upgrade -r requirements-build.txt @@ -45,12 +38,6 @@ popd REM ----- Build -REM Arno: When adding files here, make sure tribler.nsi actually -REM packs them in the installer .EXE - -ECHO Install pip dependencies for correct py-installer's work -call python3 -m pip install --upgrade -r build\win\requirements.txt - REM Sandip 2024-03-22: Deprecated, we are not using PyInstaller anymore because of issue with False Malware detections. REM %PYTHONHOME%\Scripts\pyinstaller.exe tribler.spec --log-level=%LOG_LEVEL% || exit /b ECHO Building Tribler using Cx_Freeze @@ -64,15 +51,10 @@ REM copy Tribler\Main\Build\Win\tribler.exe.manifest dist\tribler mkdir dist\tribler\tools copy build\win\tools\reset*.bat dist\tribler\tools -REM Laurens, 2016-04-20: Copy the redistributables of 2008, 2012 and 2015 to the install dir -REM Sandip, 2019-10-24: redistributables 2008, 2012 are not necessary anymore -REM copy C:\build\vc_redist_110.exe dist\tribler -copy C:\build\vc_redist_140.exe dist\tribler - REM Copy various libraries required on runtime (libsodium and openssl) -copy C:\build\libsodium.dll dist\tribler\lib +move src\libsodium.dll dist\tribler\lib REM Sandip, 2024-03-26: Some openssl dlls are missing so need to be copied manually. -copy C:\build\openssl\*.dll dist\tribler\lib +copy C:\Program Files\OpenSSL\*.dll dist\tribler\lib @echo Running NSIS diff --git a/build/win/replace_nsi.py b/build/win/replace_nsi.py deleted file mode 100644 index bb35d758c9..0000000000 --- a/build/win/replace_nsi.py +++ /dev/null @@ -1,32 +0,0 @@ -import logging -from argparse import ArgumentParser -from pathlib import Path - -logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO) - - -def parse_arguments(): - parser = ArgumentParser(description='Update Tribler NSI') - parser.add_argument('-r', '--repo', type=str, help='path to a repository folder', default='.') - parser.add_argument('-a', '--architecture', type=str, help='architecture (x86 or x64)', default='x86') - return parser.parse_args() - - -if __name__ == '__main__': - arguments = parse_arguments() - - version = Path('.TriblerVersion').read_text().lstrip('v').rstrip('\n') - tribler_nsi = Path(arguments.repo) / 'build/win/resources/tribler.nsi' - content = tribler_nsi.read_text() - - content = content.replace('__GIT__', version) - if arguments.architecture == 'x64': - content = content.replace('x86', 'x64') - content = content.replace('"32"', '"64"') - content = content.replace('$PROGRAMFILES', '$PROGRAMFILES64') - - tribler_nsi.write_text(content) - - logger.info(f'Content of tribler.nsi: {content}') - diff --git a/build/win/requirements.txt b/build/win/requirements.txt deleted file mode 100644 index 0e56891899..0000000000 --- a/build/win/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ ---no-binary :all: - -typing_extensions==4.2.0 -pydantic==1.9.0 diff --git a/build/win/resources/tribler.nsi b/build/win/resources/tribler.nsi index 447090ce75..d0b9cb50bd 100644 --- a/build/win/resources/tribler.nsi +++ b/build/win/resources/tribler.nsi @@ -1,8 +1,5 @@ !define PRODUCT "Tribler" -; Laurens, 2016-03-14: The __GIT__ string will be replaced by update_version_from_git.py -; with the current version of the build. -!define VERSION "__GIT__" -; Laurens, 2016-03-14: The _x86 will be replaced by _x64 if needed in update_version_from_git.py +!define VERSION $%GITHUB_TAG% !define BITVERSION "x86" !include "MUI2.nsh" @@ -122,14 +119,16 @@ Section "!Main EXE" SecMain ; ExecWait "$INSTDIR\vc_redist_110.exe /q /norestart" ; Libraries dependant on 2015 are: Python, Qt5 - File vc_redist_140.exe - ExecWait "$INSTDIR\vc_redist_140.exe /q /norestart" + File "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Redist\MSVC\v143\vc_redist.x64.exe" + ExecWait "$INSTDIR\vc_redist.x64.exe /q /norestart" FileOpen $9 "$INSTDIR\tribler.exe.log" w FileWrite $9 "" FileClose $9 - AccessControl::GrantOnFile "$INSTDIR\tribler.exe.log" "(BU)" "FullAccess" - AccessControl::GrantOnFile "$INSTDIR\tribler.exe.log" "(S-1-5-32-545)" "FullAccess" + Exec 'icacls "$INSTDIR\tribler.exe.log" /grant *BU:F' + #AccessControl::GrantOnFile "$INSTDIR\tribler.exe.log" "(BU)" "FullAccess" + Exec 'icacls "$INSTDIR\tribler.exe.log" /grant *S-1-5-32-545:F' + #AccessControl::GrantOnFile "$INSTDIR\tribler.exe.log" "(S-1-5-32-545)" "FullAccess" ; End SetOutPath "$INSTDIR" @@ -199,9 +198,7 @@ SectionEnd !insertmacro MUI_DESCRIPTION_TEXT ${SecDesk} $(DESC_SecDesk) !insertmacro MUI_DESCRIPTION_TEXT ${SecStart} $(DESC_SecStart) !insertmacro MUI_DESCRIPTION_TEXT ${SecDefaultTorrent} $(DESC_SecDefaultTorrent) -!insertmacro MUI_DESCRIPTION_TEXT ${SecDefaultTStream} $(DESC_SecDefaultTStream) !insertmacro MUI_DESCRIPTION_TEXT ${SecDefaultMagnet} $(DESC_SecDefaultMagnet) -!insertmacro MUI_DESCRIPTION_TEXT ${SecDefaultPpsp} $(DESC_SecDefaultPpsp) !insertmacro MUI_FUNCTION_DESCRIPTION_END ;-------------------------------- diff --git a/requirements-build.txt b/requirements-build.txt index cf088a6f22..0a9dca929f 100644 --- a/requirements-build.txt +++ b/requirements-build.txt @@ -1,19 +1,7 @@ -# These are specific versions of the packages that are used for making the build work. -# These are installed before other dependencies. -PyQt5==5.15.10 -PyQt5-sip==12.13.0 -pyqtgraph==0.13.3 - -r requirements.txt -cx_Freeze==7.0.0; sys_platform != 'darwin' -PyInstaller==6.6.0; sys_platform == 'darwin' - -setuptools==65.5.1; sys_platform == 'darwin' -text-unidecode==1.3; sys_platform == 'darwin' - -defusedxml==0.7.1; sys_platform == 'linux2' or sys_platform == 'linux' -markupsafe==2.0.1; sys_platform == 'linux2' or sys_platform == 'linux' -PyGObject==3.44.1; sys_platform == 'linux2' or sys_platform == 'linux' +cx_Freeze; sys_platform != 'darwin' +PyInstaller; sys_platform == 'darwin' -requests==2.31.0 +setuptools +requests diff --git a/setup.py b/setup.py index 2bc3deed6e..e91c228082 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,8 @@ def read_version_from_file(file_path: str) -> str: with open(file_path, encoding="utf-8") as file: file_content = file.read() # Use regular expression to find the version pattern - version_match = re.search(r"^version_id = ['\"]([^'\"]*)['\"]", file_content, re.M) + version_match = re.search(r"^version_id = ['\"]([^'\"]*)['\"]", + file_content, re.MULTILINE) if version_match: version_str = version_match.group(1) return version_str.split("-")[0] @@ -66,7 +67,7 @@ def read_requirements(file_name: str, directory: str = ".") -> list[str]: setup( name="tribler", - version=version, + version=os.getenv("GITHUB_TAG"), description="Privacy enhanced BitTorrent client with P2P content discovery", long_description=Path("README.rst").read_text(encoding="utf-8"), long_description_content_type="text/x-rst",