diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index b9f0439a..ecce3d79 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -36,7 +36,7 @@ jobs: # Checkout the code base # ########################## - name: Checkout Code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: # Full git history is needed to get a proper list of changed files within `super-linter` fetch-depth: 0 diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 98fcc60c..a55f9885 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Cancel previous workflow runs - uses: styfle/cancel-workflow-action@0.11.0 + uses: styfle/cancel-workflow-action@0.12.1 with: access_token: ${{ github.token }} @@ -38,8 +38,8 @@ jobs: outputs: run-core: ${{ steps.filter.outputs.all_count != steps.filter.outputs.non-core-files_count }} steps: - - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 id: filter with: filters: | @@ -71,7 +71,7 @@ jobs: rm -rf ${{ github.workspace }}/* || true rm -rf ${{ github.workspace }}/.* || true - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup repo copy run: | git clone $GITHUB_WORKSPACE ${{ env.REMOTE_WORK_DIR }} @@ -88,6 +88,7 @@ jobs: conda activate $PWD/.conda-env git clone https://github.com/riscv-software-src/riscv-isa-sim.git cd riscv-isa-sim + git checkout 4d8651b mkdir build cd build ../configure --prefix=$RISCV --with-boost=no --with-boost-asio=no --with-boost-regex=no diff --git a/.github/workflows/weekly-build.yml b/.github/workflows/weekly-build.yml index a240fef9..16f72276 100644 --- a/.github/workflows/weekly-build.yml +++ b/.github/workflows/weekly-build.yml @@ -25,7 +25,7 @@ jobs: rm -rf ${{ env.REMOTE_WORK_DIR }}/.* || true rm -rf ${{ github.workspace }}/* || true rm -rf ${{ github.workspace }}/.* || true - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup repo copy run: | git clone $GITHUB_WORKSPACE ${{ env.REMOTE_WORK_DIR }} diff --git a/conda-reqs.yaml b/conda-reqs.yaml index a0cd4e70..3c1e22e8 100644 --- a/conda-reqs.yaml +++ b/conda-reqs.yaml @@ -9,7 +9,7 @@ platforms: dependencies: - qemu # from ucb-bar channel - https://github.com/ucb-bar/qemu-feedstock - - python>=3.8 + - python>=3.9,<3.11 - rsync - psutil - doit>=0.34.0 @@ -27,8 +27,8 @@ dependencies: - sphinx - sphinx_rtd_theme - unzip - - gcc<13 - - gxx<13 + - gcc=13.2 + - gxx=13.2 - conda-gcc-specs - binutils - sysroot_linux-64=2.17 diff --git a/riscv-tools.yaml b/riscv-tools.yaml index 35f89fe6..c7ac4ae2 100644 --- a/riscv-tools.yaml +++ b/riscv-tools.yaml @@ -8,4 +8,4 @@ platforms: - linux-64 dependencies: - - riscv-tools==1.0.4 # from ucb-bar channel - https://github.com/ucb-bar/riscv-tools-feedstock + - riscv-tools==1.0.6 # from ucb-bar channel - https://github.com/ucb-bar/riscv-tools-feedstock diff --git a/wlutil/wlutil.py b/wlutil/wlutil.py index 23441006..e9c5caab 100644 --- a/wlutil/wlutil.py +++ b/wlutil/wlutil.py @@ -569,45 +569,88 @@ def waitpid(pid): time.sleep(0.25) -if sp.run(['/usr/bin/sudo', '-ln', 'true'], stderr=sp.DEVNULL, stdout=sp.DEVNULL).returncode == 0: - # User has passwordless sudo available, use the mount command (much faster) - sudoCmd = ["/usr/bin/sudo"] +sudoCmd = ["/usr/bin/sudo"] +pwdlessSudoCmd = [] # set if pwdless sudo is enabled - @contextmanager - def mountImg(imgPath, mntPath): - run(sudoCmd + ["mount", "-o", "loop", imgPath, mntPath]) - try: - yield mntPath - finally: - run_with_retries(sudoCmd + ['umount', mntPath]) -else: - # User doesn't have sudo (use guestmount, slow but reliable) - sudoCmd = [] - - @contextmanager - def mountImg(imgPath, mntPath): - run(['guestmount', '--pid-file', 'guestmount.pid', '-a', imgPath, '-m', '/dev/sda', mntPath]) + +def runnableWithSudo(cmd): + global sudoCmd + return sp.run(sudoCmd + ['-ln', cmd], stderr=sp.DEVNULL, stdout=sp.DEVNULL).returncode == 0 + + +if runnableWithSudo('true'): + # User has passwordless sudo available + pwdlessSudoCmd = sudoCmd + + +def existsAndRunnableWithSudo(cmd): + global sudoCmd + return os.path.exists(cmd) and runnableWithSudo(cmd) + + +@contextmanager +def mountImg(imgPath, mntPath): + global sudoCmd + global pwdlessSudoCmd + + assert imgPath.is_file(), f"Unable to find {imgPath} to mount" + ret = run(["mountpoint", mntPath], check=False).returncode + assert ret == 1, f"{mntPath} already mounted. Somethings wrong" + + uid = sp.run(['id', '-u'], capture_output=True, text=True).stdout.strip() + gid = sp.run(['id', '-g'], capture_output=True, text=True).stdout.strip() + + if pwdlessSudoCmd: + # use faster mount without firesim script since we have pwdless sudo + run(pwdlessSudoCmd + ["mount", "-o", "loop", imgPath, mntPath]) + run(pwdlessSudoCmd + ["chown", "-R", f"{uid}:{gid}", mntPath]) try: - with open('./guestmount.pid', 'r') as pidFile: - mntPid = int(pidFile.readline()) yield mntPath finally: - run(['guestunmount', mntPath]) - os.remove('./guestmount.pid') + run_with_retries(pwdlessSudoCmd + ['umount', mntPath]) + else: + # use either firesim-*mount* cmds if available/useable or default to guestmount (slower but reliable) + fsimMountCmd = '/usr/local/bin/firesim-mount-with-uid-gid' + fsimUnmountCmd = '/usr/local/bin/firesim-unmount' + + if existsAndRunnableWithSudo(fsimMountCmd) and existsAndRunnableWithSudo(fsimUnmountCmd): + run(sudoCmd + [fsimMountCmd, imgPath, mntPath, uid, gid]) + try: + yield mntPath + finally: + run_with_retries(sudoCmd + [fsimUnmountCmd, mntPath]) + else: + pidPath = './guestmount.pid' + run(['guestmount', '--pid-file', pidPath, '-o', f'uid={uid}', '-o', f'gid={gid}', '-a', imgPath, '-m', '/dev/sda', mntPath]) + try: + with open(pidPath, 'r') as pidFile: + mntPid = int(pidFile.readline()) + yield mntPath + finally: + run(['guestunmount', mntPath]) + os.remove(pidPath) - # There is a race-condition in guestmount where a background task keeps - # modifying the image for a period after unmount. This is the documented - # best-practice (see man guestmount). - waitpid(mntPid) + # There is a race-condition in guestmount where a background task keeps + # modifying the image for a period after unmount. This is the documented + # best-practice (see man guestmount). + waitpid(mntPid) def toCpio(src, dst): + global sudoCmd + global pwdlessSudoCmd + log = logging.getLogger() log.debug("Creating Cpio archive from " + str(src)) - with open(dst, 'wb') as outCpio: - p = sp.run(sudoCmd + ["sh", "-c", "find -print0 | cpio --owner root:root --null -ov --format=newc"], - stderr=sp.PIPE, stdout=outCpio, cwd=src) - log.debug(p.stderr.decode('utf-8')) + + fsimCpioCmd = '/usr/local/bin/firesim-cpio' + if existsAndRunnableWithSudo(fsimCpioCmd): + run(sudoCmd + [fsimCpioCmd, src, dst]) + else: + with open(dst, 'wb') as outCpio: + p = sp.run(pwdlessSudoCmd + ["sh", "-c", "find -print0 | cpio --owner root:root --null -ov --format=newc"], + stderr=sp.PIPE, stdout=outCpio, cwd=src) + log.debug(p.stderr.decode('utf-8')) def resizeFS(img, newSize=0): @@ -649,16 +692,50 @@ def copyImgFiles(img, files, direction): files - list of FileSpecs to use direction - "in" or "out" for copying files into or out of the image (respectively) """ + log = logging.getLogger() + assert direction in ['in', 'out'], f"direction={direction} must be either 'in' or 'out'" with mountImg(img, getOpt('mnt-dir')): for f in files: - if direction == 'in': - dst = str(getOpt('mnt-dir') / f.dst.relative_to('/')) - run(sudoCmd + ['cp', '-a', str(f.src), dst]) - elif direction == 'out': - src = str(getOpt('mnt-dir') / f.src.relative_to('/')) - run(sudoCmd + ['cp', '-a', src, str(f.dst)]) - else: - raise ValueError("direction option must be either 'in' or 'out'") + cpSrcMaybeRelPath = f.src if direction == 'in' else f.src.relative_to('/') + cpDstMaybeRelPath = f.dst.relative_to('/') if direction == 'in' else f.dst + cpSrcResPath = cpSrcMaybeRelPath if direction == 'in' else getOpt('mnt-dir') / cpSrcMaybeRelPath + cpDstResPath = getOpt('mnt-dir') / cpDstMaybeRelPath if direction == 'in' else cpDstMaybeRelPath + + # modify perms for dirs to always be able to copy in/out + oldPerms = {} # store old permissions + relaxedPerms = 0o777 # arb. chosen to be very permissive + dirsToModify = [] + + # irrespective if the mountpoint is src/dst, modify all dirs up to mountpoint (including the src/dst dir) + withinMountRelPath = cpDstMaybeRelPath if direction == 'in' else cpSrcMaybeRelPath + withinMountPath = getOpt('mnt-dir') / withinMountRelPath + parents = withinMountRelPath.parents if withinMountRelPath.parents else '.' + dirsToModify.extend([getOpt('mnt-dir') / e for e in reversed(parents)]) + dirsToModify.extend([withinMountPath] if withinMountPath.is_dir() else []) + # also ensure that if copying a directory into a mountpoint, that directory can be written in the mountpoint + dirsToModify.extend([cpDstResPath / cpSrcResPath.name] if direction == 'in' and cpSrcResPath.is_dir() else []) + + # remove duplicates but keep order + dirsToModify = list(dict.fromkeys(dirsToModify)) + + for dirPath in dirsToModify: + perms = int(oct(os.stat(dirPath).st_mode)[-3:], 8) + log.debug(f"Changing permissions of {dirPath} from {oct(perms)}:{type(perms)} to {oct(relaxedPerms)} temporarily") + assert dirPath not in oldPerms, f"Something went wrong. Expected that {dirPath}'s permissions aren't already set" + oldPerms[dirPath] = perms + os.chmod(dirPath, relaxedPerms) + newPerms = int(oct(os.stat(dirPath).st_mode)[-3:], 8) + assert newPerms == relaxedPerms, f"Unable to set perms of {dirPath} to {oct(relaxedPerms)}" + + try: + run(['cp', '-a', '-f', cpSrcResPath, cpDstResPath]) + finally: + for dirPath in dirsToModify: + perms = oldPerms[dirPath] + log.debug(f"Changing permissions of {dirPath} back from {oct(relaxedPerms)} to {oct(perms)}:{type(perms)}") + os.chmod(dirPath, perms) + # verify it's right + assert int(oct(os.stat(dirPath).st_mode)[-3:], 8) == perms, "Unable to revert permissions" def applyOverlay(img, overlay):