diff --git a/src/cstrans-df-run.cc b/src/cstrans-df-run.cc index b218350a..4142e457 100644 --- a/src/cstrans-df-run.cc +++ b/src/cstrans-df-run.cc @@ -50,17 +50,20 @@ class DockerFileTransformer { const bool verbose_; ///< --verbose on cmd-line int lineNum_; ///< line number being read - bool transformRunLine(std::string *); + void transformRunLine(std::string *); /// match ... in RUN ... const RE reLineRun_ = RE("^RUN (.*)$"); - /// match ... in RUN [...] - const RE reLineRunExec_ = RE("^RUN *\\[(.*)\\] *$"); - /// match ... in ... BS-NL const RE reLineCont_ = RE("(^.*[^\\\\])\\\\ *$"); + // split RUN directive with options from the actual command + const RE reLineRunOpts_ = RE("^(RUN +(?:--[A-Za-z0-9_]+=[^ ]+ +)*)(.*)$"); + + /// match ... in RUN [...] + const RE reLineRunExec_ = RE("^\\[(.*)\\] *$"); + /// match in-line comments const RE reComment_ = RE("^\\s*#.*$"); }; @@ -148,10 +151,10 @@ std::string runQuoteArg(std::string arg) return arg; } -std::string runLineFromExecList(const TStringList &execList) +std::string runCmdFromExecList(const TStringList &execList) { // construct RUN ["cmd", "arg1", "arg2", ...] from execList - std::string runLine = "RUN ["; + std::string runLine = "["; int i = 0; for (const std::string &arg : execList) { if (i++) @@ -163,32 +166,28 @@ std::string runLineFromExecList(const TStringList &execList) return runLine; } -bool DockerFileTransformer::transformRunLine(std::string *pRunLine) +void DockerFileTransformer::transformRunLine(std::string *pRunLine) { - // start with the prefix specified on cmd-line - TStringList execList = prefixCmd_; + // split RUN directive with options from the actual command + boost::smatch sm; + if (!boost::regex_match(*pRunLine, sm, reLineRunOpts_)) + // should never happen + throw std::runtime_error("internal error"); - try { - boost::smatch sm; - if (boost::regex_match(*pRunLine, sm, reLineRunExec_)) - // RUN ["cmd", "arg1", "arg2", ...] - appendExecArgs(&execList, sm[1]); + std::string newRunLine = sm[1]; + const std::string cmd = sm[2]; - else if (boost::regex_match(*pRunLine, sm, reLineRun_)) - // RUN arbitrary shell code... - appendShellExec(&execList, sm[1]); + // start with the prefix specified on cmd-line + TStringList execList = prefixCmd_; - else - // should never happen - throw std::runtime_error("internal error"); - } - catch (const std::runtime_error &e) { - std::cerr << prog_name << "error: parsing error on line " - << lineNum_ << ": " << e.what() << std::endl; - return false; - } + if (boost::regex_match(cmd, sm, reLineRunExec_)) + // ["cmd", "arg1", "arg2", ...] + appendExecArgs(&execList, sm[1]); + else + // arbitrary shell code... + appendShellExec(&execList, cmd); - const std::string newRunLine = runLineFromExecList(execList); + newRunLine += runCmdFromExecList(execList); if (verbose_) { // diagnostic output printed with --verbose std::cerr << prog_name << " <<< " << *pRunLine << std::endl; @@ -197,7 +196,6 @@ bool DockerFileTransformer::transformRunLine(std::string *pRunLine) // return the result of a successful transformation *pRunLine = newRunLine; - return true; } bool DockerFileTransformer::transform(std::istream &in, std::ostream &out) @@ -237,8 +235,14 @@ bool DockerFileTransformer::transform(std::istream &in, std::ostream &out) continue; // transform the linearized RUN line - if (!this->transformRunLine(&runLine)) + try { + this->transformRunLine(&runLine); + } + catch (const std::runtime_error &e) { + std::cerr << prog_name << "error: parsing error on line " + << lineNum_ << ": " << e.what() << std::endl; anyError = true; + } // write the transformed RUN line and update state out << runLine << std::endl; diff --git a/tests/cstrans-df-run/0011-stdin.txt b/tests/cstrans-df-run/0011-stdin.txt new file mode 100644 index 00000000..866f4b07 --- /dev/null +++ b/tests/cstrans-df-run/0011-stdin.txt @@ -0,0 +1,38 @@ +## Python cuda base ################################################################# +FROM cuda-base AS python-cuda-base + +ENV VIRTUAL_ENV=/opt/vllm +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +# install cuda and common dependencies +RUN --mount=type=cache,target=/root/.cache/pip \ + --mount=type=bind,source=requirements-common.txt,target=requirements-common.txt \ + --mount=type=bind,source=requirements-cuda.txt,target=requirements-cuda.txt \ + pip install \ + -r requirements-cuda.txt + +## Development ################################################################# +FROM python-cuda-base AS dev + +# install build and runtime dependencies +RUN --mount=type=cache,target=/root/.cache/pip \ + --mount=type=bind,source=requirements-common.txt,target=requirements-common.txt \ + --mount=type=bind,source=requirements-cuda.txt,target=requirements-cuda.txt \ + --mount=type=bind,source=requirements-dev.txt,target=requirements-dev.txt \ + --mount=type=bind,source=requirements-lint.txt,target=requirements-lint.txt \ + --mount=type=bind,source=requirements-test.txt,target=requirements-test.txt \ + pip3 install \ + -r requirements-cuda.txt \ + -r requirements-dev.txt + +## Builder ##################################################################### +FROM dev AS build + +# install build dependencies +RUN --mount=type=cache,target=/root/.cache/pip \ + --mount=type=bind,source=requirements-build.txt,target=requirements-build.txt \ + pip install -r requirements-build.txt + +# install compiler cache to speed up compilation leveraging local or remote caching +# git is required for the cutlass kernels +RUN rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && rpm -ql epel-release && microdnf install -y git ccache && microdnf clean all diff --git a/tests/cstrans-df-run/0011-stdout.txt b/tests/cstrans-df-run/0011-stdout.txt new file mode 100644 index 00000000..b5614087 --- /dev/null +++ b/tests/cstrans-df-run/0011-stdout.txt @@ -0,0 +1,24 @@ +## Python cuda base ################################################################# +FROM cuda-base AS python-cuda-base + +ENV VIRTUAL_ENV=/opt/vllm +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +# install cuda and common dependencies +RUN --mount=type=cache,target=/root/.cache/pip --mount=type=bind,source=requirements-common.txt,target=requirements-common.txt --mount=type=bind,source=requirements-cuda.txt,target=requirements-cuda.txt ["/opt/cov-sa-2019.09/bin/cov-build", "--dir=/cov", "--append-log", "sh", "-c", "pip install -r requirements-cuda.txt"] + +## Development ################################################################# +FROM python-cuda-base AS dev + +# install build and runtime dependencies +RUN --mount=type=cache,target=/root/.cache/pip --mount=type=bind,source=requirements-common.txt,target=requirements-common.txt --mount=type=bind,source=requirements-cuda.txt,target=requirements-cuda.txt --mount=type=bind,source=requirements-dev.txt,target=requirements-dev.txt --mount=type=bind,source=requirements-lint.txt,target=requirements-lint.txt --mount=type=bind,source=requirements-test.txt,target=requirements-test.txt ["/opt/cov-sa-2019.09/bin/cov-build", "--dir=/cov", "--append-log", "sh", "-c", "pip3 install -r requirements-cuda.txt -r requirements-dev.txt"] + +## Builder ##################################################################### +FROM dev AS build + +# install build dependencies +RUN --mount=type=cache,target=/root/.cache/pip --mount=type=bind,source=requirements-build.txt,target=requirements-build.txt ["/opt/cov-sa-2019.09/bin/cov-build", "--dir=/cov", "--append-log", "sh", "-c", "pip install -r requirements-build.txt"] + +# install compiler cache to speed up compilation leveraging local or remote caching +# git is required for the cutlass kernels +RUN ["/opt/cov-sa-2019.09/bin/cov-build", "--dir=/cov", "--append-log", "sh", "-c", "rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && rpm -ql epel-release && microdnf install -y git ccache && microdnf clean all"] diff --git a/tests/cstrans-df-run/CMakeLists.txt b/tests/cstrans-df-run/CMakeLists.txt index 4f0dbb35..b5b13fc6 100644 --- a/tests/cstrans-df-run/CMakeLists.txt +++ b/tests/cstrans-df-run/CMakeLists.txt @@ -36,3 +36,4 @@ tests_cstrans_df_run(0007) tests_cstrans_df_run(0008) tests_cstrans_df_run(0009) tests_cstrans_df_run(0010) +tests_cstrans_df_run(0011) diff --git a/tests/cstrans-df-run/sync.sh b/tests/cstrans-df-run/sync.sh new file mode 100755 index 00000000..ae57d797 --- /dev/null +++ b/tests/cstrans-df-run/sync.sh @@ -0,0 +1,21 @@ +#!/bin/zsh +set -exo pipefail + +TEST_ARGS=(-- /opt/cov-sa-2019.09/bin/cov-build --dir=/cov --append-log) + +# set path to project root +PROJECT_ROOT="../.." + +if [[ $# -eq 0 ]]; then + tests=( *-stdin.txt ) +else + tests=( "$@" ) +fi + +for tst in "${tests[@]}"; do + tst=${tst%-stdin.txt} + ${PROJECT_ROOT}/csdiff_build/src/cstrans-df-run \ + ${TEST_ARGS[@]} \ + < ${tst}-stdin.txt \ + > ${tst}-stdout.txt +done