cstrans-df-run: propagate args of the RUN directive

src/cstrans-df-run.cc

@@ -50,17 +50,20 @@ class DockerFileTransformer {
         const bool          verbose_;           ///< --verbose on cmd-line
         int                 lineNum_;           ///< line number being read
 
-        bool transformRunLine(std::string *);
+        void transformRunLine(std::string *);
 
         /// match ... in RUN ...
         const RE reLineRun_     = RE("^RUN (.*)$");
 
-        /// match ... in RUN [...]
-        const RE reLineRunExec_ = RE("^RUN  *\\[(.*)\\] *$");
-
         /// match ... in ... BS-NL
         const RE reLineCont_    = RE("(^.*[^\\\\])\\\\ *$");
 
+        // split RUN directive with options from the actual command
+        const RE reLineRunOpts_ = RE("^(RUN +(?:--[A-Za-z0-9_]+=[^ ]+ +)*)(.*)$");
+
+        /// match ... in RUN [...]
+        const RE reLineRunExec_ = RE("^\\[(.*)\\] *$");
+
         /// match in-line comments
         const RE reComment_     = RE("^\\s*#.*$");
 };
@@ -148,10 +151,10 @@ std::string runQuoteArg(std::string arg)
     return arg;
 }
 
-std::string runLineFromExecList(const TStringList &execList)
+std::string runCmdFromExecList(const TStringList &execList)
 {
     // construct RUN ["cmd", "arg1", "arg2", ...] from execList
-    std::string runLine = "RUN [";
+    std::string runLine = "[";
     int i = 0;
     for (const std::string &arg : execList) {
         if (i++)
@@ -163,32 +166,28 @@ std::string runLineFromExecList(const TStringList &execList)
     return runLine;
 }
 
-bool DockerFileTransformer::transformRunLine(std::string *pRunLine)
+void DockerFileTransformer::transformRunLine(std::string *pRunLine)
 {
-    // start with the prefix specified on cmd-line
-    TStringList execList = prefixCmd_;
+    // split RUN directive with options from the actual command
+    boost::smatch sm;
+    if (!boost::regex_match(*pRunLine, sm, reLineRunOpts_))
+        // should never happen
+        throw std::runtime_error("internal error");
 
-    try {
-        boost::smatch sm;
-        if (boost::regex_match(*pRunLine, sm, reLineRunExec_))
-            // RUN ["cmd", "arg1", "arg2", ...]
-            appendExecArgs(&execList, sm[1]);
+    std::string newRunLine = sm[1];
+    const std::string cmd = sm[2];
 
-        else if (boost::regex_match(*pRunLine, sm, reLineRun_))
-            // RUN arbitrary shell code...
-            appendShellExec(&execList, sm[1]);
+    // start with the prefix specified on cmd-line
+    TStringList execList = prefixCmd_;
 
-        else
-            // should never happen
-            throw std::runtime_error("internal error");
-    }
-    catch (const std::runtime_error &e) {
-        std::cerr << prog_name << "error: parsing error on line "
-            << lineNum_ << ": " << e.what() << std::endl;
-        return false;
-    }
+    if (boost::regex_match(cmd, sm, reLineRunExec_))
+        // ["cmd", "arg1", "arg2", ...]
+        appendExecArgs(&execList, sm[1]);
+    else
+        // arbitrary shell code...
+        appendShellExec(&execList, cmd);
 
-    const std::string newRunLine = runLineFromExecList(execList);
+    newRunLine += runCmdFromExecList(execList);
     if (verbose_) {
         // diagnostic output printed with --verbose
         std::cerr << prog_name << " <<< " << *pRunLine << std::endl;
@@ -197,7 +196,6 @@ bool DockerFileTransformer::transformRunLine(std::string *pRunLine)
 
     // return the result of a successful transformation
     *pRunLine = newRunLine;
-    return true;
 }
 
 bool DockerFileTransformer::transform(std::istream &in, std::ostream &out)
@@ -237,8 +235,14 @@ bool DockerFileTransformer::transform(std::istream &in, std::ostream &out)
             continue;
 
         // transform the linearized RUN line
-        if (!this->transformRunLine(&runLine))
+        try {
+            this->transformRunLine(&runLine);
+        }
+        catch (const std::runtime_error &e) {
+            std::cerr << prog_name << "error: parsing error on line "
+                << lineNum_ << ": " << e.what() << std::endl;
             anyError = true;
+        }
 
         // write the transformed RUN line and update state
         out << runLine << std::endl;

tests/cstrans-df-run/0011-stdin.txt

@@ -0,0 +1,38 @@
+## Python cuda base #################################################################
+FROM cuda-base AS python-cuda-base
+
+ENV VIRTUAL_ENV=/opt/vllm
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+# install cuda and common dependencies
+RUN --mount=type=cache,target=/root/.cache/pip \
+    --mount=type=bind,source=requirements-common.txt,target=requirements-common.txt \
+    --mount=type=bind,source=requirements-cuda.txt,target=requirements-cuda.txt \
+    pip install \
+        -r requirements-cuda.txt
+
+## Development #################################################################
+FROM python-cuda-base AS dev
+
+# install build and runtime dependencies
+RUN --mount=type=cache,target=/root/.cache/pip \
+    --mount=type=bind,source=requirements-common.txt,target=requirements-common.txt \
+    --mount=type=bind,source=requirements-cuda.txt,target=requirements-cuda.txt \
+    --mount=type=bind,source=requirements-dev.txt,target=requirements-dev.txt \
+    --mount=type=bind,source=requirements-lint.txt,target=requirements-lint.txt \
+    --mount=type=bind,source=requirements-test.txt,target=requirements-test.txt \
+    pip3 install \
+        -r requirements-cuda.txt \
+        -r requirements-dev.txt
+
+## Builder #####################################################################
+FROM dev AS build
+
+# install build dependencies
+RUN --mount=type=cache,target=/root/.cache/pip \
+    --mount=type=bind,source=requirements-build.txt,target=requirements-build.txt \
+    pip install -r requirements-build.txt
+
+# install compiler cache to speed up compilation leveraging local or remote caching
+# git is required for the cutlass kernels
+RUN rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && rpm -ql epel-release && microdnf install -y git ccache && microdnf clean all

tests/cstrans-df-run/0011-stdout.txt

@@ -0,0 +1,24 @@
+## Python cuda base #################################################################
+FROM cuda-base AS python-cuda-base
+
+ENV VIRTUAL_ENV=/opt/vllm
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+# install cuda and common dependencies
+RUN --mount=type=cache,target=/root/.cache/pip     --mount=type=bind,source=requirements-common.txt,target=requirements-common.txt     --mount=type=bind,source=requirements-cuda.txt,target=requirements-cuda.txt     ["/opt/cov-sa-2019.09/bin/cov-build", "--dir=/cov", "--append-log", "sh", "-c", "pip install         -r requirements-cuda.txt"]
+
+## Development #################################################################
+FROM python-cuda-base AS dev
+
+# install build and runtime dependencies
+RUN --mount=type=cache,target=/root/.cache/pip     --mount=type=bind,source=requirements-common.txt,target=requirements-common.txt     --mount=type=bind,source=requirements-cuda.txt,target=requirements-cuda.txt     --mount=type=bind,source=requirements-dev.txt,target=requirements-dev.txt     --mount=type=bind,source=requirements-lint.txt,target=requirements-lint.txt     --mount=type=bind,source=requirements-test.txt,target=requirements-test.txt     ["/opt/cov-sa-2019.09/bin/cov-build", "--dir=/cov", "--append-log", "sh", "-c", "pip3 install         -r requirements-cuda.txt         -r requirements-dev.txt"]
+
+## Builder #####################################################################
+FROM dev AS build
+
+# install build dependencies
+RUN --mount=type=cache,target=/root/.cache/pip     --mount=type=bind,source=requirements-build.txt,target=requirements-build.txt     ["/opt/cov-sa-2019.09/bin/cov-build", "--dir=/cov", "--append-log", "sh", "-c", "pip install -r requirements-build.txt"]
+
+# install compiler cache to speed up compilation leveraging local or remote caching
+# git is required for the cutlass kernels
+RUN ["/opt/cov-sa-2019.09/bin/cov-build", "--dir=/cov", "--append-log", "sh", "-c", "rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && rpm -ql epel-release && microdnf install -y git ccache && microdnf clean all"]

tests/cstrans-df-run/CMakeLists.txt

@@ -36,3 +36,4 @@ tests_cstrans_df_run(0007)
 tests_cstrans_df_run(0008)
 tests_cstrans_df_run(0009)
 tests_cstrans_df_run(0010)
+tests_cstrans_df_run(0011)

tests/cstrans-df-run/sync.sh

@@ -0,0 +1,21 @@
+#!/bin/zsh
+set -exo pipefail
+
+TEST_ARGS=(-- /opt/cov-sa-2019.09/bin/cov-build --dir=/cov --append-log)
+
+# set path to project root
+PROJECT_ROOT="../.."
+
+if [[ $# -eq 0 ]]; then
+    tests=( *-stdin.txt )
+else
+    tests=( "$@" )
+fi
+
+for tst in "${tests[@]}"; do
+    tst=${tst%-stdin.txt}
+    ${PROJECT_ROOT}/csdiff_build/src/cstrans-df-run \
+        ${TEST_ARGS[@]} \
+        < ${tst}-stdin.txt \
+        > ${tst}-stdout.txt
+done