diff --git a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/SourceCodeMinimizer.java b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/SourceCodeMinimizer.java index 85dc0fe..36bccb8 100644 --- a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/SourceCodeMinimizer.java +++ b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/SourceCodeMinimizer.java @@ -8,6 +8,7 @@ import java.math.BigInteger; import java.nio.charset.Charset; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -38,6 +39,7 @@ private static final class ExitException extends Exception { } private final MinimizationStrategy strategy; private final List cutters; private List currentRoots; + private List fileMappings; public SourceCodeMinimizer(SCMConfiguration configuration) throws IOException { MessageDigest md = null; @@ -56,7 +58,8 @@ public SourceCodeMinimizer(SCMConfiguration configuration) throws IOException { Charset sourceCharset = configuration.getSourceCharset(); cutters = new ArrayList<>(); - for (SCMConfiguration.FileMapping mapping: configuration.getFileMappings()) { + fileMappings = configuration.getFileMappings(); + for (SCMConfiguration.FileMapping mapping: fileMappings) { Files.copy(mapping.input, mapping.output, StandardCopyOption.REPLACE_EXISTING); ASTCutter cutter = new ASTCutter(parser, sourceCharset, mapping.output); cutters.add(cutter); @@ -85,6 +88,15 @@ public boolean allInputsAreParseable() throws IOException { return true; } + @Override + public List getScratchFileNames() { + List result = new ArrayList<>(); + for (SCMConfiguration.FileMapping mapping: fileMappings) { + result.add(mapping.output); + } + return result; + } + @Override public NodeInformationProvider getNodeInformationProvider() { return language.getNodeInformationProvider(); diff --git a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractExternalProcessInvariant.java b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractExternalProcessInvariant.java index 491a3e6..bb0e32f 100644 --- a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractExternalProcessInvariant.java +++ b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractExternalProcessInvariant.java @@ -8,6 +8,7 @@ import com.beust.jcommander.Parameter; +import java.io.IOException; import java.io.PrintStream; /** @@ -34,30 +35,40 @@ public String getName() { } } - private InvariantOperations ops; - private String[] commandArgs; + private final String compilerCommandLine; + protected InvariantOperations ops; private int spawnCount; private int fruitfulTests; - private static String[] createCommandLine(AbstractConfiguration configuration) { - if (SystemUtils.IS_OS_WINDOWS) { - return new String[] { "cmd.exe", "/C", configuration.compilerCommandLine }; - } else { - return new String[] { "/bin/sh", "-c", configuration.compilerCommandLine }; - } - } - protected AbstractExternalProcessInvariant(AbstractConfiguration configuration) { - commandArgs = createCommandLine(configuration); + compilerCommandLine = configuration.compilerCommandLine; } @Override - public void initialize(InvariantOperations ops) { + public void initialize(InvariantOperations ops) throws IOException { this.ops = ops; } protected abstract boolean testSatisfied(ProcessBuilder pb) throws Exception; + protected String getCompilerCommandLine() { + return compilerCommandLine; + } + + protected ProcessBuilder createProcessBuilder() { + ProcessBuilder pb = new ProcessBuilder(); + if (SystemUtils.IS_OS_WINDOWS) { + pb.command("cmd.exe", "/C", compilerCommandLine); + } else { + pb.command("/bin/sh", "-c", compilerCommandLine); + } + return pb; + } + + protected boolean testSatisfied() throws Exception { + return testSatisfied(createProcessBuilder()); + } + @Override public boolean checkIsSatisfied() throws Exception { // First, make a fast check that the source can be parsed at all @@ -67,7 +78,7 @@ public boolean checkIsSatisfied() throws Exception { // then proceed to spawning subprocess spawnCount += 1; - boolean result = testSatisfied(new ProcessBuilder().command(commandArgs)); + boolean result = testSatisfied(); fruitfulTests += result ? 1 : 0; return result; diff --git a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractForkServerAwareProcessInvariant.java b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractForkServerAwareProcessInvariant.java new file mode 100644 index 0000000..6bc0563 --- /dev/null +++ b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractForkServerAwareProcessInvariant.java @@ -0,0 +1,326 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.scm.invariants; + +import com.beust.jcommander.Parameter; +import org.apache.commons.lang3.SystemUtils; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * The compiler may have large start-up time. In some cases the + * fork-server + * approach can be applied. Specifically, the compiler should be + *
    + *
  • single-threaded (w.r.t. OS threads)
    • + *
  • not reading input files during start-up
    • + *
+ * + * Protocol: + *
    + *
  • + * Unlike in the traditional AFL forkserver, all communication with JVM + * goes through the same stdout and stderr streams + *
    • + *
  • + * Forkserver communications are distinguished from regular compiler output + * via {@link #FORKSERVER_TO_SCM_MARKER} immediately followed by reply and + * '\n' character + *
    • + *
  • + * JVM process always read output streams until both are terminated via + * {@link #FORKSERVER_TO_SCM_MARKER}, so fork child process timeout is essential + * to return from the fork child to main forkserver loop + *
    • + *
  • + * Spawning a fork server child is performed by writing a single byte (with any value) + * to stdin after {@link #FORKSERVER_TO_SCM_MARKER} was read. + *
    • + *
  • + * The initial forkserver reply should be {@link #FORKSERVER_INIT_REPLY} on stdout, + * after that every fork child execution is summarized with reply containing + * decimal exit code. Reply on stderr should always be empty (i.e., just + * {@link #FORKSERVER_TO_SCM_MARKER} followed by a newline character). + *
    • + *
+ */ +public abstract class AbstractForkServerAwareProcessInvariant extends AbstractExternalProcessInvariant { + /** + * Variable used by the system linker for shared object preloading. + */ + private final static String PRELOAD_VAR = "LD_PRELOAD"; + + /** + * Environment variable name: when set to non-empty value, instructs + * the dynamic linker to resolve all symbols at startup. + * + * Just like in AFL, used to perform as much work in the fork parent + * as possible and not re-do this in every child. + */ + private final static String BIND_NOW_VAR = "LD_BIND_NOW"; + + /** + * Environment variable name: timeout (in seconds) for forkserver child. + */ + private final static String SCM_TIMEOUT_VAR = "__SCM_TIMEOUT"; + + /** + * Environment variable name: i-th input file (0-indexed). + * + * Should be set for continuous range i in [0, N], first absent + * variable terminates the list. + */ + private final static String SCM_INPUT_VAR_FORMAT = "__SCM_INPUT_%d"; + + /** + * Marker string signifying the end of output stream for current child + * or readiness for the first spawn. + * + * Should be printed independently on stdout and stderr. + */ + private final static String FORKSERVER_TO_SCM_MARKER = "## FORKSERVER -> SCM ##"; + + /** + * Marker for the initial forkserver reply. + */ + private final static String FORKSERVER_INIT_REPLY = "INIT"; + + protected abstract static class AbstractConfigurationWithForkServer extends AbstractConfiguration { + @Parameter(names = "--forkserver", description = "Use forkserver (Linux-only)") + private boolean useForkserver; + + @Parameter(names = "--forkserver-child-timeout", description = "Timeout for a single fork server child process") + private int timeoutSec = 1; + } + + protected abstract static class AbstractFactoryWithForkServer extends AbstractFactory { + AbstractFactoryWithForkServer(String name) { + super(name); + } + } + + private final boolean useForkserver; + private final int timeoutSec; + + /** + * Path to compiled native shared object to be injected into the compiler. + */ + private Path preloadedObject; + + private Process forkServer; + private OutputStream forkServerStdin; + private BufferedReader forkServerStdout; + private BufferedReader forkServerStderr; + + /** + * Misc files to be cleaned up just before JVM termination. + */ + private final List deleteAtExit = new ArrayList<>(); + + protected AbstractForkServerAwareProcessInvariant(AbstractConfigurationWithForkServer configuration) { + super(configuration); + useForkserver = configuration.useForkserver; + timeoutSec = configuration.timeoutSec; + Runtime.getRuntime().addShutdownHook(createCleanUpHook()); + } + + private Thread createCleanUpHook() { + return new Thread() { + @Override + public void run() { + for (Path path: deleteAtExit) { + try { + Files.delete(path); + } catch (IOException ex) { + // do nothing + } + } + if (forkServer != null) { + forkServer.destroy(); + } + } + }; + } + + protected Charset getCharset() { + // We need some charset since we are parsing output streams. + // Particular invariants can request some specific one, + // but any ASCII-compatible one is enough for forkserver operation. + return Charset.defaultCharset(); + } + + /** + * Reads all lines of output until the {@link #FORKSERVER_TO_SCM_MARKER}. + * + * @param reader A stream to read from + * @param stdoutContents An output buffer to place lines into + * @return A forkserver reply string (the sequence of characters + * after {@link #FORKSERVER_TO_SCM_MARKER} until '\n') + * or null on unexpected EOF + * @throws IOException + */ + private String readUntilMarker(BufferedReader reader, List stdoutContents) throws IOException { + while (true) { + String line = reader.readLine(); + if (line == null) { + return null; + } + int indexOfMarker = line.indexOf(FORKSERVER_TO_SCM_MARKER); + if (indexOfMarker == -1) { + stdoutContents.add(line); + } else { + stdoutContents.add(line.substring(0, indexOfMarker)); + return line.substring(indexOfMarker + FORKSERVER_TO_SCM_MARKER.length()); + } + } + } + + /** + * Compiles the preloaded shared object on the user's Linux system. + * + * @return A path to the resulted .so-file + * @throws IOException + */ + private Path compilePreloadedObject() throws IOException { + try { + Path input = Files.createTempFile("pmd-scm-forkserver-", ".c"); + deleteAtExit.add(input); + Path output = Files.createTempFile("pmd-scm-forkserver-", ".so"); + deleteAtExit.add(output); + Files.copy(getClass().getResourceAsStream("forksrv-preload.c"), input, StandardCopyOption.REPLACE_EXISTING); + String compiler = System.getenv("CC"); + if (compiler == null) { + compiler = "cc"; + } + Process compilerProcess = new ProcessBuilder() + .command(compiler, "--shared", "-fPIC", input.toString(), "-o", output.toString()) + .inheritIO() + .start(); + int exitCode = compilerProcess.waitFor(); + if (exitCode != 0) { + throw new IOException("Cannot compile forkserver preloaded object: compiler exited with code " + exitCode); + } + return output; + } catch (Exception ex) { + throw new IOException("Cannot compile forkserver preloaded object", ex); + } + } + + @Override + protected ProcessBuilder createProcessBuilder() { + if (useForkserver) { + // Cannot set via ProcessBuilder.environment, otherwise /bin/sh itself + // will be killed on fork(), execve(), etc. + String setEnvString = PRELOAD_VAR + "=" + preloadedObject.toAbsolutePath().toString(); + ProcessBuilder pb = new ProcessBuilder(); + pb.command("/bin/sh", "-c", setEnvString + " " + getCompilerCommandLine()); + Map environment = pb.environment(); + environment.put(SCM_TIMEOUT_VAR, Integer.toString(timeoutSec)); + environment.put(BIND_NOW_VAR, "1"); + List scratchFiles = ops.getScratchFileNames(); + for (int i = 0; i < scratchFiles.size(); ++i) { + environment.put(String.format(SCM_INPUT_VAR_FORMAT, i), scratchFiles.get(i).toString()); + } + return pb; + } else { + return super.createProcessBuilder(); + } + } + + @Override + public void initialize(InvariantOperations ops) throws IOException { + super.initialize(ops); + if (useForkserver) { + // Check that OS is supported + if (!SystemUtils.IS_OS_LINUX) { + throw new IllegalArgumentException("Forkserver is requested on an unsupported OS"); + } + + // Start the forkserver + preloadedObject = compilePreloadedObject(); + forkServer = createProcessBuilder().start(); + forkServerStdin = forkServer.getOutputStream(); + forkServerStdout = new BufferedReader(new InputStreamReader(forkServer.getInputStream(), getCharset())); + forkServerStderr = new BufferedReader(new InputStreamReader(forkServer.getErrorStream(), getCharset())); + + // Read startup messages + List stdoutContents = new ArrayList<>(); + List stderrContents = new ArrayList<>(); + String msgStdOut = readUntilMarker(forkServerStdout, stdoutContents); + String msgStdErr = readUntilMarker(forkServerStderr, stderrContents); + + // Check that the forkserver answered as expected + if (!FORKSERVER_INIT_REPLY.equals(msgStdOut) || !"".equals(msgStdErr)) { + System.err.println("Fork server did not start properly, check your command line."); + System.err.println("Possible causes:"); + System.err.println(" * the compiler process tried to spawn thread or subprocess"); + System.err.println(" * the compiler have not touched any input file"); + System.err.println("STDOUT:"); + for (String line: stdoutContents) { + System.err.println(line); + } + System.err.println("STDERR:"); + for (String line: stderrContents) { + System.err.println(line); + } + throw new IOException("Invalid forkserver reply: stdout[" + msgStdOut + "], stderr[" + msgStdErr + "]"); + } else { + System.out.println("Connected to fork server.\n"); + } + } + } + + protected abstract boolean testSatisfied(int exitCode, List stdout, List stderr); + + // should be called after receiving previous reply + private void spawnForkChild() throws IOException { + byte[] dummy = new byte[1]; + forkServerStdin.write(dummy); + forkServerStdin.flush(); + } + + @Override + protected boolean testSatisfied() throws Exception { + if (useForkserver) { + spawnForkChild(); + + // Read the entire fork child output streams + List stdoutContents = new ArrayList<>(); + List stderrContents = new ArrayList<>(); + String msgStdOut = readUntilMarker(forkServerStdout, stdoutContents); + String msgStdErr = readUntilMarker(forkServerStderr, stderrContents); + + // Check that the forkserver is still functioning correctly + if (msgStdOut == null || msgStdErr == null) { + throw new IOException("Forkserver terminated unexpectedly"); + } + String errorMessage = "Invalid forkserver reply: stdout[" + msgStdOut + "], stderr[" + msgStdErr + "]"; + if (!msgStdErr.isEmpty()) { + throw new IOException(errorMessage); + } + + // Parse the forkserver reply to get exit code + int exitCode; + try { + exitCode = Integer.parseInt(msgStdOut); + } catch (NumberFormatException ex) { + throw new IOException(errorMessage, ex); + } + return testSatisfied(exitCode, stdoutContents, stderrContents); + } else { + return testSatisfied(createProcessBuilder()); + } + } +} diff --git a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/ExitCodeInvariant.java b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/ExitCodeInvariant.java index 3568fe3..7768c13 100644 --- a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/ExitCodeInvariant.java +++ b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/ExitCodeInvariant.java @@ -6,11 +6,13 @@ import com.beust.jcommander.Parameter; +import java.util.List; + /** * Checks that compiler exits with code from the specified range. */ -public class ExitCodeInvariant extends AbstractExternalProcessInvariant { - public static final class Configuration extends AbstractConfiguration { +public class ExitCodeInvariant extends AbstractForkServerAwareProcessInvariant { + public static final class Configuration extends AbstractConfigurationWithForkServer { @Parameter(names = "--min-return", description = "Minimum exit code value (inclusive)") private int min = 1; @@ -38,7 +40,7 @@ public Invariant createChecker() { } } - public static final InvariantConfigurationFactory FACTORY = new AbstractFactory("exitcode") { + public static final InvariantConfigurationFactory FACTORY = new AbstractFactoryWithForkServer("exitcode") { @Override public InvariantConfiguration createConfiguration() { return new Configuration(); @@ -60,13 +62,18 @@ private ExitCodeInvariant(Configuration configuration) { } } + @Override + protected boolean testSatisfied(int exitCode, List stdout, List stderr) { + return min <= exitCode && exitCode <= max; + } + @Override protected boolean testSatisfied(ProcessBuilder pb) throws Exception { Process process = pb.start(); int returnCode = process.waitFor(); - return min <= returnCode && returnCode <= max; + return testSatisfied(returnCode, null, null); } @Override diff --git a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/Invariant.java b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/Invariant.java index bedcb39..0fbaa8a 100644 --- a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/Invariant.java +++ b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/Invariant.java @@ -4,6 +4,7 @@ package net.sourceforge.pmd.scm.invariants; +import java.io.IOException; import java.io.PrintStream; /** @@ -15,7 +16,7 @@ public interface Invariant { * * @param ops Operations provided by the SourceCodeMinimizer */ - void initialize(InvariantOperations ops); + void initialize(InvariantOperations ops) throws IOException; /** * Check that the scratch file in its current state satisfies the invariant. diff --git a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/InvariantOperations.java b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/InvariantOperations.java index 9660038..5bd8742 100644 --- a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/InvariantOperations.java +++ b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/InvariantOperations.java @@ -5,8 +5,8 @@ package net.sourceforge.pmd.scm.invariants; import java.io.IOException; - -import net.sourceforge.pmd.lang.Parser; +import java.nio.file.Path; +import java.util.List; /** * A public interface provided by the {@link net.sourceforge.pmd.scm.SourceCodeMinimizer} to @@ -17,4 +17,9 @@ public interface InvariantOperations { * Test for syntactical validity of all input files. */ boolean allInputsAreParseable() throws IOException; + + /** + * Get names of compiler input files. + */ + List getScratchFileNames(); } diff --git a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/PrintedMessageInvariant.java b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/PrintedMessageInvariant.java index e9c3138..75270ef 100644 --- a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/PrintedMessageInvariant.java +++ b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/PrintedMessageInvariant.java @@ -7,6 +7,7 @@ import java.io.BufferedReader; import java.io.InputStreamReader; import java.nio.charset.Charset; +import java.util.List; import net.sourceforge.pmd.scm.SCMConfiguration; @@ -15,8 +16,8 @@ /** * Checks that the compiler printed the specified message to its stdout or stderr during execution. */ -public class PrintedMessageInvariant extends AbstractExternalProcessInvariant { - public static final class Configuration extends AbstractConfiguration { +public class PrintedMessageInvariant extends AbstractForkServerAwareProcessInvariant { + public static final class Configuration extends AbstractConfigurationWithForkServer { @Parameter(names = "--printed-message", description = "Message that should be printed by the compiler", required = true) private String message; @@ -38,7 +39,7 @@ public PrintedMessageInvariant createChecker() { } } - public static final InvariantConfigurationFactory FACTORY = new AbstractFactory("message") { + public static final InvariantConfigurationFactory FACTORY = new AbstractFactoryWithForkServer("message") { @Override public InvariantConfiguration createConfiguration() { return new Configuration(); @@ -54,11 +55,31 @@ private PrintedMessageInvariant(Configuration configuration) { charset = configuration.charset; } + @Override + protected Charset getCharset() { + return charset; + } + + @Override + protected boolean testSatisfied(int exitCode, List stdout, List stderr) { + for (String line: stdout) { + if (line.contains(message)) { + return true; + } + } + for (String line: stderr) { + if (line.contains(message)) { + return true; + } + } + return false; + } + @Override protected boolean testSatisfied(ProcessBuilder pb) throws Exception { Process process = pb.redirectErrorStream(true).start(); - try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream(), charset))) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream(), getCharset()))) { while (true) { String line = reader.readLine(); if (line == null) { diff --git a/pmd-scm/src/main/resources/net/sourceforge/pmd/scm/invariants/forksrv-preload.c b/pmd-scm/src/main/resources/net/sourceforge/pmd/scm/invariants/forksrv-preload.c new file mode 100644 index 0000000..15b769f --- /dev/null +++ b/pmd-scm/src/main/resources/net/sourceforge/pmd/scm/invariants/forksrv-preload.c @@ -0,0 +1,323 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +#define _GNU_SOURCE +#ifdef NDEBUG +#undef NDEBUG +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TO_SCM_MARK "## FORKSERVER -> SCM ##" + +#if defined(__i386__) +# define SC_NUM_REG REG_EAX +# define ARG_REG_1 REG_EBX +# define ARG_REG_2 REG_ECX +# define ARG_REG_3 REG_EDX +# define ARG_REG_4 REG_ESI +# define ARG_REG_5 REG_EDI +# define ARG_REG_6 REG_EBP +# define RET_REG_1 REG_EAX +#elif defined(__amd64__) +# define SC_NUM_REG REG_RAX +# define ARG_REG_1 REG_RDI +# define ARG_REG_2 REG_RSI +# define ARG_REG_3 REG_RDX +# define ARG_REG_4 REG_R10 +# define ARG_REG_5 REG_R8 +# define ARG_REG_6 REG_R9 +# define RET_REG_1 REG_RAX +#else +# error Unknown CPU architecture +#endif + +#define MARKER 0x12345678 + +#define MAX_BPF_OPS 128 +#define MAX_INPUTS 1024 + +// Inspecting 6-args syscalls is not supported +static int inspected_syscalls[] = { + SYS_open, SYS_openat, SYS_stat, + SYS_execve, SYS_execveat, + SYS_fork, SYS_vfork, SYS_clone, +}; + +struct file_id { + dev_t dev; + ino_t inode; +}; + +static struct file_id input_ids[MAX_INPUTS]; +static int input_id_count; +static int fork_child_timeout; + +static struct file_id get_file_id(const char *name, int force) +{ + struct stat statbuf; + struct file_id result; + + // Bypass seccomp filter + int ret = syscall(SYS_stat, name, &statbuf, 0, 0, 0, MARKER); + + if (ret == 0) { + result.dev = statbuf.st_dev; + result.inode = statbuf.st_ino; + } else { + fprintf(stderr, "Cannot stat: error %d\n", ret); + result.dev = 0; + result.inode = 0; + if (force) { + abort(); + } + } + + return result; +} + +// Print without buffering AND using fflush from a signal handler +static void print(int fd, const char *message) +{ + int len = 0; + for (len = 0; message[len]; ++len); // strlen + write(fd, message, len); +} + +static void write_reply(const char *reply) +{ + // the actual reply string is communicated via stdout + print(STDOUT_FILENO, TO_SCM_MARK); + print(STDOUT_FILENO, reply); + print(STDOUT_FILENO, "\n"); + // stderr reply is always empty + print(STDERR_FILENO, TO_SCM_MARK "\n"); +} + +static void sigalrm_handler(int sig) +{ + abort(); +} + +// TODO make this function be more safe to execute from a SIGSYS handler +static void start_forkserver(void) +{ + // Do not re-enter the forkserver loop + static int started = 0; + if (started) { + return; + } + started = 1; + + // For debug + print(STDERR_FILENO, "Initializing fork server...\n"); + // Make JVM know the forkserver is ready + write_reply("INIT"); + + while (1) { + // wait for command from JVM + char ch; + assert(read(STDIN_FILENO, &ch, 1) == 1); + + // Bypass seccomp filter + int pid = syscall(SYS_fork, 0, 0, 0, 0, 0, MARKER); + if (pid == 0) { + // in child process + struct rlimit rlim; + rlim.rlim_cur = fork_child_timeout; + rlim.rlim_max = fork_child_timeout; + int ret = setrlimit(RLIMIT_CPU, &rlim); + assert(ret == 0); + + signal(SIGALRM, sigalrm_handler); + alarm(fork_child_timeout); + + return; + } + + // in parent process + int status; + int ret = waitpid(pid, &status, 0); + if (ret < 0) { + perror("waitpid"); + abort(); + } + int exit_code; + if (WIFEXITED(status)) { + exit_code = WEXITSTATUS(status); + } else if (WIFSIGNALED(status)) { + exit_code = WTERMSIG(status); + } else { + fprintf(stderr, "waitpid: unknown status %d\n", status); + abort(); + } + char buf[32]; + sprintf(buf, "%d", exit_code); + write_reply(buf); + } +} + +static int is_input_name(const char *name) +{ + struct file_id id = get_file_id(name, 0); + for (int i = 0; i < input_id_count; ++i) { + if (input_ids[i].dev == id.dev && input_ids[i].inode == id.inode) { + return 1; + } + } + return 0; +} + +static void handle_sigsys(int num, siginfo_t *si, void *arg) +{ + ucontext_t *ctx = arg; + greg_t *gregs = ctx->uc_mcontext.gregs; + int sc_num = gregs[SC_NUM_REG]; + switch (sc_num) { + case SYS_open: + if (is_input_name((const char *) gregs[ARG_REG_1])) { + fprintf(stderr, "Opening %s, starting fork server.\n", (const char *) gregs[ARG_REG_1]); + start_forkserver(); + } + break; + case SYS_openat: + if (is_input_name((const char *) gregs[ARG_REG_2])) { + fprintf(stderr, "Opening %s, starting fork server.\n", (const char *) gregs[ARG_REG_2]); + start_forkserver(); + } + break; + case SYS_stat: + if (is_input_name((const char *) gregs[ARG_REG_1])) { + fprintf(stderr, "Calling stat() on %s, starting fork server\n", (const char *) gregs[ARG_REG_1]); + start_forkserver(); + } + break; + case SYS_execve: + case SYS_execveat: + fprintf(stderr, "Process is trying to call exec(...), exiting.\n"); + abort(); + break; + case SYS_fork: + case SYS_vfork: + case SYS_clone: + fprintf(stderr, "Process is trying to spawn a thread or a subprocess, exiting.\n"); + abort(); + break; + default: + start_forkserver(); + } + gregs[RET_REG_1] = syscall(sc_num, + gregs[ARG_REG_1], gregs[ARG_REG_2], gregs[ARG_REG_3], + gregs[ARG_REG_4], gregs[ARG_REG_5], MARKER); +} + +static struct sock_filter *create_filter(int *length) +{ + const int inspected_syscall_count = sizeof(inspected_syscalls) / sizeof(inspected_syscalls[0]); + struct sock_filter *filter = calloc(MAX_BPF_OPS, sizeof(struct sock_filter)); + int filter_length = 0; + + // Test for system call re-entry: 4-byte (BPF_W) MARKER should be set as the 5th arg (0-indexed) + // do not intercepting 6-args syscalls, so it's OK + filter[filter_length++] = (struct sock_filter) BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[5])); + const int reenter_test_index = filter_length; + filter_length += 1; + // Load syscall number + filter[filter_length++] = (struct sock_filter) BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)); + // Decide what to do for this syscall number + // To be filled after exit instruction indices will be known + const int syscall_jumps_start = filter_length; + filter_length += inspected_syscall_count; + // if not inspected, then ALLOW + const int allow_exit_index = filter_length; + filter[filter_length++] = (struct sock_filter) BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW); + const int trap_exit_index = filter_length; + filter[filter_length++] = (struct sock_filter) BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_TRAP); + assert(filter_length < MAX_BPF_OPS); + + // Fill re-enter test: + filter[reenter_test_index] = (struct sock_filter) BPF_JUMP( + BPF_JMP | BPF_JEQ | BPF_K, + MARKER, + allow_exit_index - (reenter_test_index + 1), + 0); + // Fill syscall filter: + for (int i = 0; i < inspected_syscall_count; ++i) { + int ind = syscall_jumps_start + i; + filter[ind] = (struct sock_filter) BPF_JUMP( + BPF_JMP | BPF_JEQ | BPF_K, + inspected_syscalls[i], + trap_exit_index - (ind + 1), 0); + } + *length = filter_length; + return filter; +} + +static void initialize_signal_interceptor(void) +{ + int filter_length; + struct sock_filter *filter = create_filter(&filter_length); + + struct sock_fprog program = { filter_length, filter }; + + struct sigaction sig; + memset(&sig, 0, sizeof(sig)); + sig.sa_sigaction = handle_sigsys; + sig.sa_flags = SA_SIGINFO | SA_NODEFER; + sigaction(SIGSYS, &sig, NULL); + + prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); + int ret = syscall(SYS_seccomp, SECCOMP_SET_MODE_FILTER, 0, &program); + if (ret != 0) { + perror("seccomp"); + abort(); + } + + free(filter); +} + +static void initialize_inputs(void) +{ + char env_name[32]; + for (input_id_count = 0; input_id_count < MAX_INPUTS; ++input_id_count) { + sprintf(env_name, "__SCM_INPUT_%d", input_id_count); + const char *file_name = getenv(env_name); + if (file_name == NULL) { + break; + } else { + fprintf(stderr, "Fetched input name #%d: %s\n", input_id_count, file_name); + input_ids[input_id_count] = get_file_id(file_name, 1); + } + } +} + +static void __attribute__((constructor)) constr(void) +{ + const char *timeout_str = getenv("__SCM_TIMEOUT"); + assert(timeout_str != NULL); + fork_child_timeout = atoi(timeout_str); + + initialize_inputs(); + initialize_signal_interceptor(); +} diff --git a/pmd-scm/src/test/java/net/sourceforge/pmd/scm/AbstractTestWithFiles.java b/pmd-scm/src/test/java/net/sourceforge/pmd/scm/AbstractTestWithFiles.java new file mode 100644 index 0000000..1cfc494 --- /dev/null +++ b/pmd-scm/src/test/java/net/sourceforge/pmd/scm/AbstractTestWithFiles.java @@ -0,0 +1,39 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.scm; + +import org.junit.After; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; + +public class AbstractTestWithFiles { + private final List filesToRemove = new ArrayList<>(); + + @After + public void tearDown() throws IOException { + for (Path file: filesToRemove) { + Files.delete(file); + } + filesToRemove.clear(); + } + + protected Path removeAfterTest(Path path) { + filesToRemove.add(path); + return path; + } + + + protected Path copyToTemporaryFile(InputStream stream, String suffix) throws IOException { + Path file = removeAfterTest(Files.createTempFile("pmd-test-", suffix)); + Files.copy(stream, file, StandardCopyOption.REPLACE_EXISTING); + return file; + } +} diff --git a/pmd-scm/src/test/java/net/sourceforge/pmd/scm/ForkServerAwareInvariantTest.java b/pmd-scm/src/test/java/net/sourceforge/pmd/scm/ForkServerAwareInvariantTest.java new file mode 100644 index 0000000..aac9164 --- /dev/null +++ b/pmd-scm/src/test/java/net/sourceforge/pmd/scm/ForkServerAwareInvariantTest.java @@ -0,0 +1,50 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.scm; + +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.SystemUtils; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +public class ForkServerAwareInvariantTest extends AbstractTestWithFiles { + private void performTesting(String command, String... invariantArgs) throws Exception { + Assume.assumeTrue(SystemUtils.IS_OS_LINUX); + + SCMConfiguration configuration = new SCMConfiguration(); + Path inputFile = copyToTemporaryFile(getClass().getResourceAsStream("test-input.txt"), ".in"); + Path outputFile = removeAfterTest(Files.createTempFile("pmd-test-", ".out")); + String cmdline = command + " " + outputFile.toString(); + + String[] baseArgs = { + "--language", "java", + "--input-file", inputFile.toString(), "--output-file", outputFile.toString(), + "--forkserver", "--command-line", cmdline, + "--strategy", "greedy" + }; + + String[] args = ArrayUtils.addAll(baseArgs, invariantArgs); + configuration.parse(args); + Assert.assertNull(configuration.getErrorString()); + SourceCodeMinimizer minimizer = new SourceCodeMinimizer(configuration); + minimizer.runMinimization(); + TestHelper.assertResultedSourceEquals(StandardCharsets.UTF_8, getClass().getResource("greedy-test-retained-testRemoval.txt"), outputFile); + } + + @Test + public void testExitCode() throws Exception { + performTesting("grep -q -F testRemoval", "--invariant", "exitcode", "--exact-return", "0"); + } + + @Test + public void testPrintedMessage() throws Exception { + performTesting("cat", "--invariant", "message", "--printed-message", "testRemoval"); + } +} diff --git a/pmd-scm/src/test/java/net/sourceforge/pmd/scm/GreedyStrategyTest.java b/pmd-scm/src/test/java/net/sourceforge/pmd/scm/GreedyStrategyTest.java index 9c39972..1c641bb 100644 --- a/pmd-scm/src/test/java/net/sourceforge/pmd/scm/GreedyStrategyTest.java +++ b/pmd-scm/src/test/java/net/sourceforge/pmd/scm/GreedyStrategyTest.java @@ -12,25 +12,19 @@ import java.util.List; import net.sourceforge.pmd.scm.invariants.AbstractExternalProcessInvariant; -import org.apache.commons.lang3.SystemUtils; import org.junit.Assert; import org.junit.Test; -public class GreedyStrategyTest { +public class GreedyStrategyTest extends AbstractTestWithFiles { private int getSpawnCount(SourceCodeMinimizer minimizer) { return ((AbstractExternalProcessInvariant) minimizer.getInvariant()).getSpawnCount(); } private void testRetention(String textToRetain, int maxSpawns, String inputFileName, String referenceFileName) throws Exception { SCMConfiguration configuration = new SCMConfiguration(); - Path inputFile = TestHelper.copyToTemporaryFile(getClass().getResourceAsStream(inputFileName), ".in"); - Path outputFile = Files.createTempFile("pmd-test-", ".out"); - String cmdline; - if (SystemUtils.IS_OS_WINDOWS) { - cmdline = "type " + outputFile.toString(); - } else { - cmdline = "cat " + outputFile.toString(); - } + Path inputFile = copyToTemporaryFile(getClass().getResourceAsStream(inputFileName), ".in"); + Path outputFile = removeAfterTest(Files.createTempFile("pmd-test-", ".out")); + String cmdline = TestHelper.printFileContentsCmd(outputFile.toString()); String[] args = { "--language", "java", "--input-file", inputFile.toString(), "--output-file", outputFile.toString(), "--invariant", "message", "--printed-message", textToRetain, "--command-line", cmdline, @@ -58,10 +52,10 @@ public void performanceTest() throws Exception { @Test public void multiFileJavaMinimization() throws Exception { SCMConfiguration configuration = new SCMConfiguration(); - Path input1 = TestHelper.copyToTemporaryFile(getClass().getResourceAsStream("greedy-multifile-1.java"), ".java"); - Path input2 = TestHelper.copyToTemporaryFile(getClass().getResourceAsStream("greedy-multifile-2.java"), ".java"); + Path input1 = copyToTemporaryFile(getClass().getResourceAsStream("greedy-multifile-1.java"), ".java"); + Path input2 = copyToTemporaryFile(getClass().getResourceAsStream("greedy-multifile-2.java"), ".java"); - Path fileList = Files.createTempFile("pmd-test-file-list", ".txt"); + Path fileList = removeAfterTest(Files.createTempFile("pmd-test-file-list", ".txt")); List fileNames = new ArrayList<>(); fileNames.add(input1.toString()); fileNames.add(input2.toString()); diff --git a/pmd-scm/src/test/java/net/sourceforge/pmd/scm/ScmConfigurationTest.java b/pmd-scm/src/test/java/net/sourceforge/pmd/scm/ScmConfigurationTest.java index 5b66e94..1b2b0da 100644 --- a/pmd-scm/src/test/java/net/sourceforge/pmd/scm/ScmConfigurationTest.java +++ b/pmd-scm/src/test/java/net/sourceforge/pmd/scm/ScmConfigurationTest.java @@ -79,6 +79,12 @@ public class ScmConfigurationTest { + " Compiler should exit with this specific exit value only (implies min == \n" + " max) \n" + " Default: -1\n" + + " --forkserver\n" + + " Use forkserver (Linux-only)\n" + + " Default: false\n" + + " --forkserver-child-timeout\n" + + " Timeout for a single fork server child process\n" + + " Default: 1\n" + " --max-return\n" + " Maximum exit code value (inclusive)\n" + " Default: 2147483647\n" @@ -90,6 +96,12 @@ public class ScmConfigurationTest { + " Options:\n" + " * --command-line\n" + " Command line for running a compiler on a source to be minimized\n" + + " --forkserver\n" + + " Use forkserver (Linux-only)\n" + + " Default: false\n" + + " --forkserver-child-timeout\n" + + " Timeout for a single fork server child process\n" + + " Default: 1\n" + " * --printed-message\n" + " Message that should be printed by the compiler\n" + " --printed-message-encoding\n" diff --git a/pmd-scm/src/test/java/net/sourceforge/pmd/scm/TestHelper.java b/pmd-scm/src/test/java/net/sourceforge/pmd/scm/TestHelper.java index 5e1e12b..2b9e853 100644 --- a/pmd-scm/src/test/java/net/sourceforge/pmd/scm/TestHelper.java +++ b/pmd-scm/src/test/java/net/sourceforge/pmd/scm/TestHelper.java @@ -5,14 +5,13 @@ package net.sourceforge.pmd.scm; import java.io.IOException; -import java.io.InputStream; import java.net.URL; import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.SystemUtils; import org.junit.Assert; public final class TestHelper { @@ -24,9 +23,11 @@ public static void assertResultedSourceEquals(Charset charset, URL expected, Pat Assert.assertEquals(expectedContents, actualContents); } - public static Path copyToTemporaryFile(InputStream stream, String suffix) throws IOException { - Path file = Files.createTempFile("pmd-test-", suffix); - Files.copy(stream, file, StandardCopyOption.REPLACE_EXISTING); - return file; + public static String printFileContentsCmd(String fileName) { + if (SystemUtils.IS_OS_WINDOWS) { + return "type " + fileName; + } else { + return "cat " + fileName; + } } }