From d1773e1990f71d4807179a7e954f7c8401e245c0 Mon Sep 17 00:00:00 2001 From: Anatoly Trosinenko Date: Sun, 5 Jan 2020 18:13:14 +0300 Subject: [PATCH 1/5] Implement injection of a forkserver into the compiler process Implement a forkserver inspired by AFL fuzzer. Unlike the original design, this one relies on library preloading and seccomp to eliminate static or dynamic recompilation of the compiler. For single-threaded, single-process compilers this can be several times faster, provided that the interacting with any input files is delayed until absolutely necessary. For example, it is the case when the compiler process first loads the standard library for several seconds and then processes scratch files in several milliseconds. --- .../pmd/scm/SourceCodeMinimizer.java | 14 +- .../AbstractExternalProcessInvariant.java | 37 +- ...stractForkServerAwareProcessInvariant.java | 348 ++++++++++++++++++ .../pmd/scm/invariants/ExitCodeInvariant.java | 15 +- .../pmd/scm/invariants/Invariant.java | 3 +- .../scm/invariants/InvariantOperations.java | 9 +- .../invariants/PrintedMessageInvariant.java | 29 +- .../pmd/scm/invariants/forksrv-preload.c | 309 ++++++++++++++++ .../pmd/scm/AbstractTestWithFiles.java | 39 ++ .../pmd/scm/ForkServerAwareInvariantTest.java | 50 +++ .../pmd/scm/GreedyStrategyTest.java | 20 +- .../pmd/scm/ScmConfigurationTest.java | 12 + .../net/sourceforge/pmd/scm/TestHelper.java | 13 +- 13 files changed, 854 insertions(+), 44 deletions(-) create mode 100644 pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractForkServerAwareProcessInvariant.java create mode 100644 pmd-scm/src/main/resources/net/sourceforge/pmd/scm/invariants/forksrv-preload.c create mode 100644 pmd-scm/src/test/java/net/sourceforge/pmd/scm/AbstractTestWithFiles.java create mode 100644 pmd-scm/src/test/java/net/sourceforge/pmd/scm/ForkServerAwareInvariantTest.java diff --git a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/SourceCodeMinimizer.java b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/SourceCodeMinimizer.java index 85dc0fe..36bccb8 100644 --- a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/SourceCodeMinimizer.java +++ b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/SourceCodeMinimizer.java @@ -8,6 +8,7 @@ import java.math.BigInteger; import java.nio.charset.Charset; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -38,6 +39,7 @@ private static final class ExitException extends Exception { } private final MinimizationStrategy strategy; private final List cutters; private List currentRoots; + private List fileMappings; public SourceCodeMinimizer(SCMConfiguration configuration) throws IOException { MessageDigest md = null; @@ -56,7 +58,8 @@ public SourceCodeMinimizer(SCMConfiguration configuration) throws IOException { Charset sourceCharset = configuration.getSourceCharset(); cutters = new ArrayList<>(); - for (SCMConfiguration.FileMapping mapping: configuration.getFileMappings()) { + fileMappings = configuration.getFileMappings(); + for (SCMConfiguration.FileMapping mapping: fileMappings) { Files.copy(mapping.input, mapping.output, StandardCopyOption.REPLACE_EXISTING); ASTCutter cutter = new ASTCutter(parser, sourceCharset, mapping.output); cutters.add(cutter); @@ -85,6 +88,15 @@ public boolean allInputsAreParseable() throws IOException { return true; } + @Override + public List getScratchFileNames() { + List result = new ArrayList<>(); + for (SCMConfiguration.FileMapping mapping: fileMappings) { + result.add(mapping.output); + } + return result; + } + @Override public NodeInformationProvider getNodeInformationProvider() { return language.getNodeInformationProvider(); diff --git a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractExternalProcessInvariant.java b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractExternalProcessInvariant.java index 491a3e6..bb0e32f 100644 --- a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractExternalProcessInvariant.java +++ b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractExternalProcessInvariant.java @@ -8,6 +8,7 @@ import com.beust.jcommander.Parameter; +import java.io.IOException; import java.io.PrintStream; /** @@ -34,30 +35,40 @@ public String getName() { } } - private InvariantOperations ops; - private String[] commandArgs; + private final String compilerCommandLine; + protected InvariantOperations ops; private int spawnCount; private int fruitfulTests; - private static String[] createCommandLine(AbstractConfiguration configuration) { - if (SystemUtils.IS_OS_WINDOWS) { - return new String[] { "cmd.exe", "/C", configuration.compilerCommandLine }; - } else { - return new String[] { "/bin/sh", "-c", configuration.compilerCommandLine }; - } - } - protected AbstractExternalProcessInvariant(AbstractConfiguration configuration) { - commandArgs = createCommandLine(configuration); + compilerCommandLine = configuration.compilerCommandLine; } @Override - public void initialize(InvariantOperations ops) { + public void initialize(InvariantOperations ops) throws IOException { this.ops = ops; } protected abstract boolean testSatisfied(ProcessBuilder pb) throws Exception; + protected String getCompilerCommandLine() { + return compilerCommandLine; + } + + protected ProcessBuilder createProcessBuilder() { + ProcessBuilder pb = new ProcessBuilder(); + if (SystemUtils.IS_OS_WINDOWS) { + pb.command("cmd.exe", "/C", compilerCommandLine); + } else { + pb.command("/bin/sh", "-c", compilerCommandLine); + } + return pb; + } + + protected boolean testSatisfied() throws Exception { + return testSatisfied(createProcessBuilder()); + } + @Override public boolean checkIsSatisfied() throws Exception { // First, make a fast check that the source can be parsed at all @@ -67,7 +78,7 @@ public boolean checkIsSatisfied() throws Exception { // then proceed to spawning subprocess spawnCount += 1; - boolean result = testSatisfied(new ProcessBuilder().command(commandArgs)); + boolean result = testSatisfied(); fruitfulTests += result ? 1 : 0; return result; diff --git a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractForkServerAwareProcessInvariant.java b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractForkServerAwareProcessInvariant.java new file mode 100644 index 0000000..0f58e68 --- /dev/null +++ b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractForkServerAwareProcessInvariant.java @@ -0,0 +1,348 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.scm.invariants; + +import com.beust.jcommander.Parameter; +import org.apache.commons.lang3.SystemUtils; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * The compiler may have large start-up time. In some cases the + * fork-server + * approach can be applied. Specifically, the compiler should be + *
    + *
  • single-threaded (w.r.t. OS threads)
    • + *
  • not reading input files during start-up
    • + *
+ * + * Protocol: + *
    + *
  • + * Unlike in the traditional AFL forkserver, all communication with JVM + * goes through the same stdout and stderr streams + *
    • + *
  • + * Forkserver communications are distinguished from regular compiler output + * via {@link #FORKSERVER_TO_SCM_MARKER} immediately followed by reply and + * '\n' character + *
    • + *
  • + * JVM process always read output streams until both are terminated via + * {@link #FORKSERVER_TO_SCM_MARKER}, so fork child process timeout is essential + * to return from the fork child to main forkserver loop + *
    • + *
  • + * Spawning a fork server child is performed by writing a single byte (with any value) + * to stdin after {@link #FORKSERVER_TO_SCM_MARKER} was read. + *
    • + *
  • + * The initial forkserver reply should be {@link #FORKSERVER_INIT_REPLY} on stdout, + * after that every fork child execution is summarized with reply containing + * decimal exit code. Reply on stderr should always be empty (i.e., just + * {@link #FORKSERVER_TO_SCM_MARKER} followed by a newline character). + *
    • + *
+ */ +public abstract class AbstractForkServerAwareProcessInvariant extends AbstractExternalProcessInvariant { + /** + * Variable used by the system linker for shared object preloading. + */ + private final static String PRELOAD_VAR = "LD_PRELOAD"; + + /** + * Environment variable name: timeout (in seconds) for forkserver child. + */ + private final static String SCM_TIMEOUT_VAR = "__SCM_TIMEOUT"; + + /** + * Environment variable name: i-th input file (0-indexed). + * + * Should be set for continuous range i in [0, N], first absent + * variable terminates the list. + */ + private final static String SCM_INPUT_VAR_FORMAT = "__SCM_INPUT_%d"; + + /** + * Marker string signifying the end of output stream for current child + * or readiness for the first spawn. + * + * Should be printed independently on stdout and stderr. + */ + private final static String FORKSERVER_TO_SCM_MARKER = "## FORKSERVER -> SCM ##"; + + /** + * Marker for the initial forkserver reply. + */ + private final static String FORKSERVER_INIT_REPLY = "INIT"; + + /** + * The size of buffer for a single output line read by this class. + */ + private final static int MAX_LINE_LENGTH = 65536; + + protected abstract static class AbstractConfigurationWithForkServer extends AbstractConfiguration { + @Parameter(names = "--forkserver", description = "Use forkserver (Linux-only)") + private boolean useForkserver; + + @Parameter(names = "--forkserver-child-timeout", description = "Timeout for a single fork server child process") + private int timeoutSec = 1; + } + + protected abstract static class AbstractFactoryWithForkServer extends AbstractFactory { + AbstractFactoryWithForkServer(String name) { + super(name); + } + } + + private final boolean useForkserver; + private final int timeoutSec; + + /** + * Path to compiled native shared object to be injected into the compiler. + */ + private Path preloadedObject; + + private Process forkServer; + private OutputStream forkServerStdin; + private InputStreamReader forkServerStdout; + private InputStreamReader forkServerStderr; + + /** + * Misc files to be cleaned up just before JVM termination. + */ + private final List deleteAtExit = new ArrayList<>(); + + protected AbstractForkServerAwareProcessInvariant(AbstractConfigurationWithForkServer configuration) { + super(configuration); + useForkserver = configuration.useForkserver; + timeoutSec = configuration.timeoutSec; + Runtime.getRuntime().addShutdownHook(createCleanUpHook()); + } + + private Thread createCleanUpHook() { + return new Thread() { + @Override + public void run() { + for (Path path: deleteAtExit) { + try { + Files.delete(path); + } catch (IOException ex) { + // do nothing + } + } + if (forkServer != null) { + forkServer.destroy(); + } + } + }; + } + + protected Charset getCharset() { + // We need some charset since we are parsing output streams. + // Particular invariants can request some specific one, + // but any ASCII-compatible one is enough for forkserver operation. + return Charset.defaultCharset(); + } + + /** + * Reads a line of text until '\n', byte-by-byte. + * + * This is used to avoid deadlocks: more output may be read from the forkserver, + * but only after writing request to stdin. But first we should know that + * the forkserver is ready -- by reading its output streams. + * @param reader A stream to read from + * @param lineBuffer A scratch buffer + * @return A single line of output until '\n' (excluding) or EOF (if non-empty) + * or null, if got EOF as the very first character + * @throws IOException + */ + private String readString(InputStreamReader reader, char[] lineBuffer) throws IOException { + int lineLength; + for (lineLength = 0; lineLength < lineBuffer.length; ++lineLength) { + int ch = reader.read(); + if (ch == -1 && lineLength == 0) { + return null; + } + if (ch == -1 || ch == '\n') { + break; + } + lineBuffer[lineLength] = (char) ch; + } + return new String(lineBuffer, 0, lineLength); + } + + /** + * Reads all lines of output until the {@link #FORKSERVER_TO_SCM_MARKER}. + * + * @param reader A stream to read from + * @param stdoutContents An output buffer to place lines into + * @return A forkserver reply string (the sequence of characters + * after {@link #FORKSERVER_TO_SCM_MARKER} until '\n') + * or null on unexpected EOF + * @throws IOException + */ + private String readUntilMarker(InputStreamReader reader, List stdoutContents) throws IOException { + char[] lineBuffer = new char[MAX_LINE_LENGTH]; + while (true) { + String line = readString(reader, lineBuffer); + if (line == null) { + return null; + } + int indexOfMarker = line.indexOf(FORKSERVER_TO_SCM_MARKER); + if (indexOfMarker == -1) { + stdoutContents.add(line); + } else { + stdoutContents.add(line.substring(0, indexOfMarker)); + return line.substring(indexOfMarker + FORKSERVER_TO_SCM_MARKER.length()); + } + } + } + + /** + * Compiles the preloaded shared object on the user's Linux system. + * + * @return A path to the resulted .so-file + * @throws IOException + */ + private Path compilePreloadedObject() throws IOException { + try { + Path input = Files.createTempFile("pmd-scm-forkserver-", ".c"); + deleteAtExit.add(input); + Path output = Files.createTempFile("pmd-scm-forkserver-", ".so"); + deleteAtExit.add(output); + Files.copy(getClass().getResourceAsStream("forksrv-preload.c"), input, StandardCopyOption.REPLACE_EXISTING); + String compiler = System.getenv("CC"); + if (compiler == null) { + compiler = "cc"; + } + Process compilerProcess = new ProcessBuilder() + .command(compiler, "--shared", "-fPIC", input.toString(), "-o", output.toString()) + .inheritIO() + .start(); + int exitCode = compilerProcess.waitFor(); + if (exitCode != 0) { + throw new IOException("Cannot compile forkserver preloaded object: compiler exited with code " + exitCode); + } + return output; + } catch (Exception ex) { + throw new IOException("Cannot compile forkserver preloaded object", ex); + } + } + + @Override + protected ProcessBuilder createProcessBuilder() { + if (useForkserver) { + // Cannot set via ProcessBuilder.environment, otherwise /bin/sh itself + // will be killed on fork(), execve(), etc. + String setEnvString = PRELOAD_VAR + "=" + preloadedObject.toAbsolutePath().toString(); + ProcessBuilder pb = new ProcessBuilder(); + pb.command("/bin/sh", "-c", setEnvString + " " + getCompilerCommandLine()); + Map environment = pb.environment(); + environment.put(SCM_TIMEOUT_VAR, Integer.toString(timeoutSec)); + List scratchFiles = ops.getScratchFileNames(); + for (int i = 0; i < scratchFiles.size(); ++i) { + environment.put(String.format(SCM_INPUT_VAR_FORMAT, i), scratchFiles.get(i).toString()); + } + return pb; + } else { + return super.createProcessBuilder(); + } + } + + @Override + public void initialize(InvariantOperations ops) throws IOException { + super.initialize(ops); + if (useForkserver) { + // Check that OS is supported + if (!SystemUtils.IS_OS_LINUX) { + throw new IllegalArgumentException("Forkserver is requested on an unsupported OS"); + } + + // Start the forkserver + preloadedObject = compilePreloadedObject(); + forkServer = createProcessBuilder().start(); + forkServerStdin = forkServer.getOutputStream(); + forkServerStdout = new InputStreamReader(forkServer.getInputStream(), getCharset()); + forkServerStderr = new InputStreamReader(forkServer.getErrorStream(), getCharset()); + + // Read startup messages + List stdoutContents = new ArrayList<>(); + List stderrContents = new ArrayList<>(); + String msgStdOut = readUntilMarker(forkServerStdout, stdoutContents); + String msgStdErr = readUntilMarker(forkServerStderr, stderrContents); + + // Check that the forkserver answered as expected + if (!FORKSERVER_INIT_REPLY.equals(msgStdOut) || !"".equals(msgStdErr)) { + System.err.println("Fork server did not start properly, check your command line."); + System.err.println("Possible causes:"); + System.err.println(" * the compiler process tried to spawn thread or subprocess"); + System.err.println(" * the compiler have not touched any input file"); + System.err.println("STDOUT:"); + for (String line: stdoutContents) { + System.err.println(line); + } + System.err.println("STDERR:"); + for (String line: stderrContents) { + System.err.println(line); + } + throw new IOException("Invalid forkserver reply: stdout[" + msgStdOut + "], stderr[" + msgStdErr + "]"); + } else { + System.out.println("Connected to fork server.\n"); + } + } + } + + protected abstract boolean testSatisfied(int exitCode, List stdout, List stderr); + + // should be called after receiving previous reply + private void spawnForkChild() throws IOException { + byte[] dummy = new byte[1]; + forkServerStdin.write(dummy); + forkServerStdin.flush(); + } + + @Override + protected boolean testSatisfied() throws Exception { + if (useForkserver) { + spawnForkChild(); + + // Read the entire fork child output streams + List stdoutContents = new ArrayList<>(); + List stderrContents = new ArrayList<>(); + String msgStdOut = readUntilMarker(forkServerStdout, stdoutContents); + String msgStdErr = readUntilMarker(forkServerStderr, stderrContents); + + // Check that the forkserver is still functioning correctly + if (msgStdOut == null || msgStdErr == null) { + throw new IOException("Forkserver terminated unexpectedly"); + } + String errorMessage = "Invalid forkserver reply: stdout[" + msgStdOut + "], stderr[" + msgStdErr + "]"; + if (!msgStdErr.isEmpty()) { + throw new IOException(errorMessage); + } + + // Parse the forkserver reply to get exit code + int exitCode; + try { + exitCode = Integer.parseInt(msgStdOut); + } catch (NumberFormatException ex) { + throw new IOException(errorMessage, ex); + } + return testSatisfied(exitCode, stdoutContents, stderrContents); + } else { + return testSatisfied(createProcessBuilder()); + } + } +} diff --git a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/ExitCodeInvariant.java b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/ExitCodeInvariant.java index 3568fe3..7768c13 100644 --- a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/ExitCodeInvariant.java +++ b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/ExitCodeInvariant.java @@ -6,11 +6,13 @@ import com.beust.jcommander.Parameter; +import java.util.List; + /** * Checks that compiler exits with code from the specified range. */ -public class ExitCodeInvariant extends AbstractExternalProcessInvariant { - public static final class Configuration extends AbstractConfiguration { +public class ExitCodeInvariant extends AbstractForkServerAwareProcessInvariant { + public static final class Configuration extends AbstractConfigurationWithForkServer { @Parameter(names = "--min-return", description = "Minimum exit code value (inclusive)") private int min = 1; @@ -38,7 +40,7 @@ public Invariant createChecker() { } } - public static final InvariantConfigurationFactory FACTORY = new AbstractFactory("exitcode") { + public static final InvariantConfigurationFactory FACTORY = new AbstractFactoryWithForkServer("exitcode") { @Override public InvariantConfiguration createConfiguration() { return new Configuration(); @@ -60,13 +62,18 @@ private ExitCodeInvariant(Configuration configuration) { } } + @Override + protected boolean testSatisfied(int exitCode, List stdout, List stderr) { + return min <= exitCode && exitCode <= max; + } + @Override protected boolean testSatisfied(ProcessBuilder pb) throws Exception { Process process = pb.start(); int returnCode = process.waitFor(); - return min <= returnCode && returnCode <= max; + return testSatisfied(returnCode, null, null); } @Override diff --git a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/Invariant.java b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/Invariant.java index bedcb39..0fbaa8a 100644 --- a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/Invariant.java +++ b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/Invariant.java @@ -4,6 +4,7 @@ package net.sourceforge.pmd.scm.invariants; +import java.io.IOException; import java.io.PrintStream; /** @@ -15,7 +16,7 @@ public interface Invariant { * * @param ops Operations provided by the SourceCodeMinimizer */ - void initialize(InvariantOperations ops); + void initialize(InvariantOperations ops) throws IOException; /** * Check that the scratch file in its current state satisfies the invariant. diff --git a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/InvariantOperations.java b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/InvariantOperations.java index 9660038..5bd8742 100644 --- a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/InvariantOperations.java +++ b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/InvariantOperations.java @@ -5,8 +5,8 @@ package net.sourceforge.pmd.scm.invariants; import java.io.IOException; - -import net.sourceforge.pmd.lang.Parser; +import java.nio.file.Path; +import java.util.List; /** * A public interface provided by the {@link net.sourceforge.pmd.scm.SourceCodeMinimizer} to @@ -17,4 +17,9 @@ public interface InvariantOperations { * Test for syntactical validity of all input files. */ boolean allInputsAreParseable() throws IOException; + + /** + * Get names of compiler input files. + */ + List getScratchFileNames(); } diff --git a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/PrintedMessageInvariant.java b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/PrintedMessageInvariant.java index e9c3138..75270ef 100644 --- a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/PrintedMessageInvariant.java +++ b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/PrintedMessageInvariant.java @@ -7,6 +7,7 @@ import java.io.BufferedReader; import java.io.InputStreamReader; import java.nio.charset.Charset; +import java.util.List; import net.sourceforge.pmd.scm.SCMConfiguration; @@ -15,8 +16,8 @@ /** * Checks that the compiler printed the specified message to its stdout or stderr during execution. */ -public class PrintedMessageInvariant extends AbstractExternalProcessInvariant { - public static final class Configuration extends AbstractConfiguration { +public class PrintedMessageInvariant extends AbstractForkServerAwareProcessInvariant { + public static final class Configuration extends AbstractConfigurationWithForkServer { @Parameter(names = "--printed-message", description = "Message that should be printed by the compiler", required = true) private String message; @@ -38,7 +39,7 @@ public PrintedMessageInvariant createChecker() { } } - public static final InvariantConfigurationFactory FACTORY = new AbstractFactory("message") { + public static final InvariantConfigurationFactory FACTORY = new AbstractFactoryWithForkServer("message") { @Override public InvariantConfiguration createConfiguration() { return new Configuration(); @@ -54,11 +55,31 @@ private PrintedMessageInvariant(Configuration configuration) { charset = configuration.charset; } + @Override + protected Charset getCharset() { + return charset; + } + + @Override + protected boolean testSatisfied(int exitCode, List stdout, List stderr) { + for (String line: stdout) { + if (line.contains(message)) { + return true; + } + } + for (String line: stderr) { + if (line.contains(message)) { + return true; + } + } + return false; + } + @Override protected boolean testSatisfied(ProcessBuilder pb) throws Exception { Process process = pb.redirectErrorStream(true).start(); - try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream(), charset))) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream(), getCharset()))) { while (true) { String line = reader.readLine(); if (line == null) { diff --git a/pmd-scm/src/main/resources/net/sourceforge/pmd/scm/invariants/forksrv-preload.c b/pmd-scm/src/main/resources/net/sourceforge/pmd/scm/invariants/forksrv-preload.c new file mode 100644 index 0000000..e39a86c --- /dev/null +++ b/pmd-scm/src/main/resources/net/sourceforge/pmd/scm/invariants/forksrv-preload.c @@ -0,0 +1,309 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +#define _GNU_SOURCE +#ifdef NDEBUG +#undef NDEBUG +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TO_SCM_MARK "## FORKSERVER -> SCM ##" + +#if defined(__i386__) +# define SC_NUM_REG REG_EAX +# define ARG_REG_1 REG_EBX +# define ARG_REG_2 REG_ECX +# define ARG_REG_3 REG_EDX +# define ARG_REG_4 REG_ESI +# define ARG_REG_5 REG_EDI +# define ARG_REG_6 REG_EBP +# define RET_REG_1 REG_EAX +#elif defined(__amd64__) +# define SC_NUM_REG REG_RAX +# define ARG_REG_1 REG_RDI +# define ARG_REG_2 REG_RSI +# define ARG_REG_3 REG_RDX +# define ARG_REG_4 REG_R10 +# define ARG_REG_5 REG_R8 +# define ARG_REG_6 REG_R9 +# define RET_REG_1 REG_RAX +#else +# error Unknown CPU architecture +#endif + +#define MARKER 0x12345678 + +#define MAX_BPF_OPS 128 +#define MAX_INPUTS 1024 + +// Inspecting 6-args syscalls is not supported +static int inspected_syscalls[] = { SYS_execve, SYS_execveat, SYS_fork, SYS_vfork, SYS_clone, SYS_openat, SYS_stat }; + +struct file_id { + dev_t dev; + ino_t inode; +}; + +static struct file_id input_ids[MAX_INPUTS]; +static int input_id_count; +static int fork_child_timeout; + +static struct file_id get_file_id(const char *name, int force) +{ + struct stat statbuf; + struct file_id result; + + // Bypass seccomp filter + int ret = syscall(SYS_stat, name, &statbuf, 0, 0, 0, MARKER); + + if (ret == 0) { + result.dev = statbuf.st_dev; + result.inode = statbuf.st_ino; + } else { + fprintf(stderr, "Cannot stat: error %d\n", ret); + result.dev = 0; + result.inode = 0; + if (force) { + abort(); + } + } + + return result; +} + +// Print without buffering AND using fflush from a signal handler +static void print(int fd, const char *message) +{ + int len = 0; + for (len = 0; message[len]; ++len); // strlen + write(fd, message, len); +} + +static void write_reply(const char *reply) +{ + // the actual reply string is communicated via stdout + print(STDOUT_FILENO, TO_SCM_MARK); + print(STDOUT_FILENO, reply); + print(STDOUT_FILENO, "\n"); + // stderr reply is always empty + print(STDERR_FILENO, TO_SCM_MARK "\n"); +} + +static void sigalrm_handler(int sig) +{ + abort(); +} + +// TODO make this function be more safe to execute from a SIGSYS handler +static void start_forkserver(void) +{ + // Do not re-enter the forkserver loop + static int started = 0; + if (started) { + return; + } + started = 1; + + // For debug + print(STDERR_FILENO, "Initializing fork server...\n"); + // Make JVM know the forkserver is ready + write_reply("INIT"); + + while (1) { + // wait for command from JVM + char ch; + assert(read(STDIN_FILENO, &ch, 1) == 1); + + // Bypass seccomp filter + int pid = syscall(SYS_fork, 0, 0, 0, 0, 0, MARKER); + if (pid == 0) { + // in child process + struct rlimit rlim; + rlim.rlim_cur = fork_child_timeout; + rlim.rlim_max = fork_child_timeout; + int ret = setrlimit(RLIMIT_CPU, &rlim); + assert(ret == 0); + + signal(SIGALRM, sigalrm_handler); + alarm(fork_child_timeout); + + return; + } + + // in parent process + int status; + int ret = waitpid(pid, &status, 0); + if (ret < 0) { + perror("waitpid"); + abort(); + } + int exit_code; + if (WIFEXITED(status)) { + exit_code = WEXITSTATUS(status); + } else if (WIFSIGNALED(status)) { + exit_code = WTERMSIG(status); + } else { + fprintf(stderr, "waitpid: unknown status %d\n", status); + abort(); + } + char buf[32]; + sprintf(buf, "%d", exit_code); + write_reply(buf); + } +} + +static int is_input_name(const char *name) +{ + struct file_id id = get_file_id(name, 0); + for (int i = 0; i < input_id_count; ++i) { + if (input_ids[i].dev == id.dev && input_ids[i].inode == id.inode) { + return 1; + } + } + return 0; +} + +static void handle_sigsys(int num, siginfo_t *si, void *arg) +{ + ucontext_t *ctx = arg; + greg_t *gregs = ctx->uc_mcontext.gregs; + int sc_num = gregs[SC_NUM_REG]; + switch (sc_num) { + case SYS_openat: + if (is_input_name((const char *) gregs[ARG_REG_2])) { + fprintf(stderr, "Opening %s, starting fork server.\n", (const char *) gregs[ARG_REG_2]); + start_forkserver(); + } + break; + case SYS_stat: + if (is_input_name((const char *) gregs[ARG_REG_1])) { + fprintf(stderr, "Calling stat() on %s, starting fork server\n", (const char *) gregs[ARG_REG_1]); + start_forkserver(); + } + break; + case SYS_execve: + case SYS_execveat: + fprintf(stderr, "Process is trying to call exec(...), exiting.\n"); + abort(); + break; + case SYS_fork: + case SYS_vfork: + case SYS_clone: + fprintf(stderr, "Process is trying to spawn a thread or a subprocess, exiting.\n"); + abort(); + break; + default: + start_forkserver(); + } + gregs[RET_REG_1] = syscall(sc_num, + gregs[ARG_REG_1], gregs[ARG_REG_2], gregs[ARG_REG_3], + gregs[ARG_REG_4], gregs[ARG_REG_5], MARKER); +} + +static struct sock_filter *create_filter(int *length) +{ + const int inspected_syscall_count = sizeof(inspected_syscalls) / sizeof(inspected_syscalls[0]); + struct sock_filter *filter = calloc(MAX_BPF_OPS, sizeof(struct sock_filter)); + int filter_length = 0; + + // Test for system call re-entry: 4-byte (BPF_W) MARKER should be set as the 5th arg (0-indexed) + // do not intercepting 6-args syscalls, so it's OK + filter[filter_length++] = (struct sock_filter) BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[5])); + const int reenter_test_index = filter_length; + filter_length += 1; + // Load syscall number + filter[filter_length++] = (struct sock_filter) BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)); + // Decide what to do for this syscall number + // To be filled after exit instruction indices will be known + const int syscall_jumps_start = filter_length; + filter_length += inspected_syscall_count; + // if not inspected, then ALLOW + const int allow_exit_index = filter_length; + filter[filter_length++] = (struct sock_filter) BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW); + const int trap_exit_index = filter_length; + filter[filter_length++] = (struct sock_filter) BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_TRAP); + assert(filter_length < MAX_BPF_OPS); + + // Fill re-enter test: + filter[reenter_test_index] = (struct sock_filter) BPF_JUMP( + BPF_JMP | BPF_JEQ | BPF_K, + MARKER, + allow_exit_index - (reenter_test_index + 1), + 0); + // Fill syscall filter: + for (int i = 0; i < inspected_syscall_count; ++i) { + int ind = syscall_jumps_start + i; + filter[ind] = (struct sock_filter) BPF_JUMP( + BPF_JMP | BPF_JEQ | BPF_K, + inspected_syscalls[i], + trap_exit_index - (ind + 1), 0); + } + *length = filter_length; + return filter; +} + +static void initialize_signal_interceptor(void) +{ + int filter_length; + struct sock_filter *filter = create_filter(&filter_length); + + struct sock_fprog program = { filter_length, filter }; + + struct sigaction sig; + memset(&sig, 0, sizeof(sig)); + sig.sa_sigaction = handle_sigsys; + sig.sa_flags = SA_SIGINFO | SA_NODEFER; + sigaction(SIGSYS, &sig, NULL); + + prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); + syscall(SYS_seccomp, SECCOMP_SET_MODE_FILTER, 0, &program); + + free(filter); +} + +static void initialize_inputs(void) +{ + char env_name[32]; + for (input_id_count = 0; input_id_count < MAX_INPUTS; ++input_id_count) { + sprintf(env_name, "__SCM_INPUT_%d", input_id_count); + const char *file_name = getenv(env_name); + if (file_name == NULL) { + break; + } else { + fprintf(stderr, "Fetched input name #%d: %s\n", input_id_count, file_name); + input_ids[input_id_count] = get_file_id(file_name, 1); + } + } +} + +static void __attribute__((constructor)) constr(void) +{ + const char *timeout_str = getenv("__SCM_TIMEOUT"); + assert(timeout_str != NULL); + fork_child_timeout = atoi(timeout_str); + + initialize_inputs(); + initialize_signal_interceptor(); +} diff --git a/pmd-scm/src/test/java/net/sourceforge/pmd/scm/AbstractTestWithFiles.java b/pmd-scm/src/test/java/net/sourceforge/pmd/scm/AbstractTestWithFiles.java new file mode 100644 index 0000000..1cfc494 --- /dev/null +++ b/pmd-scm/src/test/java/net/sourceforge/pmd/scm/AbstractTestWithFiles.java @@ -0,0 +1,39 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.scm; + +import org.junit.After; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.List; + +public class AbstractTestWithFiles { + private final List filesToRemove = new ArrayList<>(); + + @After + public void tearDown() throws IOException { + for (Path file: filesToRemove) { + Files.delete(file); + } + filesToRemove.clear(); + } + + protected Path removeAfterTest(Path path) { + filesToRemove.add(path); + return path; + } + + + protected Path copyToTemporaryFile(InputStream stream, String suffix) throws IOException { + Path file = removeAfterTest(Files.createTempFile("pmd-test-", suffix)); + Files.copy(stream, file, StandardCopyOption.REPLACE_EXISTING); + return file; + } +} diff --git a/pmd-scm/src/test/java/net/sourceforge/pmd/scm/ForkServerAwareInvariantTest.java b/pmd-scm/src/test/java/net/sourceforge/pmd/scm/ForkServerAwareInvariantTest.java new file mode 100644 index 0000000..aac9164 --- /dev/null +++ b/pmd-scm/src/test/java/net/sourceforge/pmd/scm/ForkServerAwareInvariantTest.java @@ -0,0 +1,50 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.scm; + +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.SystemUtils; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +public class ForkServerAwareInvariantTest extends AbstractTestWithFiles { + private void performTesting(String command, String... invariantArgs) throws Exception { + Assume.assumeTrue(SystemUtils.IS_OS_LINUX); + + SCMConfiguration configuration = new SCMConfiguration(); + Path inputFile = copyToTemporaryFile(getClass().getResourceAsStream("test-input.txt"), ".in"); + Path outputFile = removeAfterTest(Files.createTempFile("pmd-test-", ".out")); + String cmdline = command + " " + outputFile.toString(); + + String[] baseArgs = { + "--language", "java", + "--input-file", inputFile.toString(), "--output-file", outputFile.toString(), + "--forkserver", "--command-line", cmdline, + "--strategy", "greedy" + }; + + String[] args = ArrayUtils.addAll(baseArgs, invariantArgs); + configuration.parse(args); + Assert.assertNull(configuration.getErrorString()); + SourceCodeMinimizer minimizer = new SourceCodeMinimizer(configuration); + minimizer.runMinimization(); + TestHelper.assertResultedSourceEquals(StandardCharsets.UTF_8, getClass().getResource("greedy-test-retained-testRemoval.txt"), outputFile); + } + + @Test + public void testExitCode() throws Exception { + performTesting("grep -q -F testRemoval", "--invariant", "exitcode", "--exact-return", "0"); + } + + @Test + public void testPrintedMessage() throws Exception { + performTesting("cat", "--invariant", "message", "--printed-message", "testRemoval"); + } +} diff --git a/pmd-scm/src/test/java/net/sourceforge/pmd/scm/GreedyStrategyTest.java b/pmd-scm/src/test/java/net/sourceforge/pmd/scm/GreedyStrategyTest.java index 9c39972..1c641bb 100644 --- a/pmd-scm/src/test/java/net/sourceforge/pmd/scm/GreedyStrategyTest.java +++ b/pmd-scm/src/test/java/net/sourceforge/pmd/scm/GreedyStrategyTest.java @@ -12,25 +12,19 @@ import java.util.List; import net.sourceforge.pmd.scm.invariants.AbstractExternalProcessInvariant; -import org.apache.commons.lang3.SystemUtils; import org.junit.Assert; import org.junit.Test; -public class GreedyStrategyTest { +public class GreedyStrategyTest extends AbstractTestWithFiles { private int getSpawnCount(SourceCodeMinimizer minimizer) { return ((AbstractExternalProcessInvariant) minimizer.getInvariant()).getSpawnCount(); } private void testRetention(String textToRetain, int maxSpawns, String inputFileName, String referenceFileName) throws Exception { SCMConfiguration configuration = new SCMConfiguration(); - Path inputFile = TestHelper.copyToTemporaryFile(getClass().getResourceAsStream(inputFileName), ".in"); - Path outputFile = Files.createTempFile("pmd-test-", ".out"); - String cmdline; - if (SystemUtils.IS_OS_WINDOWS) { - cmdline = "type " + outputFile.toString(); - } else { - cmdline = "cat " + outputFile.toString(); - } + Path inputFile = copyToTemporaryFile(getClass().getResourceAsStream(inputFileName), ".in"); + Path outputFile = removeAfterTest(Files.createTempFile("pmd-test-", ".out")); + String cmdline = TestHelper.printFileContentsCmd(outputFile.toString()); String[] args = { "--language", "java", "--input-file", inputFile.toString(), "--output-file", outputFile.toString(), "--invariant", "message", "--printed-message", textToRetain, "--command-line", cmdline, @@ -58,10 +52,10 @@ public void performanceTest() throws Exception { @Test public void multiFileJavaMinimization() throws Exception { SCMConfiguration configuration = new SCMConfiguration(); - Path input1 = TestHelper.copyToTemporaryFile(getClass().getResourceAsStream("greedy-multifile-1.java"), ".java"); - Path input2 = TestHelper.copyToTemporaryFile(getClass().getResourceAsStream("greedy-multifile-2.java"), ".java"); + Path input1 = copyToTemporaryFile(getClass().getResourceAsStream("greedy-multifile-1.java"), ".java"); + Path input2 = copyToTemporaryFile(getClass().getResourceAsStream("greedy-multifile-2.java"), ".java"); - Path fileList = Files.createTempFile("pmd-test-file-list", ".txt"); + Path fileList = removeAfterTest(Files.createTempFile("pmd-test-file-list", ".txt")); List fileNames = new ArrayList<>(); fileNames.add(input1.toString()); fileNames.add(input2.toString()); diff --git a/pmd-scm/src/test/java/net/sourceforge/pmd/scm/ScmConfigurationTest.java b/pmd-scm/src/test/java/net/sourceforge/pmd/scm/ScmConfigurationTest.java index 5b66e94..1b2b0da 100644 --- a/pmd-scm/src/test/java/net/sourceforge/pmd/scm/ScmConfigurationTest.java +++ b/pmd-scm/src/test/java/net/sourceforge/pmd/scm/ScmConfigurationTest.java @@ -79,6 +79,12 @@ public class ScmConfigurationTest { + " Compiler should exit with this specific exit value only (implies min == \n" + " max) \n" + " Default: -1\n" + + " --forkserver\n" + + " Use forkserver (Linux-only)\n" + + " Default: false\n" + + " --forkserver-child-timeout\n" + + " Timeout for a single fork server child process\n" + + " Default: 1\n" + " --max-return\n" + " Maximum exit code value (inclusive)\n" + " Default: 2147483647\n" @@ -90,6 +96,12 @@ public class ScmConfigurationTest { + " Options:\n" + " * --command-line\n" + " Command line for running a compiler on a source to be minimized\n" + + " --forkserver\n" + + " Use forkserver (Linux-only)\n" + + " Default: false\n" + + " --forkserver-child-timeout\n" + + " Timeout for a single fork server child process\n" + + " Default: 1\n" + " * --printed-message\n" + " Message that should be printed by the compiler\n" + " --printed-message-encoding\n" diff --git a/pmd-scm/src/test/java/net/sourceforge/pmd/scm/TestHelper.java b/pmd-scm/src/test/java/net/sourceforge/pmd/scm/TestHelper.java index 5e1e12b..2b9e853 100644 --- a/pmd-scm/src/test/java/net/sourceforge/pmd/scm/TestHelper.java +++ b/pmd-scm/src/test/java/net/sourceforge/pmd/scm/TestHelper.java @@ -5,14 +5,13 @@ package net.sourceforge.pmd.scm; import java.io.IOException; -import java.io.InputStream; import java.net.URL; import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.SystemUtils; import org.junit.Assert; public final class TestHelper { @@ -24,9 +23,11 @@ public static void assertResultedSourceEquals(Charset charset, URL expected, Pat Assert.assertEquals(expectedContents, actualContents); } - public static Path copyToTemporaryFile(InputStream stream, String suffix) throws IOException { - Path file = Files.createTempFile("pmd-test-", suffix); - Files.copy(stream, file, StandardCopyOption.REPLACE_EXISTING); - return file; + public static String printFileContentsCmd(String fileName) { + if (SystemUtils.IS_OS_WINDOWS) { + return "type " + fileName; + } else { + return "cat " + fileName; + } } } From d4dff842e30919bae3e94c0539a21be6fc4c7d31 Mon Sep 17 00:00:00 2001 From: Anatoly Trosinenko Date: Mon, 6 Jan 2020 10:47:51 +0300 Subject: [PATCH 2/5] Replace home-grown buffering with BufferedReader --- ...stractForkServerAwareProcessInvariant.java | 41 ++++--------------- 1 file changed, 7 insertions(+), 34 deletions(-) diff --git a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractForkServerAwareProcessInvariant.java b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractForkServerAwareProcessInvariant.java index 0f58e68..e635b86 100644 --- a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractForkServerAwareProcessInvariant.java +++ b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractForkServerAwareProcessInvariant.java @@ -7,6 +7,7 @@ import com.beust.jcommander.Parameter; import org.apache.commons.lang3.SystemUtils; +import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStream; @@ -116,8 +117,8 @@ protected abstract static class AbstractFactoryWithForkServer extends AbstractFa private Process forkServer; private OutputStream forkServerStdin; - private InputStreamReader forkServerStdout; - private InputStreamReader forkServerStderr; + private BufferedReader forkServerStdout; + private BufferedReader forkServerStderr; /** * Misc files to be cleaned up just before JVM termination. @@ -156,33 +157,6 @@ protected Charset getCharset() { return Charset.defaultCharset(); } - /** - * Reads a line of text until '\n', byte-by-byte. - * - * This is used to avoid deadlocks: more output may be read from the forkserver, - * but only after writing request to stdin. But first we should know that - * the forkserver is ready -- by reading its output streams. - * @param reader A stream to read from - * @param lineBuffer A scratch buffer - * @return A single line of output until '\n' (excluding) or EOF (if non-empty) - * or null, if got EOF as the very first character - * @throws IOException - */ - private String readString(InputStreamReader reader, char[] lineBuffer) throws IOException { - int lineLength; - for (lineLength = 0; lineLength < lineBuffer.length; ++lineLength) { - int ch = reader.read(); - if (ch == -1 && lineLength == 0) { - return null; - } - if (ch == -1 || ch == '\n') { - break; - } - lineBuffer[lineLength] = (char) ch; - } - return new String(lineBuffer, 0, lineLength); - } - /** * Reads all lines of output until the {@link #FORKSERVER_TO_SCM_MARKER}. * @@ -193,10 +167,9 @@ private String readString(InputStreamReader reader, char[] lineBuffer) throws IO * or null on unexpected EOF * @throws IOException */ - private String readUntilMarker(InputStreamReader reader, List stdoutContents) throws IOException { - char[] lineBuffer = new char[MAX_LINE_LENGTH]; + private String readUntilMarker(BufferedReader reader, List stdoutContents) throws IOException { while (true) { - String line = readString(reader, lineBuffer); + String line = reader.readLine(); if (line == null) { return null; } @@ -274,8 +247,8 @@ public void initialize(InvariantOperations ops) throws IOException { preloadedObject = compilePreloadedObject(); forkServer = createProcessBuilder().start(); forkServerStdin = forkServer.getOutputStream(); - forkServerStdout = new InputStreamReader(forkServer.getInputStream(), getCharset()); - forkServerStderr = new InputStreamReader(forkServer.getErrorStream(), getCharset()); + forkServerStdout = new BufferedReader(new InputStreamReader(forkServer.getInputStream(), getCharset())); + forkServerStderr = new BufferedReader(new InputStreamReader(forkServer.getErrorStream(), getCharset())); // Read startup messages List stdoutContents = new ArrayList<>(); From bc0c754f002af2405283a83bd7aa902bbee48d18 Mon Sep 17 00:00:00 2001 From: Anatoly Trosinenko Date: Mon, 6 Jan 2020 16:27:08 +0300 Subject: [PATCH 3/5] Show error if cannot setup seccomp filtering --- .../net/sourceforge/pmd/scm/invariants/forksrv-preload.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pmd-scm/src/main/resources/net/sourceforge/pmd/scm/invariants/forksrv-preload.c b/pmd-scm/src/main/resources/net/sourceforge/pmd/scm/invariants/forksrv-preload.c index e39a86c..378f0dc 100644 --- a/pmd-scm/src/main/resources/net/sourceforge/pmd/scm/invariants/forksrv-preload.c +++ b/pmd-scm/src/main/resources/net/sourceforge/pmd/scm/invariants/forksrv-preload.c @@ -278,7 +278,11 @@ static void initialize_signal_interceptor(void) sigaction(SIGSYS, &sig, NULL); prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); - syscall(SYS_seccomp, SECCOMP_SET_MODE_FILTER, 0, &program); + int ret = syscall(SYS_seccomp, SECCOMP_SET_MODE_FILTER, 0, &program); + if (ret != 0) { + perror("seccomp"); + abort(); + } free(filter); } From 63cb1fd89c29fe4f6bfb6e2ee1b793b2520af045 Mon Sep 17 00:00:00 2001 From: Anatoly Trosinenko Date: Mon, 6 Jan 2020 16:36:57 +0300 Subject: [PATCH 4/5] Intercept "legacy" open syscall as well --- .../sourceforge/pmd/scm/invariants/forksrv-preload.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pmd-scm/src/main/resources/net/sourceforge/pmd/scm/invariants/forksrv-preload.c b/pmd-scm/src/main/resources/net/sourceforge/pmd/scm/invariants/forksrv-preload.c index 378f0dc..15b769f 100644 --- a/pmd-scm/src/main/resources/net/sourceforge/pmd/scm/invariants/forksrv-preload.c +++ b/pmd-scm/src/main/resources/net/sourceforge/pmd/scm/invariants/forksrv-preload.c @@ -59,7 +59,11 @@ #define MAX_INPUTS 1024 // Inspecting 6-args syscalls is not supported -static int inspected_syscalls[] = { SYS_execve, SYS_execveat, SYS_fork, SYS_vfork, SYS_clone, SYS_openat, SYS_stat }; +static int inspected_syscalls[] = { + SYS_open, SYS_openat, SYS_stat, + SYS_execve, SYS_execveat, + SYS_fork, SYS_vfork, SYS_clone, +}; struct file_id { dev_t dev; @@ -191,6 +195,12 @@ static void handle_sigsys(int num, siginfo_t *si, void *arg) greg_t *gregs = ctx->uc_mcontext.gregs; int sc_num = gregs[SC_NUM_REG]; switch (sc_num) { + case SYS_open: + if (is_input_name((const char *) gregs[ARG_REG_1])) { + fprintf(stderr, "Opening %s, starting fork server.\n", (const char *) gregs[ARG_REG_1]); + start_forkserver(); + } + break; case SYS_openat: if (is_input_name((const char *) gregs[ARG_REG_2])) { fprintf(stderr, "Opening %s, starting fork server.\n", (const char *) gregs[ARG_REG_2]); From 992a9a590e705bacf7c3812a8b15a05f616bf8d6 Mon Sep 17 00:00:00 2001 From: Anatoly Trosinenko Date: Sun, 12 Jan 2020 13:48:06 +0300 Subject: [PATCH 5/5] Misc fixes --- .../AbstractForkServerAwareProcessInvariant.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractForkServerAwareProcessInvariant.java b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractForkServerAwareProcessInvariant.java index e635b86..6bc0563 100644 --- a/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractForkServerAwareProcessInvariant.java +++ b/pmd-scm/src/main/java/net/sourceforge/pmd/scm/invariants/AbstractForkServerAwareProcessInvariant.java @@ -62,6 +62,15 @@ public abstract class AbstractForkServerAwareProcessInvariant extends AbstractEx */ private final static String PRELOAD_VAR = "LD_PRELOAD"; + /** + * Environment variable name: when set to non-empty value, instructs + * the dynamic linker to resolve all symbols at startup. + * + * Just like in AFL, used to perform as much work in the fork parent + * as possible and not re-do this in every child. + */ + private final static String BIND_NOW_VAR = "LD_BIND_NOW"; + /** * Environment variable name: timeout (in seconds) for forkserver child. */ @@ -88,11 +97,6 @@ public abstract class AbstractForkServerAwareProcessInvariant extends AbstractEx */ private final static String FORKSERVER_INIT_REPLY = "INIT"; - /** - * The size of buffer for a single output line read by this class. - */ - private final static int MAX_LINE_LENGTH = 65536; - protected abstract static class AbstractConfigurationWithForkServer extends AbstractConfiguration { @Parameter(names = "--forkserver", description = "Use forkserver (Linux-only)") private boolean useForkserver; @@ -224,6 +228,7 @@ protected ProcessBuilder createProcessBuilder() { pb.command("/bin/sh", "-c", setEnvString + " " + getCompilerCommandLine()); Map environment = pb.environment(); environment.put(SCM_TIMEOUT_VAR, Integer.toString(timeoutSec)); + environment.put(BIND_NOW_VAR, "1"); List scratchFiles = ops.getScratchFileNames(); for (int i = 0; i < scratchFiles.size(); ++i) { environment.put(String.format(SCM_INPUT_VAR_FORMAT, i), scratchFiles.get(i).toString());