Dynamic scanning model

README.md

@@ -6,6 +6,10 @@ A shared Java malware scanner capable of static and dynamic analysis.
 
 Concoction can be used either as a Java library or as a command line application.
 
+### Scan models
+
+The format of scan models is described [here](docs/ModelFormat.md).
+
 ### As a library
 
 First add Concoction as a library to your project. It is available on maven central:

concoction-lib/src/main/java/info/mmpa/concoction/Concoction.java

@@ -5,9 +5,9 @@
 import info.mmpa.concoction.input.model.InvalidModelException;
 import info.mmpa.concoction.input.model.ModelBuilder;
 import info.mmpa.concoction.output.Results;
+import info.mmpa.concoction.scan.dynamic.CoverageEntryPointSupplier;
 import info.mmpa.concoction.scan.dynamic.DynamicScanException;
 import info.mmpa.concoction.scan.dynamic.DynamicScanner;
-import info.mmpa.concoction.scan.dynamic.CoverageEntryPointSupplier;
 import info.mmpa.concoction.scan.dynamic.EntryPointDiscovery;
 import info.mmpa.concoction.scan.insn.InstructionScanner;
 import info.mmpa.concoction.scan.model.ScanModel;
@@ -30,6 +30,7 @@ public class Concoction {
 	private final Map<Path, ScanModel> scanModels = new HashMap<>();
 	private ArchiveLoadContext supportingPathLoadContext = ArchiveLoadContext.RANDOM_ACCESS_JAR;
 	private int inputDepth = 3;
+	private boolean dynamicScanning;
 
 	private Concoction() {
 	}
@@ -60,6 +61,18 @@ public Concoction withMaxInputDirectoryDepth(int inputDepth) {
 		return this;
 	}
 
+	/**
+	 * Activates dynamic scanning capabilities.
+	 * By default, they are disabled,
+	 *
+	 * @return Self.
+	 */
+	@Nonnull
+	public Concoction withDynamicScanning() {
+		this.dynamicScanning = true;
+		return this;
+	}
+
 	/**
 	 * Sets supporting path/input archive loading context.
 	 *
@@ -313,9 +326,9 @@ public NavigableMap<Path, Results> scan() throws DynamicScanException {
 		}
 
 		// Then dynamic scanning
-		List<ScanModel> dynamicModels = scanModels.values().stream()
+		List<ScanModel> dynamicModels = dynamicScanning ? scanModels.values().stream()
 				.filter(ScanModel::hasDynamicModel)
-				.collect(Collectors.toList());
+				.collect(Collectors.toList()) : Collections.emptyList();
 		if (!insnModels.isEmpty()) {
 			// TODO: Point these interfaces to proper implementations
 			//  - This will currently scan nothing

concoction-lib/src/main/java/info/mmpa/concoction/input/model/ApplicationModel.java

@@ -6,7 +6,6 @@
 import java.util.stream.Stream;
 
 import static java.util.stream.Stream.concat;
-import static java.util.stream.Stream.of;
 
 /**
  * Full application model, comprised of one or more units.

concoction-lib/src/main/java/info/mmpa/concoction/input/model/impl/BasicModelBuilder.java

@@ -28,16 +28,16 @@ public class BasicModelBuilder implements ModelBuilder {
 	@Nonnull
 	@Override
 	public ModelBuilder addSource(@Nonnull ArchiveLoadContext context, @Nonnull Path path) throws IOException {
-		ClassesAndFiles input = pathReader.from(context,path);
+		ClassesAndFiles input = pathReader.from(context, path);
 		String identifier = path.getFileName().toString();
 		sources.add(new BasicModelSource(identifier, input.getClasses(), input.getFiles()));
 		return this;
 	}
 
 	@Nonnull
 	@Override
-	public ModelBuilder addSource(@Nonnull ArchiveLoadContext context,@Nonnull String identifier, @Nonnull byte[] raw) throws IOException {
-		ClassesAndFiles input = rawReader.from(context,raw);
+	public ModelBuilder addSource(@Nonnull ArchiveLoadContext context, @Nonnull String identifier, @Nonnull byte[] raw) throws IOException {
+		ClassesAndFiles input = rawReader.from(context, raw);
 		sources.add(new BasicModelSource(identifier, input.getClasses(), input.getFiles()));
 		return this;
 	}

concoction-lib/src/main/java/info/mmpa/concoction/output/SusLevel.java

@@ -4,25 +4,25 @@
  * Levels of suspicions for report items.
  */
 public enum SusLevel {
-    /**
-     * Almost guaranteed to be malicious.
-     */
-    MAXIMUM,
-    /**
-     * Very likely to be malicious, but in some cases it may just be benign shoddy code.
-     */
-    STRONG,
-    /**
-     * Typically these are strictly context based. For instance, deleting a file is usually not a concern.
-     * Deleting core operating system files is a concern though.
-     */
-    MEDIUM,
-    /**
-     * Unlikely to be malicious in most circumstances.
-     */
-    WEAK,
-    /**
-     * Incredibly unlikely to be malicious in any circumstance.
-     */
-    NOTHING_BURGER
+	/**
+	 * Almost guaranteed to be malicious.
+	 */
+	MAXIMUM,
+	/**
+	 * Very likely to be malicious, but in some cases it may just be benign shoddy code.
+	 */
+	STRONG,
+	/**
+	 * Typically these are strictly context based. For instance, deleting a file is usually not a concern.
+	 * Deleting core operating system files is a concern though.
+	 */
+	MEDIUM,
+	/**
+	 * Unlikely to be malicious in most circumstances.
+	 */
+	WEAK,
+	/**
+	 * Incredibly unlikely to be malicious in any circumstance.
+	 */
+	NOTHING_BURGER
 }

concoction-lib/src/main/java/info/mmpa/concoction/scan/dynamic/DynamicScanner.java

@@ -2,6 +2,7 @@
 
 import dev.xdark.ssvm.execution.VMException;
 import info.mmpa.concoction.input.model.ApplicationModel;
+import info.mmpa.concoction.input.model.path.SourcePathElement;
 import info.mmpa.concoction.output.Results;
 import info.mmpa.concoction.output.ResultsSink;
 import info.mmpa.concoction.scan.model.ScanModel;
@@ -50,7 +51,7 @@ public DynamicScanner(@Nonnull EntryPointDiscovery entryPointDiscovery,
 	@SuppressWarnings("UnnecessaryLocalVariable")
 	public Results accept(@Nonnull ApplicationModel model) throws DynamicScanException {
 		ResultsSink sink = new ResultsSink();
-		SsvmContext context = new SsvmContext(model, scanModels);
+		SsvmContext context = new SsvmContext(model, sink, new SourcePathElement(model.primarySource()), scanModels);
 
 		// Visit initial entry points
 		List<EntryPoint> initialEntryPoints = entryPointDiscovery.createEntryPoints(model, context);

concoction-lib/src/main/java/info/mmpa/concoction/scan/dynamic/EntryPoint.java

@@ -140,7 +140,7 @@ public String toString() {
 	@Override
 	public int compareTo(@Nonnull EntryPoint o) {
 		int cmp = className.compareTo(o.className);
-		if (cmp != 0){
+		if (cmp != 0) {
 			cmp = methodName.compareTo(o.methodName);
 			if (cmp != 0) cmp = methodDescriptor.compareTo(o.methodDescriptor);
 		}

concoction-lib/src/main/java/info/mmpa/concoction/scan/dynamic/SsvmContext.java

@@ -4,12 +4,14 @@
 import dev.xdark.ssvm.api.MethodInvoker;
 import dev.xdark.ssvm.api.VMInterface;
 import dev.xdark.ssvm.classloading.SupplyingClassLoaderInstaller;
-import dev.xdark.ssvm.filesystem.FileManager;
 import dev.xdark.ssvm.invoke.InvocationUtil;
 import dev.xdark.ssvm.mirror.type.InstanceClass;
 import dev.xdark.ssvm.thread.OSThread;
 import info.mmpa.concoction.input.model.ApplicationModel;
 import info.mmpa.concoction.input.model.ModelSource;
+import info.mmpa.concoction.input.model.path.SourcePathElement;
+import info.mmpa.concoction.output.DetectionArchetype;
+import info.mmpa.concoction.output.ResultsSink;
 import info.mmpa.concoction.scan.model.ScanModel;
 import org.objectweb.asm.Opcodes;
 
@@ -36,22 +38,20 @@ public class SsvmContext {
 	/**
 	 * @param model
 	 * 		Model to pass to the VM.
+	 * @param sink
+	 * 		Sink to feed match results into.
+	 * @param sourcePath
+	 * 		Current method path to the containing input source.
+	 * 		SSVM will pass along the rest of the path details.
 	 * @param scanModel
 	 * 		List of detection models to scan for.
 	 */
-	public SsvmContext(@Nonnull ApplicationModel model, @Nonnull Collection<ScanModel> scanModel) {
-		// TODO: Supply dynamic models to match against
-		//  - Tweak VM initialization to track information that is needed for the models to match against.
-		//    - Method enter/exit listeners
-		//    - Method instruction interceptors for some edge cases perhaps?
-
+	public SsvmContext(@Nonnull ApplicationModel model,
+					   @Nonnull ResultsSink sink,
+					   @Nonnull SourcePathElement sourcePath,
+					   @Nonnull Collection<ScanModel> scanModel) {
 		// Create and initialize the VM.
-		VirtualMachine vm = new VirtualMachine() {
-			@Override
-			protected FileManager createFileManager() {
-				return new CustomFileManager();
-			}
-		};
+		VirtualMachine vm = new VirtualMachineExt() ;
 		vm.getProperties().put("java.class.path", ""); // Hide class path of concoction from the VM
 		vm.bootstrap();
 
@@ -60,14 +60,23 @@ protected FileManager createFileManager() {
 		vmi.registerMethodEnterListener(ctx -> {
 			OSThread thread = vm.currentJavaThread().getOsThread();
 			Stack<CallStackFrame> stack = threadFrameMap.computeIfAbsent(thread, t -> new Stack<>());
-			stack.push(new CallStackFrame(ctx));
+			CallStackFrame frame = new CallStackFrame(ctx);
+			stack.push(frame);
+			for (ScanModel modelEntry : scanModel) {
+				DetectionArchetype archetype = modelEntry.getDetectionArchetype();
+				modelEntry.getDynamicMatchingModel().matchOnEnter(sink, archetype, sourcePath, frame);
+			}
 		});
 		vmi.registerMethodExitListener(ctx -> {
 			OSThread thread = vm.currentJavaThread().getOsThread();
 			Stack<CallStackFrame> stack = threadFrameMap.get(thread);
 			if (stack == null || stack.isEmpty())
 				throw new IllegalArgumentException("Cannot pop call stack frame from thread with no prior stack history");
-			stack.pop();
+			CallStackFrame frame = stack.pop();
+			for (ScanModel modelEntry : scanModel) {
+				DetectionArchetype archetype = modelEntry.getDetectionArchetype();
+				modelEntry.getDynamicMatchingModel().matchOnExit(sink, archetype, sourcePath, frame);
+			}
 		});
 
 		// Some patches to circumvent bugs arising from VM implementation changes in later versions
@@ -78,7 +87,7 @@ protected FileManager createFileManager() {
 
 			// SSVM manages its own memory, and this conflicts with it. Stubbing it out keeps everyone happy.
 			InstanceClass bits = (InstanceClass) vm.findBootstrapClass("java/nio/Bits");
-			vmi.setInvoker(bits.getMethod("reserveMemory", "(JJ)V"), MethodInvoker.noop());
+			if (bits != null) vmi.setInvoker(bits.getMethod("reserveMemory", "(JJ)V"), MethodInvoker.noop());
 		}
 
 		// Store VM instance.
@@ -131,7 +140,7 @@ public InvocationUtil getInvoker() {
 				deencapsulate.invoke(null, SsvmContext.class);
 			}
 		} catch (Exception ex) {
-			throw new IllegalStateException(ex);
+			throw new IllegalStateException("Failed to unlock reflection access", ex);
 		}
 	}
 }

concoction-lib/src/main/java/info/mmpa/concoction/scan/dynamic/VirtualMachineExt.java

@@ -0,0 +1,34 @@
+package info.mmpa.concoction.scan.dynamic;
+
+import dev.xdark.ssvm.VirtualMachine;
+import dev.xdark.ssvm.filesystem.FileManager;
+import dev.xdark.ssvm.mirror.type.JavaClass;
+
+import javax.annotation.Nonnull;
+
+/**
+ * Basic extension of the VM class, tweaking some default manager implementations
+ * and providing access to additional data.
+ */
+public class VirtualMachineExt extends VirtualMachine {
+	private JavaClass charSequence;
+
+	@Override
+	public void bootstrap() {
+		super.bootstrap();
+		charSequence = findBootstrapClass("java/lang/CharSequence");
+	}
+
+	@Override
+	protected FileManager createFileManager() {
+		return new CustomFileManager();
+	}
+
+	/**
+	 * @return {@link CharSequence} type.
+	 */
+	@Nonnull
+	public JavaClass getCharSequence() {
+		return charSequence;
+	}
+}

concoction-lib/src/main/java/info/mmpa/concoction/scan/insn/InstructionScanner.java

@@ -50,7 +50,7 @@ public Results accept(@Nonnull ApplicationModel model) {
 			try {
 				ClassNode classNode = AsmUtil.node(classEntry.getValue());
 
-				// TODO #4: Class structure matchers
+				// TODO #4: Class structure matchers, then refactor this class name to 'StaticScanner'
 
 				// Run per-method matchers (instruction matching models)
 				for (MethodNode methodNode : classNode.methods) {

concoction-lib/src/main/java/info/mmpa/concoction/scan/model/MultiMatchMode.java

@@ -0,0 +1,65 @@
+package info.mmpa.concoction.scan.model;
+
+import info.mmpa.concoction.scan.model.dynamic.entry.*;
+import info.mmpa.concoction.scan.model.insn.entry.*;
+
+import javax.annotation.Nonnull;
+import java.util.List;
+
+import static java.util.Collections.unmodifiableList;
+
+/**
+ * Mode for subtypes of multi-instruction matchers.
+ */
+public enum MultiMatchMode {
+	/**
+	 * @see AllMultiInstruction
+	 */
+	ALL,
+	/**
+	 * @see AnyMultiInstruction
+	 */
+	ANY,
+	/**
+	 * @see NoneMultiInstruction
+	 */
+	NONE;
+
+	/**
+	 * @param entries
+	 * 		Entries to wrap into a multi-instruction matcher subtype.
+	 *
+	 * @return Instance of multi-instruction matcher subtype.
+	 */
+	@Nonnull
+	public MultiInstruction createMultiInsn(@Nonnull List<InstructionMatchEntry> entries) {
+		switch (this) {
+			case NONE:
+				return new NoneMultiInstruction(unmodifiableList(entries));
+			case ALL:
+				return new AllMultiInstruction(unmodifiableList(entries));
+			case ANY:
+			default:
+				return new AnyMultiInstruction(unmodifiableList(entries));
+		}
+	}
+
+	/**
+	 * @param entries
+	 * 		Entries to wrap into a multi-dynamic matcher subtype.
+	 *
+	 * @return Instance of multi-dynamic matcher subtype.
+	 */
+	@Nonnull
+	public MultiDynamic createMultiDynamic(@Nonnull List<DynamicMatchEntry> entries) {
+		switch (this) {
+			case NONE:
+				return new NoneMultiDynamic(unmodifiableList(entries));
+			case ALL:
+				return new AllMultiDynamic(unmodifiableList(entries));
+			case ANY:
+			default:
+				return new AnyMultiDynamic(unmodifiableList(entries));
+		}
+	}
+}