Skip to content

Commit

Permalink
WIP: Support additional file handlers, fixes & cleanup
Browse files Browse the repository at this point in the history 
- Fix file reads being empty on later handlers
- Remove MDC stuff, cleanup some logging stuff
- Update dependencies
  • Loading branch information
@gschneider-r7
gschneider-r7 committed Jun 6, 2019
1 parent bd88d14 commit 9481c41
Show file tree
Hide file tree
Showing 3 changed files with 88 additions and 91 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
<commons.codec.version>1.12</commons.codec.version>
<commons.compress.version>1.18</commons.compress.version>
<commons.io.version>2.6</commons.io.version>
<jackson.version>2.9.8</jackson.version>
<jackson.version>[2.9.9,2.10.0)</jackson.version>
<slf4j.version>1.7.25</slf4j.version>
<!-- test dependencies -->
<hamcrest.version>1.3</hamcrest.version>
Expand Down
15 changes: 4 additions & 11 deletions src/main/java/com/rapid7/container/analyzer/docker/fingerprinter/OsReleaseFingerprinter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,8 @@
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;

public class OsReleaseFingerprinter implements LayerFileHandler {

private static final Logger LOGGER = LoggerFactory.getLogger(OsReleaseFingerprinter.class);
private static final Pattern FILENAME_PATTERN = Pattern.compile("(?i:^((?:\\.)?(?:/)?etc/.*-release|(?:\\.)?(?:/)?usr/lib/os-release)$)");
private Fingerprinter osReleaseParser;
Expand All @@ -30,16 +28,11 @@ public void handle(String name, TarArchiveEntry entry, InputStream contents, Ima
if (layer.getOperatingSystem() == null || name.endsWith("/os-release")) {
OperatingSystem os = osReleaseParser.parse(contents, name, convert(configuration.getArchitecture()));
if (os != null) {
MDC.put("operating_system", os.toString());
try {
LOGGER.info("Operating system detected on layer.");
layer.setOperatingSystem(os);
LOGGER.debug("Operating system detected on layer.");
layer.setOperatingSystem(os);

// given all packages detected on the layer this OS reference
layer.getPackages().forEach(pkg -> pkg.setOperatingSystem(os));
} finally {
MDC.remove("operating_system");
}
// given all packages detected on the layer this OS reference
layer.getPackages().forEach(pkg -> pkg.setOperatingSystem(os));
}
}
}
Expand Down
162 changes: 83 additions & 79 deletions src/main/java/com/rapid7/container/analyzer/docker/service/DockerImageAnalyzerService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@
import com.rapid7.container.analyzer.docker.util.InstantParser;
import com.rapid7.container.analyzer.docker.util.InstantParserModule;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
Expand All @@ -45,23 +47,23 @@
import java.nio.file.Files;
import java.nio.file.InvalidPathException;
import java.nio.file.Paths;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipException;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.utils.IOUtils;
import org.apache.commons.io.FilenameUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import static org.apache.commons.io.FileUtils.deleteQuietly;
import static org.slf4j.helpers.MessageFormatter.arrayFormat;
import static org.slf4j.helpers.MessageFormatter.format;

public class DockerImageAnalyzerService {

private static final Logger LOGGER = LoggerFactory.getLogger(DockerImageAnalyzerService.class);
private static final String WHITEOUT_AUFS_PREFIX = ".wh.";
private ObjectMapper objectMapper;
Expand All @@ -81,12 +83,10 @@ public DockerImageAnalyzerService(String rpmDockerImage) {
layerHandlers.add(new ApkgFingerprinter(new ApkgParser()));
layerHandlers.add(new PacmanFingerprinter(new PacmanPackageParser()));
layerHandlers.add(new FileFingerprinter());

initialize();
}

private void initialize() {
layerHandlers.forEach(handler -> LOGGER.debug("Handler {}", handler.getClass().getSimpleName()));
public void addFileHandler(LayerFileHandler handler) {
layerHandlers.add(Objects.requireNonNull(handler));
}

public ImageId getId(File imageTar) throws IOException {
Expand Down Expand Up @@ -157,7 +157,6 @@ public <M extends Manifest> Image analyze(File outputDirectory, ImageId id, Dige
int emptyLayerIdSuffix = 0;
Layer previousLayer = null;
for (HistoryJson layerHistory : layerHistories) {

Instant start = Instant.now();
boolean empty = layerHistory.isEmpty();
LayerId layerId = null;
Expand All @@ -166,79 +165,55 @@ public <M extends Manifest> Image analyze(File outputDirectory, ImageId id, Dige
else
layerId = new LayerId(id.getString() + "_empty_" + emptyLayerIdSuffix++);

MDC.put("layer_id", layerId.getString());
LOGGER.info("Processing layer.");
try {
Layer layer = new Layer(layerId);
layer.setCommand(layerHistory.getCommand());
layer.setComment(layerHistory.getComment());
layer.setAuthor(layerHistory.getAuthor());
layer.setEmpty(empty);

// Images built with a tool called buildkit can have history entries with empty created date, so fall back to previous layer or epoch
if (layer.getCreated() != null)
layer.setCreated(InstantParser.parse(layerHistory.getCreated()));
else if (previousLayer != null && previousLayer.getCreated() != null)
layer.setCreated(previousLayer.getCreated());
else
layer.setCreated(Instant.EPOCH);

// if the layer is non-empty, process it
if (!empty) {

// locate the layer to process
LayerId layerRetrievalId = layerBlobIds.get(layerIndex);
File layerTar = layerSupplier.getLayer(layerRetrievalId);
layer.setSize(layerTar.length());

MDC.put("layer_blob_id", layerRetrievalId.getString());
LOGGER.debug("Processing layer {}.", layerId);

Layer layer = new Layer(layerId);
layer.setCommand(layerHistory.getCommand());
layer.setComment(layerHistory.getComment());
layer.setAuthor(layerHistory.getAuthor());
layer.setEmpty(empty);

// Images built with a tool called buildkit can have history entries with empty created date, so fall back to previous layer or epoch
if (layer.getCreated() != null)
layer.setCreated(InstantParser.parse(layerHistory.getCreated()));
else if (previousLayer != null && previousLayer.getCreated() != null)
layer.setCreated(previousLayer.getCreated());
else
layer.setCreated(Instant.EPOCH);

// if the layer is non-empty, process it
if (!empty) {

// locate the layer to process
LayerId layerRetrievalId = layerBlobIds.get(layerIndex);
File layerTar = layerSupplier.getLayer(layerRetrievalId);
layer.setSize(layerTar.length());
LOGGER.debug("Processing layer tar.");

// extract layer
processLayer(image, configuration, layer, layerTar);

// attach additional layer information
File layerInfoFile = new File(layerTar.getParentFile(), "json");
if (layerInfoFile.exists()) {
try {
LOGGER.info("Processing layer tar.");

// extract layer
processLayer(image, configuration, layer, layerTar);

// attach additional layer information
File layerInfoFile = new File(layerTar.getParentFile(), "json");
if (layerInfoFile.exists()) {
try {
LayerJson layerInfo = parseLayerConfiguration(layerInfoFile);
layer.setCreated(layerInfo.getCreated());
layer.setParentId(layerInfo.getParentId());
} catch (Exception exception) {
LOGGER.warn("Failed to parse layer configuration json. Skipping setting of created data and parent identifier.", exception);
}
} else if (previousLayer != null) {
layer.setParentId(previousLayer.getId());
}
} finally {
MDC.remove("layer_blob_id");
LayerJson layerInfo = parseLayerConfiguration(layerInfoFile);
layer.setCreated(layerInfo.getCreated());
layer.setParentId(layerInfo.getParentId());
} catch (Exception exception) {
LOGGER.warn("Failed to parse layer configuration json. Skipping setting of created data and parent identifier.", exception);
}

layerIndex++;
previousLayer = layer;
} else if (previousLayer != null) {
layer.setParentId(previousLayer.getId());
}

image.addLayer(layer);

if (layer.getOperatingSystem() != null)
MDC.put("operating_system", layer.getOperatingSystem().toString());

MDC.put("packages", Integer.toString(layer.getPackages().size()));
try {
Duration duration = Duration.between(start, Instant.now());
MDC.put("payload_process_time_ms", String.valueOf(duration.toMillis()));
LOGGER.info("Layer processed.");
} finally {
MDC.remove("operating_system");
MDC.remove("packages");
MDC.remove("payload_process_time_ms");
}
} finally {
MDC.remove("layer_id");
layerIndex++;
previousLayer = layer;
} else if (previousLayer != null) {
layer.setParentId(previousLayer.getId());
}

image.addLayer(layer);
}

// post-process image handlers
Expand Down Expand Up @@ -322,20 +297,49 @@ public void untar(File tar, File destination) throws FileNotFoundException, IOEx

private void processLayer(Image image, Configuration configuration, Layer layer, File tar) throws FileNotFoundException, IOException {

try (TarArchiveInputStream tarIn = new TarArchiveInputStream(new BufferedInputStream(new FileInputStream(tar)))) {
TarArchiveEntry entry = null;
while ((entry = tarIn.getNextTarEntry()) != null) {
String name = entry.getName();
try (TarArchiveInputStream tarIn = new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(tar), 65536))) {
processLayerTar(image, configuration, layer, tar, tarIn);
} catch (ZipException ze) {
try (TarArchiveInputStream tarIn = new TarArchiveInputStream(new BufferedInputStream(new FileInputStream(tar), 65536))) {
processLayerTar(image, configuration, layer, tar, tarIn);
}
}
}

private void processLayerTar(Image image, Configuration configuration, Layer layer, File tar, TarArchiveInputStream tarIn) throws IOException {
TarArchiveEntry entry = null;
while ((entry = tarIn.getNextTarEntry()) != null) {
String name = entry.getName();
if (LOGGER.isTraceEnabled())
LOGGER.trace(arrayFormat("[Image: {}] Processing {} {}.", new Object[]{tar.getName(), entry.isDirectory() ? "directory" : "file", name}).getMessage());

int size;
if (entry.getSize() > 1_048_576)
size = 1_048_576;
else
size = (int) entry.getSize();

try (ConvertibleOutputStream out = new ConvertibleOutputStream(size)) {
IOUtils.copy(tarIn, out);
InputStream contents = out.toInputStream();
for (LayerFileHandler handler : layerHandlers) {
handler.handle(name, entry, new NonClosingInputStream(tarIn), image, configuration, layer);
handler.handle(name, entry, contents, image, configuration, layer);
contents.reset();
}
}
}
}

private static class ConvertibleOutputStream extends ByteArrayOutputStream {
public ConvertibleOutputStream(int size) {
super(size);
}
// Creates InputStream without actually copying the buffer and using up memory for that.
public InputStream toInputStream() {
return new ByteArrayInputStream(buf, 0, count);
}
}

private static class NonClosingInputStream extends FilterInputStream {
NonClosingInputStream(InputStream in) {
super(in);
Expand Down

0 comments on commit 9481c41

Please sign in to comment.