Add support for reading artifacts from global bundle pools

Since a while P2 offers a utility method to get access to the shared
bundlepools that allow to reuse artifacts already downloaded.

This now adds a first implementation of ArtifactDownloadProvider that
make use of this shared pools to provide artifacts from the local pools
instead of download them through remote sites.

p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/repository/DefaultSimpleArtifactRepositoryFactory.java

@@ -0,0 +1,36 @@
+/*******************************************************************************
+ * Copyright (c) 2024 Christoph Läubrich and others.
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *    Christoph Läubrich - initial API and implementation
+ *******************************************************************************/
+package org.eclipse.tycho.p2maven.repository;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+
+import org.eclipse.equinox.internal.p2.artifact.repository.simple.SimpleArtifactRepositoryFactory;
+import org.eclipse.equinox.p2.core.IProvisioningAgent;
+
+@Named
+public class DefaultSimpleArtifactRepositoryFactory extends SimpleArtifactRepositoryFactory {
+
+	private IProvisioningAgent agent;
+
+	@Inject
+	public DefaultSimpleArtifactRepositoryFactory(IProvisioningAgent agent) {
+		this.agent = agent;
+	}
+
+	@Override
+	protected IProvisioningAgent getAgent() {
+		return agent;
+	}
+
+}

p2-maven-plugin/src/main/java/org/eclipse/tycho/p2maven/transport/BundlePoolArtifactDownloadProvider.java

@@ -0,0 +1,205 @@
+/*******************************************************************************
+ * Copyright (c) 2024 Christoph Läubrich and others.
+ * This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License 2.0
+ * which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *    Christoph Läubrich - initial API and implementation
+ *******************************************************************************/
+package org.eclipse.tycho.p2maven.transport;
+
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.file.Path;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Stream;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+
+import org.apache.commons.io.FileUtils;
+import org.codehaus.plexus.logging.Logger;
+import org.eclipse.core.runtime.IStatus;
+import org.eclipse.core.runtime.Status;
+import org.eclipse.equinox.internal.p2.artifact.repository.simple.SimpleArtifactRepositoryFactory;
+import org.eclipse.equinox.internal.p2.repository.DownloadStatus;
+import org.eclipse.equinox.internal.p2.repository.helpers.ChecksumHelper;
+import org.eclipse.equinox.internal.p2.repository.helpers.RepositoryHelper;
+import org.eclipse.equinox.p2.core.ProvisionException;
+import org.eclipse.equinox.p2.repository.artifact.IArtifactDescriptor;
+import org.eclipse.equinox.p2.repository.artifact.IArtifactRepository;
+import org.eclipse.tycho.TychoConstants;
+import org.eclipse.tycho.helper.MavenPropertyHelper;
+import org.eclipse.tycho.transport.ArtifactDownloadProvider;
+
+/**
+ * Provides artifacts already available from the users bundle pools
+ */
+@Named
+public class BundlePoolArtifactDownloadProvider implements ArtifactDownloadProvider {
+
+	private SimpleArtifactRepositoryFactory artifactRepositoryFactory;
+	private Map<Path, IArtifactRepository> repositoryMap = new ConcurrentHashMap<>();
+	private TransportCacheConfig cacheConfig;
+	private Logger logger;
+	private boolean useSharedPools;
+	private boolean useWorkspacePools;
+	private int priority;
+
+	@Inject
+	public BundlePoolArtifactDownloadProvider(SimpleArtifactRepositoryFactory artifactRepositoryFactory,
+			TransportCacheConfig cacheConfig, Logger logger, MavenPropertyHelper propertyHelper) {
+		this.artifactRepositoryFactory = artifactRepositoryFactory;
+		this.cacheConfig = cacheConfig;
+		this.logger = logger;
+		useSharedPools = propertyHelper.getGlobalBooleanProperty("tycho.p2.transport.bundlepools.shared", true);
+		useWorkspacePools = propertyHelper.getGlobalBooleanProperty("tycho.p2.transport.bundlepools.workspace", true);
+		priority = propertyHelper.getGlobalIntProperty("tycho.p2.transport.bundlepools.priority", 100);
+
+	}
+
+	@Override
+	public IStatus downloadArtifact(URI source, OutputStream target, IArtifactDescriptor originalDescriptor) {
+		return pools().parallel().map(path -> {
+			IArtifactRepository repository = getRepository(path);
+			if (repository != null) {
+				IArtifactDescriptor[] descriptors = repository
+						.getArtifactDescriptors(originalDescriptor.getArtifactKey());
+				for (IArtifactDescriptor descriptor : descriptors) {
+					if (isMatching(descriptor, originalDescriptor)) {
+						return new RepositoryCandidate(path, repository, descriptor);
+					}
+				}
+			}
+			return null;
+		}).filter(Objects::nonNull).findAny().map(candidate -> {
+			IArtifactRepository repository = candidate.repository();
+			Path path = candidate.path();
+			IArtifactDescriptor descriptor = candidate.descriptor();
+			if (cacheConfig.isInteractive()) {
+				logger.info("Reading from " + path + ": " + descriptor.getArtifactKey());
+			}
+			IStatus status = repository.getArtifact(descriptor, target, null);
+			if (status instanceof DownloadStatus dls) {
+				if (cacheConfig.isInteractive()) {
+					logger.info(String.format("Read from %s (%s): %s (%s at %s/s)", repository.getName(), path,
+							descriptor.getArtifactKey().toExternalForm(),
+							FileUtils.byteCountToDisplaySize(dls.getFileSize()),
+							FileUtils.byteCountToDisplaySize(dls.getTransferRate())));
+				}
+				// we don't want to let the mirror getting a high rating just because we have
+				// actually downloaded this from the disk and not the supplied URL
+				dls.setTransferRate(0);
+			}
+			return status;
+		}).orElse(Status.CANCEL_STATUS);
+	}
+
+	/**
+	 * Test if two descriptors have at least one matching hashsum in which case we
+	 * assume they are describing the same artifact and not only have the same
+	 * version/id, this should not happen usually, but as we use global pools here
+	 * it is better to be safe than sorry.
+	 * 
+	 * @param repositoryDescriptor the descriptor of the repository we want to use
+	 * @param originalDescriptor   the original descriptor queried
+	 * @return <code>true</code> if at least one hashsum matches in both descriptors
+	 */
+	private boolean isMatching(IArtifactDescriptor repositoryDescriptor, IArtifactDescriptor originalDescriptor) {
+		for (Entry<String, String> entry : originalDescriptor.getProperties().entrySet()) {
+			String key = entry.getKey();
+			if (key.startsWith(TychoConstants.PROP_DOWNLOAD_CHECKSUM_PREFIX)) {
+				String property = repositoryDescriptor.getProperty(key);
+				if (property != null) {
+					String value = entry.getValue();
+					return value.equals(property);
+				}
+			}
+		}
+		if (fileSizeMatch(repositoryDescriptor, originalDescriptor)) {
+			// if we are here, then it means no download checksums where present for
+			// comparison and we need to generate one ourself
+			for (Entry<String, String> entry : originalDescriptor.getProperties().entrySet()) {
+				String key = entry.getKey();
+				if (key.startsWith(TychoConstants.PROP_DOWNLOAD_CHECKSUM_PREFIX)) {
+					try {
+						String algorithm = key.substring(TychoConstants.PROP_DOWNLOAD_CHECKSUM_PREFIX.length())
+								.toUpperCase();
+						MessageDigest md = MessageDigest.getInstance(algorithm);
+						try (DigestOutputStream outputStream = new DigestOutputStream(OutputStream.nullOutputStream(),
+								md)) {
+							IArtifactRepository repository = repositoryDescriptor.getRepository();
+							IStatus status = repository.getArtifact(repositoryDescriptor, outputStream, null);
+							if (!status.isOK()) {
+								continue;
+							}
+						}
+						return ChecksumHelper.toHexString(md.digest()).equals(entry.getValue());
+					} catch (Exception e) {
+						// can't check...
+					}
+				}
+			}
+		}
+		return false;
+	}
+
+	private boolean fileSizeMatch(IArtifactDescriptor repositoryDescriptor, IArtifactDescriptor originalDescriptor) {
+		String originalSize = originalDescriptor.getProperty(IArtifactDescriptor.DOWNLOAD_SIZE);
+		if (originalSize != null) {
+			String property = repositoryDescriptor.getProperty(IArtifactDescriptor.DOWNLOAD_SIZE);
+			if (property != null) {
+				return originalSize.equals(property);
+			}
+		}
+		// assume true for further processing
+		return true;
+	}
+
+	private Stream<Path> pools() {
+		if (useSharedPools) {
+			if (useWorkspacePools) {
+				List<Path> sharedBundlePools = RepositoryHelper.getSharedBundlePools();
+				List<Path> workspaceBundlePools = RepositoryHelper.getWorkspaceBundlePools();
+				return Stream.concat(sharedBundlePools.stream(), workspaceBundlePools.stream()).distinct();
+			} else {
+				return RepositoryHelper.getSharedBundlePools().stream();
+			}
+		} else if (useWorkspacePools) {
+			return RepositoryHelper.getWorkspaceBundlePools().stream();
+		} else {
+			return Stream.empty();
+		}
+	}
+
+	private IArtifactRepository getRepository(Path path) {
+		return repositoryMap.computeIfAbsent(path, p -> {
+			try {
+				return artifactRepositoryFactory.load(path.toUri(), 0, null);
+			} catch (ProvisionException e) {
+				return null;
+			}
+		});
+	}
+
+	private static record RepositoryCandidate(Path path, IArtifactRepository repository,
+			IArtifactDescriptor descriptor) {
+
+	}
+
+	@Override
+	public int getPriority() {
+		return priority;
+	}
+
+}

src/site/markdown/SystemProperties.md

@@ -46,3 +46,6 @@ tycho.p2.transport.cache | file path | local maven repository | Specify the loca
 tycho.p2.transport.debug | true/false | false | enable debugging of the Tycho Transport
 tycho.p2.transport.max-download-threads | number | 4 | maximum number of threads that should be used to download artifacts in parallel
 tycho.p2.transport.min-cache-minutes | number | 60 | Number of minutes that a cache entry is assumed to be fresh and is not fetched again from the server
+tycho.p2.transport.bundlepools.priority | number | 100 | priority used for bundle pools
+tycho.p2.transport.bundlepools.shared | true/false | true | query shared bundle pools for artifacts before downloading them from remote servers
+tycho.p2.transport.bundlepools.workspace | true/false | true | query Workspace bundle pools for artifacts before downloading them from remote servers

tycho-api/src/main/java/org/eclipse/tycho/TychoConstants.java

@@ -17,6 +17,8 @@
 import java.io.File;
 import java.util.regex.Pattern;
 
+import org.eclipse.equinox.p2.repository.artifact.IArtifactDescriptor;
+
 public interface TychoConstants {
 
     String USER_HOME = System.getProperty("user.home");
@@ -152,4 +154,5 @@ public interface TychoConstants {
     String SUFFIX_QUALIFIER = ".qualifier";
 
     String SUFFIX_SNAPSHOT = "-SNAPSHOT";
+    String PROP_DOWNLOAD_CHECKSUM_PREFIX = IArtifactDescriptor.DOWNLOAD_CHECKSUM + ".";
 }

tycho-spi/src/main/java/org/eclipse/tycho/helper/MavenPropertyHelper.java

@@ -90,7 +90,19 @@ public String getGlobalProperty(String key, String defaultValue) {
                 return systemProperty;
             }
         }
-        // java sysem properties last
+        // java system properties last
         return System.getProperty(key, defaultValue);
     }
+
+    public boolean getGlobalBooleanProperty(String key, boolean defaultValue) {
+        return Boolean.parseBoolean(getGlobalProperty(key, Boolean.toString(defaultValue)));
+    }
+
+    public int getGlobalIntProperty(String key, int defaultValue) {
+        try {
+            return Integer.parseInt(getGlobalProperty(key, Integer.toString(defaultValue)));
+        } catch (NumberFormatException e) {
+            return defaultValue;
+        }
+    }
 }