Skip to content

Commit

Permalink
Allow configuration of HTTP timeouts for maven repositories (#3951)
Browse files Browse the repository at this point in the history 
* Allow configuration of HTTP timeouts for maven repositories

* Applying server connection timeouts to `MavenRepositoryMirror`.

* Reverting unnecessary changes

* Increasing timeouts due to flaky connections

* Reusing repository settings when using profiles

---------

Co-authored-by: Tim te Beek <[email protected]>
  • Loading branch information
@ammachado @timtebeek
ammachado and timtebeek authored Jun 19, 2024
1 parent 1e409ed commit 52c7703
Show file tree
Hide file tree
Showing 13 changed files with 314 additions and 70 deletions.
32 changes: 29 additions & 3 deletions rewrite-core/src/main/java/org/openrewrite/ipc/http/HttpSender.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.time.Duration;
import java.util.Base64;
import java.util.LinkedHashMap;
import java.util.Map;
Expand Down Expand Up @@ -74,12 +75,16 @@ class Request {
private final byte[] entity;
private final Method method;
private final Map<String, String> requestHeaders;
private final Duration connectTimeout;
private final Duration readTimeout;

public Request(URL url, byte[] entity, Method method, Map<String, String> requestHeaders) {
public Request(URL url, byte[] entity, Method method, Map<String, String> requestHeaders, Duration connectTimeout, Duration readTimeout) {
this.url = url;
this.entity = entity;
this.method = method;
this.requestHeaders = requestHeaders;
this.connectTimeout = connectTimeout;
this.readTimeout = readTimeout;
}

public URL getUrl() {
Expand All @@ -98,6 +103,14 @@ public Map<String, String> getRequestHeaders() {
return requestHeaders;
}

public Duration getConnectTimeout() {
return connectTimeout;
}

public Duration getReadTimeout() {
return readTimeout;
}

public static Builder build(String uri, HttpSender sender) {
return new Builder(uri, sender);
}
Expand Down Expand Up @@ -125,6 +138,8 @@ public static class Builder {
private URL url;
private byte[] entity = new byte[0];
private Method method;
private Duration connectTimeout;
private Duration readTimeout;

Builder(String url, HttpSender sender) {
try {
Expand Down Expand Up @@ -277,8 +292,9 @@ public final Builder compress() throws IOException {
* @throws IOException If compression fails.
*/
public final Builder compressWhen(Supplier<Boolean> when) throws IOException {
if (when.get())
if (when.get()) {
return compress();
}
return this;
}

Expand Down Expand Up @@ -331,7 +347,17 @@ public Response send() {
}

public Request build() {
return new Request(url, entity, method, requestHeaders);
return new Request(url, entity, method, requestHeaders, connectTimeout, readTimeout);
}

public Builder withConnectTimeout(Duration connectTimeout) {
this.connectTimeout = connectTimeout;
return this;
}

public Builder withReadTimeout(Duration readTimeout) {
this.readTimeout = readTimeout;
return this;
}
}
}
Expand Down
12 changes: 10 additions & 2 deletions rewrite-core/src/main/java/org/openrewrite/ipc/http/HttpUrlConnectionSender.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,16 @@ public Response send(Request request) {
} else {
con = (HttpURLConnection) request.getUrl().openConnection();
}
con.setConnectTimeout(connectTimeoutMs);
con.setReadTimeout(readTimeoutMs);
if (request.getConnectTimeout() != null) {
con.setConnectTimeout((int) request.getConnectTimeout().toMillis());
} else {
con.setConnectTimeout(connectTimeoutMs);
}
if (request.getReadTimeout() != null) {
con.setReadTimeout((int) request.getReadTimeout().toMillis());
} else {
con.setReadTimeout(readTimeoutMs);
}
Method method = request.getMethod();
con.setRequestMethod(method.name());

Expand Down
38 changes: 29 additions & 9 deletions rewrite-maven/src/main/java/org/openrewrite/maven/MavenExecutionContextView.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,24 +256,44 @@ private static List<MavenRepositoryCredentials> mapCredentials(MavenSettings set
private static List<MavenRepositoryMirror> mapMirrors(MavenSettings settings) {
if (settings.getMirrors() != null) {
return settings.getMirrors().getMirrors().stream()
.map(mirror -> new MavenRepositoryMirror(mirror.getId(), mirror.getUrl(), mirror.getMirrorOf(), mirror.getReleases(), mirror.getSnapshots()))
.map(mirror -> new MavenRepositoryMirror(mirror.getId(), mirror.getUrl(), mirror.getMirrorOf(), mirror.getReleases(), mirror.getSnapshots(), settings.getServers()))
.collect(Collectors.toList());
}
return emptyList();
}

private List<MavenRepository> mapRepositories(MavenSettings settings, List<String> activeProfiles) {
Map<String, MavenRepository> repositories = this.getRepositories().stream()
.collect(Collectors.toMap(MavenRepository::getId, r -> r, (a, b) -> a));
return settings.getActiveRepositories(activeProfiles).stream()
.map(repo -> {
try {
return new MavenRepository(
repo.getId(),
repo.getUrl(),
repo.getReleases() == null ? null : repo.getReleases().getEnabled(),
repo.getSnapshots() == null ? null : repo.getSnapshots().getEnabled(),
null,
null
);
MavenRepository knownRepo = repositories.get(repo.getId());
if (knownRepo != null) {
return new MavenRepository(
repo.getId(),
repo.getUrl(),
repo.getReleases() != null ? repo.getReleases().getEnabled() : knownRepo.getReleases(),
repo.getSnapshots() != null ? repo.getSnapshots().getEnabled() : knownRepo.getSnapshots(),
knownRepo.isKnownToExist() && knownRepo.getUri().equals(repo.getUrl()),
knownRepo.getUsername(),
knownRepo.getPassword(),
knownRepo.getConnectTimeout(),
knownRepo.getReadTimeout(),
knownRepo.getDeriveMetadataIfMissing()
);
} else {
return new MavenRepository(
repo.getId(),
repo.getUrl(),
repo.getReleases() == null ? null : repo.getReleases().getEnabled(),
repo.getSnapshots() == null ? null : repo.getSnapshots().getEnabled(),
null,
null,
null,
null
);
}
} catch (Exception exception) {
this.getOnError().accept(new MavenParsingException(
"Unable to parse URL %s for Maven settings repository id %s",
Expand Down
19 changes: 17 additions & 2 deletions rewrite-maven/src/main/java/org/openrewrite/maven/MavenSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,11 @@
package org.openrewrite.maven;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
import lombok.*;
import lombok.experimental.FieldDefaults;
import lombok.experimental.NonFinal;
Expand Down Expand Up @@ -46,12 +49,14 @@
@EqualsAndHashCode(onlyExplicitlyIncluded = true)
@Data
@AllArgsConstructor
@JacksonXmlRootElement(localName = "settings")
public class MavenSettings {
@Nullable
String localRepository;

@Nullable
@NonFinal
@JsonIgnore
MavenRepository mavenLocal;

@Nullable
Expand Down Expand Up @@ -238,8 +243,11 @@ private ServerConfiguration interpolate(@Nullable ServerConfiguration configurat
if (configuration == null) {
return null;
}
return new ServerConfiguration(configuration.httpHeaders == null ? null :
ListUtils.map(configuration.httpHeaders, this::interpolate));
return new ServerConfiguration(
configuration.httpHeaders == null ? null : ListUtils.map(configuration.httpHeaders, this::interpolate),
configuration.connectTimeout == null ? null : configuration.connectTimeout,
configuration.readTimeout == null ? null : configuration.readTimeout
);
}

private HttpHeader interpolate(HttpHeader httpHeader) {
Expand Down Expand Up @@ -406,9 +414,16 @@ public static class Server {
@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
@Data
@With
@JsonIgnoreProperties(value = "httpHeaders")
public static class ServerConfiguration {
@JacksonXmlProperty(localName = "property")
@JacksonXmlElementWrapper(localName = "httpHeaders", useWrapping = true) // wrapping is disabled by default on MavenXmlMapper
@Nullable
List<HttpHeader> httpHeaders;
@Nullable
Long connectTimeout;
@Nullable
Long readTimeout;
}

@FieldDefaults(makeFinal = true, level = AccessLevel.PRIVATE)
Expand Down
2 changes: 2 additions & 0 deletions rewrite-maven/src/main/java/org/openrewrite/maven/UpdateMavenModel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,8 @@ public Xml visitDocument(Xml.Document document, P p) {
t.getChild("releases").flatMap(s -> s.getChildValue("enabled")).orElse(null),
t.getChild("snapshots").flatMap(s -> s.getChildValue("enabled")).orElse(null),
null,
null,
null,
null
)).collect(Collectors.toList()));
} else {
Expand Down
35 changes: 26 additions & 9 deletions rewrite-maven/src/main/java/org/openrewrite/maven/internal/MavenPomDownloader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ public MavenPomDownloader(Map<Path, Pom> projectPoms, ExecutionContext ctx,
@Nullable MavenSettings mavenSettings,
@Nullable List<String> activeProfiles) {
this(projectPoms, HttpSenderExecutionContextView.view(ctx).getHttpSender(), ctx);
this.mavenSettings = mavenSettings;
this.mavenSettings = mavenSettings != null ? mavenSettings : MavenExecutionContextView.view(ctx).getSettings();
this.mirrors = this.ctx.getMirrors(mavenSettings);
this.activeProfiles = activeProfiles;
}
Expand Down Expand Up @@ -137,6 +137,7 @@ public MavenPomDownloader(Map<Path, Pom> projectPoms, HttpSender httpSender, Exe
this.projectPomsByGav = projectPomsByGav(projectPoms);
this.httpSender = httpSender;
this.ctx = MavenExecutionContextView.view(ctx);
this.mavenSettings = this.ctx.getSettings();
this.mavenCache = this.ctx.getPomCache();
this.addCentralRepository = !Boolean.FALSE.equals(MavenExecutionContextView.view(ctx).getAddCentralRepository());
this.addLocalRepository = !Boolean.FALSE.equals(MavenExecutionContextView.view(ctx).getAddLocalRepository());
Expand Down Expand Up @@ -773,7 +774,9 @@ public MavenRepository normalizeRepository(MavenRepository originalRepository, M
repository.getReleases(),
repository.getSnapshots(),
repository.getUsername(),
repository.getPassword());
repository.getPassword(),
repository.getConnectTimeout(),
repository.getReadTimeout());
} catch (HttpSenderResponseException e) {
//Response was returned from the server, but it was not a 200 OK. The server therefore exists.
if (e.isServerReached()) {
Expand All @@ -783,7 +786,9 @@ public MavenRepository normalizeRepository(MavenRepository originalRepository, M
repository.getReleases(),
repository.getSnapshots(),
repository.getUsername(),
repository.getPassword());
repository.getPassword(),
repository.getConnectTimeout(),
repository.getReadTimeout());
}
} catch (Throwable e) {
// ok to fall through here and cache a null
Expand Down Expand Up @@ -812,7 +817,10 @@ public MavenRepository normalizeRepository(MavenRepository originalRepository, M
*/
private byte[] requestAsAuthenticatedOrAnonymous(MavenRepository repo, String uriString) throws HttpSenderResponseException, IOException {
try {
return sendRequest(applyAuthenticationToRequest(repo, httpSender.get(uriString)).build());
HttpSender.Request.Builder request = httpSender.get(uriString)
.withConnectTimeout(repo.getConnectTimeout())
.withReadTimeout(repo.getReadTimeout());
return sendRequest(applyAuthenticationToRequest(repo, request).build());
} catch (HttpSenderResponseException e) {
if (hasCredentials(repo) && e.isClientSideException()) {
return retryRequestAnonymously(uriString, e);
Expand Down Expand Up @@ -845,11 +853,20 @@ private MavenRepository applyAuthenticationToRepository(MavenRepository reposito
* Returns a request builder with Authorization header set if the provided repository specifies credentials
*/
private HttpSender.Request.Builder applyAuthenticationToRequest(MavenRepository repository, HttpSender.Request.Builder request) {
if (ctx.getSettings() != null && ctx.getSettings().getServers() != null) {
for (MavenSettings.Server server : ctx.getSettings().getServers().getServers()) {
if (server.getId().equals(repository.getId()) && server.getConfiguration() != null && server.getConfiguration().getHttpHeaders() != null) {
for (MavenSettings.HttpHeader header : server.getConfiguration().getHttpHeaders()) {
request.withHeader(header.getName(), header.getValue());
if (mavenSettings != null && mavenSettings.getServers() != null) {
for (MavenSettings.Server server : mavenSettings.getServers().getServers()) {
if (server.getId().equals(repository.getId()) && server.getConfiguration() != null) {
MavenSettings.ServerConfiguration configuration = server.getConfiguration();
if (server.getConfiguration().getHttpHeaders() != null) {
for (MavenSettings.HttpHeader header : configuration.getHttpHeaders()) {
request.withHeader(header.getName(), header.getValue());
}
}
if (configuration.getConnectTimeout() != null) {
request.withConnectTimeout(Duration.ofMillis(configuration.getConnectTimeout()));
}
if (configuration.getReadTimeout() != null) {
request.withReadTimeout(Duration.ofMillis(configuration.getReadTimeout()));
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion rewrite-maven/src/main/java/org/openrewrite/maven/internal/RawPom.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -439,7 +439,7 @@ private List<MavenRepository> mapRepositories(@Nullable RawRepositories rawRepos
pomRepositories.add(new MavenRepository(r.getId(), r.getUrl(),
r.getReleases() == null ? null : r.getReleases().getEnabled(),
r.getSnapshots() == null ? null : r.getSnapshots().getEnabled(),
false, null, null, null));
false, null, null, null, null, null));
}

}
Expand Down
Loading

0 comments on commit 52c7703

Please sign in to comment.