Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[AMORO-3256] Amoro Open Api with Swagger in Ams
Browse files Browse the repository at this point in the history
mansonliwh committed Nov 8, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent e54fbc2 commit ae5b371
Showing 43 changed files with 2,939 additions and 123 deletions.
84 changes: 83 additions & 1 deletion amoro-ams/pom.xml
Original file line number Diff line number Diff line change
@@ -35,6 +35,7 @@
<url>https://amoro.apache.org</url>

<properties>
<skip-generate-sdk>false</skip-generate-sdk>
<git-commit-id-plugin.fail-on-no-git-dir>false</git-commit-id-plugin.fail-on-no-git-dir>
</properties>

@@ -279,7 +280,7 @@
<groupId>software.amazon.awssdk</groupId>
<artifactId>url-connection-client</artifactId>
</dependency>

<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3-transfer-manager</artifactId>
@@ -408,6 +409,18 @@
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.webjars</groupId>
<artifactId>swagger-ui</artifactId>
<version>5.17.14</version>
</dependency>

<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
<version>4.5.13</version>
</dependency>

<!-- testcontainers dependencies -->
<dependency>
<groupId>org.testcontainers</groupId>
@@ -503,6 +516,69 @@
<skip>true</skip>
</configuration>
</plugin>

<plugin>
<groupId>org.openapitools</groupId>
<artifactId>openapi-generator-maven-plugin</artifactId>
<version>6.0.0</version>
<executions>
<execution>
<id>generate-sdk</id>
<goals>
<goal>generate</goal>
</goals>
<phase>generate-sources</phase>
<configuration>
<inputSpec>${project.basedir}/src/main/resources/openapi/openapi.yaml</inputSpec>
<generatorName>java</generatorName>
<output>${project.build.directory}/generated-sources/openapi</output>
<apiPackage>com.amoro.sdk.api</apiPackage>
<modelPackage>com.amoro.sdk.model</modelPackage>
<invokerPackage>com.amoro.sdk.invoker</invokerPackage>
<configOptions>
<library>apache-httpclient</library>
</configOptions>
<skip>${skip-generate-sdk}</skip>
</configuration>
</execution>
<execution>
<id>generate-doc</id>
<goals>
<goal>generate</goal>
</goals>
<phase>generate-sources</phase>
<configuration>
<inputSpec>${project.basedir}/src/main/resources/openapi/openapi.yaml</inputSpec>
<generatorName>html2</generatorName>
<output>${project.build.directory}/generated-docs/openapi</output>
<skip>${skip-generate-sdk}</skip>
</configuration>
</execution>
</executions>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<id>package-sdk</id>
<goals>
<goal>jar</goal>
</goals>
<phase>package</phase>
<configuration>
<classifier>sdk</classifier>
<classesDirectory>${project.build.directory}/generated-sources/openapi/src/main/java</classesDirectory>
<outputDirectory>${project.build.directory}</outputDirectory>
<finalName>sdk</finalName>
</configuration>
</execution>
</executions>
<configuration>
<skip>${skip-generate-sdk}</skip>
</configuration>
</plugin>
</plugins>
</build>

@@ -544,5 +620,11 @@
<git-commit-id-plugin.fail-on-no-git-dir>true</git-commit-id-plugin.fail-on-no-git-dir>
</properties>
</profile>
<profile>
<id>generate-sdk</id>
<properties>
<skip-generate-sdk>false</skip-generate-sdk>
</properties>
</profile>
</profiles>
</project>
Original file line number Diff line number Diff line change
@@ -20,6 +20,7 @@

import io.javalin.Javalin;
import io.javalin.http.HttpCode;
import io.javalin.http.staticfiles.Location;
import org.apache.amoro.Constants;
import org.apache.amoro.OptimizerProperties;
import org.apache.amoro.api.AmoroTableMetastore;
@@ -241,11 +242,14 @@ private void initHttpService() {
Javalin.create(
config -> {
config.addStaticFiles(dashboardServer.configStaticFiles());
config.addStaticFiles("/META-INF/resources/webjars", Location.CLASSPATH);
config.sessionHandler(SessionHandler::new);
config.enableCorsForAllOrigins();
config.jsonMapper(JavalinJsonMapper.createDefaultJsonMapper());
config.showJavalinBanner = false;
config.enableWebjars();
});

httpServer.routes(
() -> {
dashboardServer.endpoints().addEndpoints();
Original file line number Diff line number Diff line change
@@ -76,6 +76,7 @@ public class DashboardServer {
private static final String AUTH_TYPE_BASIC = "basic";
private static final String X_REQUEST_SOURCE_HEADER = "X-Request-Source";
private static final String X_REQUEST_SOURCE_WEB = "Web";
private static final String SWAGGER_PATH = "/swagger-ui";

private final CatalogController catalogController;
private final HealthCheckController healthCheckController;
@@ -118,7 +119,6 @@ public DashboardServer(
}

private String indexHtml = "";

// read index.html content
public String getIndexFileContent() {
try {
@@ -170,6 +170,17 @@ public EndpointGroup endpoints() {
path(
"",
() -> {
get(
"/swagger-docs",
ctx -> {
InputStream openapiStream =
getClass().getClassLoader().getResourceAsStream("openapi/openapi.yaml");
if (openapiStream == null) {
ctx.status(404).result("OpenAPI specification file not found");
} else {
ctx.result(openapiStream);
}
});
// static files
get(
"/{page}",
@@ -358,6 +369,9 @@ private EndpointGroup apiGroup() {
public void preHandleRequest(Context ctx) {
String uriPath = ctx.path();
String requestSource = ctx.header(X_REQUEST_SOURCE_HEADER);
if (uriPath.startsWith(SWAGGER_PATH)) {
return;
}
if (needApiKeyCheck(uriPath) && !X_REQUEST_SOURCE_WEB.equalsIgnoreCase(requestSource)) {
if (AUTH_TYPE_BASIC.equalsIgnoreCase(authType)) {
BasicAuthCredentials cred = ctx.basicAuthCredentials();
@@ -410,6 +424,9 @@ public void handleException(Exception e, Context ctx) {
"/index.html",
"/favicon.ico",
"/assets/*",
"/swagger-ui",
"/swagger-ui/*",
"/swagger-docs",
RestCatalogService.ICEBERG_REST_API_PREFIX + "/*"
};

Original file line number Diff line number Diff line change
@@ -87,6 +87,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nullable;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -341,7 +343,7 @@ private void collectSnapshots(

@Override
public List<PartitionFileBaseInfo> getSnapshotDetail(
AmoroTable<?> amoroTable, String snapshotId) {
AmoroTable<?> amoroTable, String snapshotId, @Nullable String ref) {
MixedTable mixedTable = getTable(amoroTable);
List<PartitionFileBaseInfo> result = new ArrayList<>();
long commitId = Long.parseLong(snapshotId);
Original file line number Diff line number Diff line change
@@ -81,10 +81,10 @@ public List<AmoroSnapshotsOfTable> getSnapshots(
}

public List<PartitionFileBaseInfo> getSnapshotDetail(
TableIdentifier tableIdentifier, String snapshotId) {
TableIdentifier tableIdentifier, String snapshotId, String ref) {
AmoroTable<?> amoroTable = loadTable(tableIdentifier);
FormatTableDescriptor formatTableDescriptor = formatDescriptorMap.get(amoroTable.format());
return formatTableDescriptor.getSnapshotDetail(amoroTable, snapshotId);
return formatTableDescriptor.getSnapshotDetail(amoroTable, snapshotId, ref);
}

public List<DDLInfo> getTableOperations(TableIdentifier tableIdentifier) {
Original file line number Diff line number Diff line change
@@ -150,10 +150,12 @@ public void getTableDetail(Context ctx) {
TableIdentifier.of(catalog, database, tableName).buildTableIdentifier()));
if (serverTableIdentifier.isPresent()) {
TableRuntime tableRuntime = tableService.getRuntime(serverTableIdentifier.get().getId());
tableSummary.setOptimizingStatus(tableRuntime.getOptimizingStatus().name());
OptimizingEvaluator.PendingInput tableRuntimeSummary = tableRuntime.getTableSummary();
if (tableRuntimeSummary != null) {
tableSummary.setHealthScore(tableRuntimeSummary.getHealthScore());
if (tableRuntime != null) {
tableSummary.setOptimizingStatus(tableRuntime.getOptimizingStatus().name());
OptimizingEvaluator.PendingInput tableRuntimeSummary = tableRuntime.getTableSummary();
if (tableRuntimeSummary != null) {
tableSummary.setHealthScore(tableRuntimeSummary.getHealthScore());
}
}
} else {
tableSummary.setOptimizingStatus(OptimizingStatus.IDLE.name());
@@ -416,10 +418,13 @@ public void getSnapshotDetail(Context ctx) {
String snapshotId = ctx.pathParam("snapshotId");
Integer page = ctx.queryParamAsClass("page", Integer.class).getOrDefault(1);
Integer pageSize = ctx.queryParamAsClass("pageSize", Integer.class).getOrDefault(20);
String ref = ctx.queryParamAsClass("ref", String.class).getOrDefault(null);

List<PartitionFileBaseInfo> result =
tableDescriptor.getSnapshotDetail(
TableIdentifier.of(catalog, database, tableName).buildTableIdentifier(), snapshotId);
TableIdentifier.of(catalog, database, tableName).buildTableIdentifier(),
snapshotId,
ref);
int offset = (page - 1) * pageSize;
PageResult<PartitionFileBaseInfo> amsPageResult = PageResult.of(result, offset, pageSize);
ctx.json(OkResponse.of(amsPageResult));
Original file line number Diff line number Diff line change
@@ -264,13 +264,13 @@ private TableOptimizingProcess planInternal(TableRuntime tableRuntime) {
try {
AmoroTable<?> table = tableManager.loadTable(tableRuntime.getTableIdentifier());
OptimizingPlanner planner =
new OptimizingPlanner(
OptimizingPlanner.createOptimizingPlanner(
tableRuntime.refresh(table),
(MixedTable) table.originalTable(),
getAvailableCore(),
maxInputSizePerThread());
if (planner.isNecessary()) {
return new TableOptimizingProcess(planner);
return new TableOptimizingProcess(planner, tableRuntime);
} else {
tableRuntime.completeEmptyProcess();
return null;
@@ -371,9 +371,9 @@ public TaskRuntime poll() {
}
}

public TableOptimizingProcess(OptimizingPlanner planner) {
public TableOptimizingProcess(OptimizingPlanner planner, TableRuntime tableRuntime) {
processId = planner.getProcessId();
tableRuntime = planner.getTableRuntime();
this.tableRuntime = tableRuntime;
optimizingType = planner.getOptimizingType();
planTime = planner.getPlanTime();
targetSnapshotId = planner.getTargetSnapshotId();
Original file line number Diff line number Diff line change
@@ -18,12 +18,12 @@

package org.apache.amoro.server.optimizing.plan;

import org.apache.amoro.ServerTableIdentifier;
import org.apache.amoro.config.OptimizingConfig;
import org.apache.amoro.optimizing.OptimizingInputProperties;
import org.apache.amoro.optimizing.RewriteFilesInput;
import org.apache.amoro.server.optimizing.OptimizingType;
import org.apache.amoro.server.optimizing.RewriteStageTask;
import org.apache.amoro.server.table.TableRuntime;
import org.apache.amoro.shade.guava32.com.google.common.collect.Lists;
import org.apache.amoro.shade.guava32.com.google.common.collect.Maps;
import org.apache.amoro.shade.guava32.com.google.common.collect.Sets;
@@ -48,7 +48,9 @@ public abstract class AbstractPartitionPlan implements PartitionEvaluator {

protected final Pair<Integer, StructLike> partition;
protected final OptimizingConfig config;
protected final TableRuntime tableRuntime;
protected final ServerTableIdentifier identifier;
protected final long lastMinorOptimizingTime;
protected final long lastFullOptimizingTime;
private CommonPartitionEvaluator evaluator;
private TaskSplitter taskSplitter;
protected MixedTable tableObject;
@@ -74,15 +76,20 @@ public abstract class AbstractPartitionPlan implements PartitionEvaluator {
protected final Set<String> reservedDeleteFiles = Sets.newHashSet();

public AbstractPartitionPlan(
TableRuntime tableRuntime,
ServerTableIdentifier identifier,
MixedTable table,
OptimizingConfig config,
Pair<Integer, StructLike> partition,
long planTime) {
long planTime,
long lastMinorOptimizingTime,
long lastFullOptimizingTime) {
this.identifier = identifier;
this.partition = partition;
this.tableObject = table;
this.config = tableRuntime.getOptimizingConfig();
this.tableRuntime = tableRuntime;
this.config = config;
this.planTime = planTime;
this.lastMinorOptimizingTime = lastMinorOptimizingTime;
this.lastFullOptimizingTime = lastFullOptimizingTime;
}

@Override
@@ -98,7 +105,8 @@ protected CommonPartitionEvaluator evaluator() {
}

protected CommonPartitionEvaluator buildEvaluator() {
return new CommonPartitionEvaluator(tableRuntime, partition, planTime);
return new CommonPartitionEvaluator(
identifier, config, partition, planTime, lastMinorOptimizingTime, lastFullOptimizingTime);
}

@Override
@@ -317,10 +325,7 @@ public RewriteStageTask buildTask(OptimizingInputProperties properties) {
MixedTableUtil.getMixedTablePartitionSpecById(tableObject, partition.first());
String partitionPath = spec.partitionToPath(partition.second());
return new RewriteStageTask(
tableRuntime.getTableIdentifier().getId(),
partitionPath,
input,
properties.getProperties());
identifier.getId(), partitionPath, input, properties.getProperties());
}
}

Loading

0 comments on commit ae5b371

Please sign in to comment.