Skip to content

Commit

Permalink
Revert "[core] Paimon catalog support immutable options (#1886)" (#1933)
Browse files Browse the repository at this point in the history
This reverts commit 6036be8.
  • Loading branch information
FangYongs authored Sep 4, 2023
1 parent 6036be8 commit 9967b53
Show file tree
Hide file tree
Showing 5 changed files with 0 additions and 351 deletions.
12 changes: 0 additions & 12 deletions docs/layouts/shortcodes/generated/catalog_configuration.html
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,6 @@
</tr>
</thead>
<tbody>
<tr>
<td><h5>data-lineage</h5></td>
<td style="word-wrap: break-word;">false</td>
<td>Boolean</td>
<td>Whether save the data lineage information or not, by default is false.</td>
</tr>
<tr>
<td><h5>fs.allow-hadoop-fallback</h5></td>
<td style="word-wrap: break-word;">true</td>
Expand Down Expand Up @@ -68,12 +62,6 @@
<td>String</td>
<td>Metastore of paimon catalog, supports filesystem and hive.</td>
</tr>
<tr>
<td><h5>table-lineage</h5></td>
<td style="word-wrap: break-word;">false</td>
<td>Boolean</td>
<td>Whether save the table lineage information or not, by default is false.</td>
</tr>
<tr>
<td><h5>table.type</h5></td>
<td style="word-wrap: break-word;">managed</td>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,11 @@

package org.apache.paimon.options;

import org.apache.paimon.annotation.Documentation;
import org.apache.paimon.options.description.Description;
import org.apache.paimon.options.description.TextElement;
import org.apache.paimon.table.TableType;

import java.lang.reflect.Field;
import java.time.Duration;
import java.util.HashSet;
import java.util.Set;

import static org.apache.paimon.options.ConfigOptions.key;

Expand Down Expand Up @@ -82,7 +78,6 @@ public class CatalogOptions {
.withDescription(
"Allow to fallback to hadoop File IO when no file io found for the scheme.");

@Documentation.Immutable
public static final ConfigOption<String> LINEAGE_META =
key("lineage-meta")
.stringType()
Expand All @@ -102,36 +97,4 @@ public class CatalogOptions {
TextElement.text(
"\"custom\": You can implement LineageMetaFactory and LineageMeta to store lineage information in customized storage."))
.build());

@Documentation.Immutable
public static final ConfigOption<Boolean> TABLE_LINEAGE =
key("table-lineage")
.booleanType()
.defaultValue(false)
.withDescription(
"Whether save the table lineage information or not, by default is false.");

@Documentation.Immutable
public static final ConfigOption<Boolean> DATA_LINEAGE =
key("data-lineage")
.booleanType()
.defaultValue(false)
.withDescription(
"Whether save the data lineage information or not, by default is false.");

public static Set<String> getImmutableOptionKeys() {
final Field[] fields = CatalogOptions.class.getFields();
final Set<String> immutableKeys = new HashSet<>(fields.length);
for (Field field : fields) {
if (ConfigOption.class.isAssignableFrom(field.getType())
&& field.getAnnotation(Documentation.Immutable.class) != null) {
try {
immutableKeys.add(((ConfigOption<?>) field.get(CatalogOptions.class)).key());
} catch (IllegalAccessException e) {
throw new RuntimeException(e);
}
}
}
return immutableKeys;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,11 @@
import org.apache.paimon.factories.FactoryUtil;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.Path;
import org.apache.paimon.options.CatalogOptions;
import org.apache.paimon.options.Options;
import org.apache.paimon.utils.Preconditions;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;

import static org.apache.paimon.catalog.CatalogOptionsManager.validateCatalogOptions;
import static org.apache.paimon.options.CatalogOptions.METASTORE;
import static org.apache.paimon.options.CatalogOptions.WAREHOUSE;
import static org.apache.paimon.utils.Preconditions.checkArgument;
Expand Down Expand Up @@ -88,33 +82,10 @@ static Catalog createCatalog(CatalogContext context, ClassLoader classLoader) {
} else {
fileIO.mkdirs(warehousePath);
}

// persist immutable catalog options
CatalogOptionsManager catalogOptionsManager =
new CatalogOptionsManager(fileIO, new Path(warehouse));
Map<String, String> immutableOptions =
immutableOptions(context, CatalogOptions.getImmutableOptionKeys());
if (fileIO.exists(catalogOptionsManager.getCatalogOptionPath())) {
Map<String, String> originImmutableOptions =
catalogOptionsManager.immutableOptions();
validateCatalogOptions(
Options.fromMap(originImmutableOptions), Options.fromMap(immutableOptions));
} else {
catalogOptionsManager.saveImmutableOptions(immutableOptions);
}
} catch (IOException e) {
throw new UncheckedIOException(e);
}

return catalogFactory.create(fileIO, warehousePath, context);
}

static Map<String, String> immutableOptions(
CatalogContext context, Set<String> immutableCatalogOptionKeys) {
Map<String, String> immutableOptions = new HashMap<>();
context.options().keySet().stream()
.filter(key -> immutableCatalogOptionKeys.contains(key))
.forEach(key -> immutableOptions.put(key, context.options().get(key)));
return immutableOptions;
}
}

This file was deleted.

Loading

0 comments on commit 9967b53

Please sign in to comment.