Skip to content

Commit

Permalink
Refactored out some unused APIs
Browse files Browse the repository at this point in the history
  • Loading branch information
@malhotrashivam
malhotrashivam committed Nov 13, 2024
1 parent 06878ae commit dc8243e
Show file tree
Hide file tree
Showing 2 changed files with 23 additions and 106 deletions.
13 changes: 3 additions & 10 deletions extensions/parquet/table/src/main/java/io/deephaven/parquet/table/MappedSchema.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,21 +26,14 @@ static MappedSchema create(
final TableDefinition definition,
final RowSet rowSet,
final Map<String, ? extends ColumnSource<?>> columnSourceMap,
final ParquetInstructions instructions,
final ColumnDefinition<?>... extraColumns) {
final ParquetInstructions instructions) {
final MessageTypeBuilder builder = Types.buildMessage();
for (final ColumnDefinition<?> columnDefinition : definition.getColumns()) {
TypeInfos.TypeInfo typeInfo =
final TypeInfos.TypeInfo typeInfo =
getTypeInfo(computedCache, columnDefinition, rowSet, columnSourceMap, instructions);
Type schemaType = typeInfo.createSchemaType(columnDefinition, instructions);
final Type schemaType = typeInfo.createSchemaType(columnDefinition, instructions);
builder.addField(schemaType);
}

for (final ColumnDefinition<?> extraColumn : extraColumns) {
builder.addField(getTypeInfo(computedCache, extraColumn, rowSet, columnSourceMap, instructions)
.createSchemaType(extraColumn, instructions));
}

final MessageType schema = builder.named("root");
return new MappedSchema(definition, schema);
}
Expand Down
116 changes: 20 additions & 96 deletions extensions/parquet/table/src/main/java/io/deephaven/parquet/table/ParquetInstructions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
import io.deephaven.hash.KeyedObjectKey;
import io.deephaven.parquet.base.ParquetUtils;
import io.deephaven.util.annotations.VisibleForTesting;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

Expand All @@ -33,7 +32,17 @@
*/
public abstract class ParquetInstructions implements ColumnToCodecMappings {

private static volatile String defaultCompressionCodecName = CompressionCodecName.SNAPPY.toString();
public static final String DEFAULT_COMPRESSION_CODEC_NAME = "SNAPPY";
public static final int DEFAULT_MAXIMUM_DICTIONARY_KEYS = 1 << 20;
public static final int DEFAULT_MAXIMUM_DICTIONARY_SIZE = 1 << 20;

public static final int MIN_TARGET_PAGE_SIZE = 1 << 11; // 2KB
private static final int minTargetPageSize = Configuration.getInstance().getIntegerWithDefault(
"Parquet.minTargetPageSize", MIN_TARGET_PAGE_SIZE);

public static final int DEFAULT_TARGET_PAGE_SIZE = 1 << 16; // 64KB
private static final int defaultTargetPageSize = Configuration.getInstance().getIntegerWithDefault(
"Parquet.defaultTargetPageSize", DEFAULT_TARGET_PAGE_SIZE);

/**
* Throws an exception if {@link ParquetInstructions#getTableDefinition()} is empty.
Expand All @@ -46,93 +55,8 @@ public static TableDefinition ensureDefinition(ParquetInstructions parquetInstru
.orElseThrow(() -> new IllegalArgumentException("Table definition must be provided"));
}

/**
* Set the default for {@link #getCompressionCodecName()}.
*
* @deprecated Use {@link Builder#setCompressionCodecName(String)} instead.
* @param name The new default
*/
@Deprecated
public static void setDefaultCompressionCodecName(final String name) {
defaultCompressionCodecName = name;
}

/**
* @return The default for {@link #getCompressionCodecName()}
*/
public static String getDefaultCompressionCodecName() {
return defaultCompressionCodecName;
}

private static volatile int defaultMaximumDictionaryKeys = 1 << 20;

/**
* Set the default for {@link #getMaximumDictionaryKeys()}.
*
* @param maximumDictionaryKeys The new default
* @see Builder#setMaximumDictionaryKeys(int)
*/
public static void setDefaultMaximumDictionaryKeys(final int maximumDictionaryKeys) {
defaultMaximumDictionaryKeys = Require.geqZero(maximumDictionaryKeys, "maximumDictionaryKeys");
}

/**
* @return The default for {@link #getMaximumDictionaryKeys()}
*/
public static int getDefaultMaximumDictionaryKeys() {
return defaultMaximumDictionaryKeys;
}

private static volatile int defaultMaximumDictionarySize = 1 << 20;

/**
* Set the default for {@link #getMaximumDictionarySize()}.
*
* @param maximumDictionarySize The new default
* @see Builder#setMaximumDictionarySize(int)
*/
public static void setDefaultMaximumDictionarySize(final int maximumDictionarySize) {
defaultMaximumDictionarySize = Require.geqZero(maximumDictionarySize, "maximumDictionarySize");
}

/**
* @return The default for {@link #getMaximumDictionarySize()}
*/
public static int getDefaultMaximumDictionarySize() {
return defaultMaximumDictionarySize;
}

public static final int MIN_TARGET_PAGE_SIZE =
Configuration.getInstance().getIntegerWithDefault("Parquet.minTargetPageSize", 1 << 11); // 2KB
private static final int DEFAULT_TARGET_PAGE_SIZE =
Configuration.getInstance().getIntegerWithDefault("Parquet.defaultTargetPageSize", 1 << 16); // 64KB
private static volatile int defaultTargetPageSize = DEFAULT_TARGET_PAGE_SIZE;

private static final boolean DEFAULT_IS_REFRESHING = false;

/**
* Set the default target page size (in bytes) used to section rows of data into pages during column writing. This
* number should be no smaller than {@link #MIN_TARGET_PAGE_SIZE}.
*
* @param newDefaultSizeBytes the new default target page size.
*/
public static void setDefaultTargetPageSize(final int newDefaultSizeBytes) {
if (newDefaultSizeBytes < MIN_TARGET_PAGE_SIZE) {
throw new IllegalArgumentException(
"Default target page size should be larger than " + MIN_TARGET_PAGE_SIZE + " bytes");
}
defaultTargetPageSize = newDefaultSizeBytes;
}

/**
* Get the current default target page size in bytes.
*
* @return the current default target page size in bytes.
*/
public static int getDefaultTargetPageSize() {
return defaultTargetPageSize;
}

public enum ParquetFileLayout {
/**
* A single parquet file.
Expand Down Expand Up @@ -168,7 +92,7 @@ public enum ParquetFileLayout {
static final String FILE_INDEX_TOKEN = "{i}";
private static final String DEFAULT_BASE_NAME_FOR_PARTITIONED_PARQUET_DATA = UUID_TOKEN;

public ParquetInstructions() {}
private ParquetInstructions() {}

public final String getColumnNameFromParquetColumnNameOrDefault(final String parquetColumnName) {
final String mapped = getColumnNameFromParquetColumnName(parquetColumnName);
Expand Down Expand Up @@ -313,17 +237,17 @@ public Object getSpecialInstructions() {

@Override
public String getCompressionCodecName() {
return defaultCompressionCodecName;
return DEFAULT_COMPRESSION_CODEC_NAME;
}

@Override
public int getMaximumDictionaryKeys() {
return defaultMaximumDictionaryKeys;
return DEFAULT_MAXIMUM_DICTIONARY_KEYS;
}

@Override
public int getMaximumDictionarySize() {
return defaultMaximumDictionarySize;
return DEFAULT_MAXIMUM_DICTIONARY_SIZE;
}

@Override
Expand Down Expand Up @@ -692,9 +616,9 @@ public static class Builder {
// We only store entries in parquetColumnNameToInstructions when the parquetColumnName is
// different than the columnName (ie, the column name mapping is not the default mapping)
private KeyedObjectHashMap<String, ColumnInstructions> parquetColumnNameToInstructions;
private String compressionCodecName = defaultCompressionCodecName;
private int maximumDictionaryKeys = defaultMaximumDictionaryKeys;
private int maximumDictionarySize = defaultMaximumDictionarySize;
private String compressionCodecName = DEFAULT_COMPRESSION_CODEC_NAME;
private int maximumDictionaryKeys = DEFAULT_MAXIMUM_DICTIONARY_KEYS;
private int maximumDictionarySize = DEFAULT_MAXIMUM_DICTIONARY_SIZE;
private boolean isLegacyParquet;
private int targetPageSize = defaultTargetPageSize;
private boolean isRefreshing = DEFAULT_IS_REFRESHING;
Expand Down Expand Up @@ -880,8 +804,8 @@ public Builder setIsLegacyParquet(final boolean isLegacyParquet) {
}

public Builder setTargetPageSize(final int targetPageSize) {
if (targetPageSize < MIN_TARGET_PAGE_SIZE) {
throw new IllegalArgumentException("Target page size should be >= " + MIN_TARGET_PAGE_SIZE);
if (targetPageSize < minTargetPageSize) {
throw new IllegalArgumentException("Target page size should be >= " + minTargetPageSize);
}
this.targetPageSize = targetPageSize;
return this;
Expand Down

0 comments on commit dc8243e

Please sign in to comment.