From 0114cdc22bd339f31ff79f3d58f895854aa8362f Mon Sep 17 00:00:00 2001 From: Scott Sandre Date: Wed, 23 Oct 2024 09:08:37 -0700 Subject: [PATCH 01/12] Create Nullable.java --- .../io/delta/kernel/annotation/Nullable.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 kernel/kernel-api/src/main/java/io/delta/kernel/annotation/Nullable.java diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/annotation/Nullable.java b/kernel/kernel-api/src/main/java/io/delta/kernel/annotation/Nullable.java new file mode 100644 index 0000000000..23e32a0f5c --- /dev/null +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/annotation/Nullable.java @@ -0,0 +1,27 @@ +/* + * Copyright (2024) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.annotation; + +import java.lang.annotation.*; + +/** + * Annotation to indicate that a field, method parameter, method return value, or local variable may + * be null. + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER, ElementType.LOCAL_VARIABLE}) +public @interface Nullable {} From fb35cd2963fe8c9121e40d5f8bbf7792e0061a68 Mon Sep 17 00:00:00 2001 From: Scott Sandre Date: Wed, 23 Oct 2024 15:46:25 -0700 Subject: [PATCH 02/12] Create ConfigurationProvider.java --- .../kernel/config/ConfigurationProvider.java | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 kernel/kernel-api/src/main/java/io/delta/kernel/config/ConfigurationProvider.java diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/config/ConfigurationProvider.java b/kernel/kernel-api/src/main/java/io/delta/kernel/config/ConfigurationProvider.java new file mode 100644 index 0000000000..98dabf0469 --- /dev/null +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/config/ConfigurationProvider.java @@ -0,0 +1,61 @@ +/* + * Copyright (2024) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.kernel.config; + +import io.delta.kernel.annotation.Evolving; +import java.util.NoSuchElementException; +import java.util.Optional; + +/** + * A generic interface for retrieving configuration values. + * + * @since 3.3.0 + */ +@Evolving +public interface ConfigurationProvider { + /** + * Retrieves the configuration value for the given key. + * + * @param key the configuration key + * @return the configuration value associated with the key, if it exists + * @throws NoSuchElementException if the key is not found + */ + String get(String key) throws NoSuchElementException; + + /** + * Retrieves the configuration value for the given key. If it doesn't exist, returns {@link + * Optional#empty}. + * + * @param key the configuration key + * @return the configuration value associated with the key, if it exists + */ + Optional getOptional(String key); + + /** + * Retrieves the configuration value for the given key, ensuring that the value is not null. + * + * @param key the configuration key + * @return the configuration value associated with the key, guaranteed to be non-null + * @throws NoSuchElementException if the key is not found in the configuration + * @throws IllegalStateException if the key exists but its value is null + */ + default String getNonNull(String key) throws NoSuchElementException, IllegalStateException { + final String value = get(key); + if (value == null) throw new IllegalStateException(String.format("%s is null", key)); + return value; + } +} From 8d4a24f943cf20ef2b8695f10df8010b0a9aee8a Mon Sep 17 00:00:00 2001 From: Scott Sandre Date: Wed, 23 Oct 2024 15:46:37 -0700 Subject: [PATCH 03/12] Create AbstractCommitCoordinatorBuilder.java --- .../AbstractCommitCoordinatorBuilder.java | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/AbstractCommitCoordinatorBuilder.java diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/AbstractCommitCoordinatorBuilder.java b/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/AbstractCommitCoordinatorBuilder.java new file mode 100644 index 0000000000..19b9885bb0 --- /dev/null +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/AbstractCommitCoordinatorBuilder.java @@ -0,0 +1,63 @@ +/* + * Copyright (2024) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.kernel.coordinatedcommits; + +import io.delta.kernel.annotation.Evolving; +import io.delta.kernel.config.ConfigurationProvider; +import java.util.Map; + +/** + * A builder class to create a {@link CommitCoordinatorClient} (CCC). + * + *

Subclasses of this class must provide a no-argument constructor. + * + *

Note: These builder classed are optional. The {@link + * io.delta.kernel.engine.Engine#getCommitCoordinatorClient} API does not prescribe how to create + * the underlying CCC and does not require a builder. + * + *

The benefit of implementing a builder for your CCC is that your Engine may then invoke {@link + * CommitCoordinatorUtils#buildCommitCoordinatorClient} to (1) instantiate your builder, and then + * (2) build a new CCC via reflection. + * + *

In short, this builder provides the ability for users to specify at runtime which builder + * implementation (e.g. x.y.z.FooCCBuilder) to use for a given commit coordinator name (e.g. "foo"), + * without imposing any restrictions on how to construct CCCs (i.e. these builders are optional). + */ +@Evolving +public abstract class AbstractCommitCoordinatorBuilder { + /** Subclasses of this class must provide a no-argument constructor. */ + public AbstractCommitCoordinatorBuilder() {} + + /** + * @return the commit coordinator name + */ + public abstract String getName(); + + /** + * Build the {@link CommitCoordinatorClient}. This may be a new instance or a cached one. + * + * @param sessionConfig The session-level configuration that may represent environment-specific + * configurations such as {@code HadoopConf} or {@code SparkConf}. This sessionConfig can be + * used by builders to lookup per-session configuration values unique to that builder. + * @param commitCoordinatorConf the parsed value of the Delta table property {@link + * io.delta.kernel.internal.TableConfig#COORDINATED_COMMITS_TABLE_CONF} and represents the + * configuration properties for describing the Delta table to the commit-coordinator. + * @return the {@link CommitCoordinatorClient} + */ + public abstract CommitCoordinatorClient build( + ConfigurationProvider sessionConfig, Map commitCoordinatorConf); +} From 8e9f0e744ea20ca016b1893db37333b0392954ce Mon Sep 17 00:00:00 2001 From: Scott Sandre Date: Wed, 23 Oct 2024 15:47:04 -0700 Subject: [PATCH 04/12] Create CommitCoordinatorUtils and update DeltaErrors --- .../CommitCoordinatorUtils.java | 85 +++++++++++++++++++ .../io/delta/kernel/internal/DeltaErrors.java | 23 ++++- 2 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/CommitCoordinatorUtils.java diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/CommitCoordinatorUtils.java b/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/CommitCoordinatorUtils.java new file mode 100644 index 0000000000..55559057cd --- /dev/null +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/CommitCoordinatorUtils.java @@ -0,0 +1,85 @@ +/* + * Copyright (2024) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.kernel.coordinatedcommits; + +import io.delta.kernel.config.ConfigurationProvider; +import io.delta.kernel.internal.DeltaErrors; +import java.lang.reflect.InvocationTargetException; +import java.util.Map; +import java.util.NoSuchElementException; + +/** Various public utility methods related to Coordinated Commits. */ +public class CommitCoordinatorUtils { + private CommitCoordinatorUtils() {} + + /** + * Builds the underlying {@link CommitCoordinatorClient} associated with the given commit + * coordinator name. + * + *

    + *
  • Determines the specific builder configuration lookup key. + *
  • Grabs the corresponding builder className for that key from the provided sessionConfig. + *
  • Instantiates a new instance of that {@link AbstractCommitCoordinatorBuilder}. + *
  • Invokes the builder's build method, passing along the sessionConfig and commit + * coordinator config map. + *
+ * + * @param commitCoordinatorName the commit coordinator name + * @param sessionConfig The session-level configuration that may represent environment-specific + * configurations such as {@code HadoopConf} or {@code SparkConf}. This sessionConfig is used + * to look up the right builder className to instantiate for the given commit coordinator + * name. It can also be used by builders to lookup per-session configuration values unique to + * that builder. + * @param commitCoordinatorConf the parsed value of the Delta table property {@link + * io.delta.kernel.internal.TableConfig#COORDINATED_COMMITS_TABLE_CONF} and represents the + * configuration properties for describing the Delta table to the commit-coordinator. + * @return the {@link CommitCoordinatorClient} corresponding to the given commit coordinator name + */ + public static CommitCoordinatorClient buildCommitCoordinatorClient( + String commitCoordinatorName, + ConfigurationProvider sessionConfig, + Map commitCoordinatorConf) { + final String builderConfKey = getCommitCoordinatorBuilderConfKey(commitCoordinatorName); + + final String builderClassName; + try { + builderClassName = sessionConfig.getNonNull(builderConfKey); + } catch (NoSuchElementException | IllegalStateException ex) { + throw DeltaErrors.unknownCommitCoordinator(commitCoordinatorName, builderConfKey); + } + + try { + return Class.forName(builderClassName) + .asSubclass(AbstractCommitCoordinatorBuilder.class) + .getConstructor() + .newInstance() + .build(sessionConfig, commitCoordinatorConf); + } catch (ClassNotFoundException + | NoSuchMethodException + | InstantiationException + | IllegalAccessException + | InvocationTargetException e) { + throw DeltaErrors.couldNotInstantiateCommitCoordinatorClient( + commitCoordinatorName, builderClassName, e); + } + } + + /** Returns the builder configuration key for the given commit coordinator name */ + public static String getCommitCoordinatorBuilderConfKey(String ccName) { + return String.format("io.delta.kernel.commitCoordinatorBuilder.%s.impl", ccName); + } +} diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaErrors.java b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaErrors.java index 6a3dc23fd4..e86f61ba71 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaErrors.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/internal/DeltaErrors.java @@ -138,7 +138,26 @@ public static KernelException invalidVersionRange(long startVersion, long endVer return new KernelException(message); } - /* ------------------------ PROTOCOL EXCEPTIONS ----------------------------- */ + /* ------------------------ COORDINATED COMMITS EXCEPTIONS - START ------------------------ */ + + public static KernelException unknownCommitCoordinator(String ccName, String ccBuilderConfKey) { + return new KernelException( + String.format( + "Unknown commit coordinator: '%s'. Please ensure that session config '%s' is set.", + ccName, ccBuilderConfKey)); + } + + public static KernelException couldNotInstantiateCommitCoordinatorClient( + String ccName, String ccBuilderClassName, Exception ex) { + return new KernelException( + String.format( + "Could not instantiate Commit Coordinator Client for '%s' using builder class '%s'.", + ccName, ccBuilderClassName), + ex); + } + + /* ------------------------ COORDINATED COMMITS EXCEPTIONS - END ------------------------ */ + /* ------------------------ PROTOCOL EXCEPTIONS - START ----------------------------- */ public static KernelException unsupportedReaderProtocol( String tablePath, int tableReaderVersion) { @@ -179,6 +198,8 @@ public static KernelException unsupportedWriterFeature(String tablePath, String return new KernelException(message); } + /* ------------------------ PROTOCOL EXCEPTIONS - END ----------------------------- */ + public static KernelException columnInvariantsNotSupported() { String message = "This version of Delta Kernel does not support writing to tables with " From 40f39ce6c4ed97c57c924b944ef5bb9f2286468f Mon Sep 17 00:00:00 2001 From: Scott Sandre Date: Fri, 25 Oct 2024 13:43:24 -0700 Subject: [PATCH 05/12] javafmt --- .../java/io/delta/kernel/config/ConfigurationProvider.java | 6 +++--- .../AbstractCommitCoordinatorBuilder.java | 4 +--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/config/ConfigurationProvider.java b/kernel/kernel-api/src/main/java/io/delta/kernel/config/ConfigurationProvider.java index 98dabf0469..b462b9fb7d 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/config/ConfigurationProvider.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/config/ConfigurationProvider.java @@ -54,8 +54,8 @@ public interface ConfigurationProvider { * @throws IllegalStateException if the key exists but its value is null */ default String getNonNull(String key) throws NoSuchElementException, IllegalStateException { - final String value = get(key); - if (value == null) throw new IllegalStateException(String.format("%s is null", key)); - return value; + final String value = get(key); + if (value == null) throw new IllegalStateException(String.format("%s is null", key)); + return value; } } diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/AbstractCommitCoordinatorBuilder.java b/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/AbstractCommitCoordinatorBuilder.java index 19b9885bb0..2fa481abcf 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/AbstractCommitCoordinatorBuilder.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/AbstractCommitCoordinatorBuilder.java @@ -42,9 +42,7 @@ public abstract class AbstractCommitCoordinatorBuilder { /** Subclasses of this class must provide a no-argument constructor. */ public AbstractCommitCoordinatorBuilder() {} - /** - * @return the commit coordinator name - */ + /** @return the commit coordinator name */ public abstract String getName(); /** From 81ddff4d2e8cb7aa3e75bf3c5b084be43f7fd72a Mon Sep 17 00:00:00 2001 From: Scott Sandre Date: Fri, 1 Nov 2024 11:42:57 -0700 Subject: [PATCH 06/12] minor updates to CCC javadoc --- .../coordinatedcommits/CommitCoordinatorClient.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/CommitCoordinatorClient.java b/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/CommitCoordinatorClient.java index 5870f570f8..40eebfde2c 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/CommitCoordinatorClient.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/CommitCoordinatorClient.java @@ -149,10 +149,11 @@ GetCommitsResponse getCommits( * * @param engine The {@link Engine} instance to use, if needed. * @param tableDescriptor The descriptor for the table. - * @param version The version until which the commit coordinator client should backfill. - * @param lastKnownBackfilledVersion The last known version that was backfilled before this API - * was called. If it is {@link Optional#empty()}, then the commit coordinator client should - * backfill from the beginning of the table. + * @param version The version (inclusive) until which the commit coordinator client should + * backfill. + * @param lastKnownBackfilledVersion The last known version (inclusive) that was backfilled before + * this API was called. If it is {@link Optional#empty()}, then the commit coordinator client + * should backfill from the beginning of the table. * @throws IOException if there is an IO error while backfilling the commits. */ void backfillToVersion( From f49a7f3f66f61d1450390e6824934a192b0c542a Mon Sep 17 00:00:00 2001 From: Scott Sandre Date: Fri, 1 Nov 2024 11:43:05 -0700 Subject: [PATCH 07/12] Refactor utils to AbstractCommitCoordinatorBuilder --- .../AbstractCommitCoordinatorBuilder.java | 82 +++++++++++++++++-- 1 file changed, 77 insertions(+), 5 deletions(-) diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/AbstractCommitCoordinatorBuilder.java b/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/AbstractCommitCoordinatorBuilder.java index 2fa481abcf..035e3dfc2c 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/AbstractCommitCoordinatorBuilder.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/AbstractCommitCoordinatorBuilder.java @@ -18,27 +18,98 @@ import io.delta.kernel.annotation.Evolving; import io.delta.kernel.config.ConfigurationProvider; +import io.delta.kernel.internal.DeltaErrors; +import java.lang.reflect.InvocationTargetException; import java.util.Map; +import java.util.NoSuchElementException; /** * A builder class to create a {@link CommitCoordinatorClient} (CCC). * *

Subclasses of this class must provide a no-argument constructor. * - *

Note: These builder classed are optional. The {@link + *

Note: These builder classes are optional. The {@link * io.delta.kernel.engine.Engine#getCommitCoordinatorClient} API does not prescribe how to create * the underlying CCC and does not require a builder. * *

The benefit of implementing a builder for your CCC is that your Engine may then invoke {@link - * CommitCoordinatorUtils#buildCommitCoordinatorClient} to (1) instantiate your builder, and then - * (2) build a new CCC via reflection. + * #buildCommitCoordinatorClient} to (1) instantiate your builder, and then (2) build a new CCC via + * reflection. * *

In short, this builder provides the ability for users to specify at runtime which builder * implementation (e.g. x.y.z.FooCCBuilder) to use for a given commit coordinator name (e.g. "foo"), * without imposing any restrictions on how to construct CCCs (i.e. these builders are optional). + * + * @since 3.3.0 */ @Evolving public abstract class AbstractCommitCoordinatorBuilder { + + /////////////////////////// + // Static Helper Methods // + /////////////////////////// + + /** + * Builds the underlying {@link CommitCoordinatorClient} associated with the given commit + * coordinator name. + * + *

    + *
  • Determines the specific builder configuration lookup key. + *
  • Grabs the corresponding builder className for that key from the provided sessionConfig. + *
  • Instantiates a new instance of that {@link AbstractCommitCoordinatorBuilder}. + *
  • Invokes the builder's build method, passing along the sessionConfig and commit + * coordinator config map. + *
+ * + * @param commitCoordinatorName the commit coordinator name + * @param sessionConfig The session-level configuration that may represent environment-specific + * configurations such as {@code HadoopConf} or {@code SparkConf}. This sessionConfig is used + * to look up the right builder className to instantiate for the given commit coordinator + * name. It can also be used by builders to lookup per-session configuration values unique to + * that builder. + * @param commitCoordinatorConf the parsed value of the Delta table property {@link + * io.delta.kernel.internal.TableConfig#COORDINATED_COMMITS_TABLE_CONF} and represents the + * configuration properties for describing the Delta table to the commit-coordinator. + * @return the {@link CommitCoordinatorClient} corresponding to the given commit coordinator name + */ + public static CommitCoordinatorClient buildCommitCoordinatorClient( + String commitCoordinatorName, + ConfigurationProvider sessionConfig, + Map commitCoordinatorConf) { + final String builderConfKey = getCommitCoordinatorBuilderConfKey(commitCoordinatorName); + + final String builderClassName; + try { + builderClassName = sessionConfig.getNonNull(builderConfKey); + } catch (NoSuchElementException | IllegalStateException ex) { + throw DeltaErrors.unknownCommitCoordinator(commitCoordinatorName, builderConfKey); + } + + try { + return Class.forName(builderClassName) + .asSubclass(AbstractCommitCoordinatorBuilder.class) + .getConstructor() + .newInstance() + .build(sessionConfig, commitCoordinatorConf); + } catch (ClassNotFoundException + | NoSuchMethodException + | InstantiationException + | IllegalAccessException + | InvocationTargetException e) { + throw DeltaErrors.couldNotInstantiateCommitCoordinatorClient( + commitCoordinatorName, builderClassName, e); + } + } + + /** Returns the builder configuration key for the given commit coordinator name */ + public static String getCommitCoordinatorBuilderConfKey(String ccName) { + return String.format("io.delta.kernel.commitCoordinatorBuilder.%s.impl", ccName); + } + + //////////////////// + // Member Methods // + //////////////////// + /** Subclasses of this class must provide a no-argument constructor. */ public AbstractCommitCoordinatorBuilder() {} @@ -52,8 +123,9 @@ public AbstractCommitCoordinatorBuilder() {} * configurations such as {@code HadoopConf} or {@code SparkConf}. This sessionConfig can be * used by builders to lookup per-session configuration values unique to that builder. * @param commitCoordinatorConf the parsed value of the Delta table property {@link - * io.delta.kernel.internal.TableConfig#COORDINATED_COMMITS_TABLE_CONF} and represents the - * configuration properties for describing the Delta table to the commit-coordinator. + * io.delta.kernel.internal.TableConfig#COORDINATED_COMMITS_COORDINATOR_CONF} which represents + * the configuration properties the commit coordinator client needs to be correctly + * instantiated and to communicate to the commit coordinator. * @return the {@link CommitCoordinatorClient} */ public abstract CommitCoordinatorClient build( From 67b18db03dbb8a555867dc3ea690dfba0ce608c7 Mon Sep 17 00:00:00 2001 From: Scott Sandre Date: Fri, 1 Nov 2024 11:43:08 -0700 Subject: [PATCH 08/12] Delete CommitCoordinatorUtils.java --- .../CommitCoordinatorUtils.java | 85 ------------------- 1 file changed, 85 deletions(-) delete mode 100644 kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/CommitCoordinatorUtils.java diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/CommitCoordinatorUtils.java b/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/CommitCoordinatorUtils.java deleted file mode 100644 index 55559057cd..0000000000 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/CommitCoordinatorUtils.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (2024) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.kernel.coordinatedcommits; - -import io.delta.kernel.config.ConfigurationProvider; -import io.delta.kernel.internal.DeltaErrors; -import java.lang.reflect.InvocationTargetException; -import java.util.Map; -import java.util.NoSuchElementException; - -/** Various public utility methods related to Coordinated Commits. */ -public class CommitCoordinatorUtils { - private CommitCoordinatorUtils() {} - - /** - * Builds the underlying {@link CommitCoordinatorClient} associated with the given commit - * coordinator name. - * - *
    - *
  • Determines the specific builder configuration lookup key. - *
  • Grabs the corresponding builder className for that key from the provided sessionConfig. - *
  • Instantiates a new instance of that {@link AbstractCommitCoordinatorBuilder}. - *
  • Invokes the builder's build method, passing along the sessionConfig and commit - * coordinator config map. - *
- * - * @param commitCoordinatorName the commit coordinator name - * @param sessionConfig The session-level configuration that may represent environment-specific - * configurations such as {@code HadoopConf} or {@code SparkConf}. This sessionConfig is used - * to look up the right builder className to instantiate for the given commit coordinator - * name. It can also be used by builders to lookup per-session configuration values unique to - * that builder. - * @param commitCoordinatorConf the parsed value of the Delta table property {@link - * io.delta.kernel.internal.TableConfig#COORDINATED_COMMITS_TABLE_CONF} and represents the - * configuration properties for describing the Delta table to the commit-coordinator. - * @return the {@link CommitCoordinatorClient} corresponding to the given commit coordinator name - */ - public static CommitCoordinatorClient buildCommitCoordinatorClient( - String commitCoordinatorName, - ConfigurationProvider sessionConfig, - Map commitCoordinatorConf) { - final String builderConfKey = getCommitCoordinatorBuilderConfKey(commitCoordinatorName); - - final String builderClassName; - try { - builderClassName = sessionConfig.getNonNull(builderConfKey); - } catch (NoSuchElementException | IllegalStateException ex) { - throw DeltaErrors.unknownCommitCoordinator(commitCoordinatorName, builderConfKey); - } - - try { - return Class.forName(builderClassName) - .asSubclass(AbstractCommitCoordinatorBuilder.class) - .getConstructor() - .newInstance() - .build(sessionConfig, commitCoordinatorConf); - } catch (ClassNotFoundException - | NoSuchMethodException - | InstantiationException - | IllegalAccessException - | InvocationTargetException e) { - throw DeltaErrors.couldNotInstantiateCommitCoordinatorClient( - commitCoordinatorName, builderClassName, e); - } - } - - /** Returns the builder configuration key for the given commit coordinator name */ - public static String getCommitCoordinatorBuilderConfKey(String ccName) { - return String.format("io.delta.kernel.commitCoordinatorBuilder.%s.impl", ccName); - } -} From ee516046a2b5654b59b4ef97f18207e55f14574a Mon Sep 17 00:00:00 2001 From: Scott Sandre Date: Fri, 1 Nov 2024 11:45:21 -0700 Subject: [PATCH 09/12] Update TableDescriptor javadoc --- .../io/delta/kernel/coordinatedcommits/TableDescriptor.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/TableDescriptor.java b/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/TableDescriptor.java index 827b31db91..468b0f6e43 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/TableDescriptor.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/TableDescriptor.java @@ -28,6 +28,8 @@ * The complete descriptor of a Coordinated Commits (CC) Delta table, including its logPath, table * identifier, and table CC configuration. * + *

The table identifier is not required for path-based tables. + * * @since 3.3.0 */ @Evolving From 3d99aaa4e8ac8511a43d1a1efce3cc4c8d235570 Mon Sep 17 00:00:00 2001 From: Scott Sandre Date: Fri, 1 Nov 2024 11:46:35 -0700 Subject: [PATCH 10/12] Delete Nullable.java --- .../io/delta/kernel/annotation/Nullable.java | 27 ------------------- 1 file changed, 27 deletions(-) delete mode 100644 kernel/kernel-api/src/main/java/io/delta/kernel/annotation/Nullable.java diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/annotation/Nullable.java b/kernel/kernel-api/src/main/java/io/delta/kernel/annotation/Nullable.java deleted file mode 100644 index 23e32a0f5c..0000000000 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/annotation/Nullable.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (2024) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.delta.kernel.annotation; - -import java.lang.annotation.*; - -/** - * Annotation to indicate that a field, method parameter, method return value, or local variable may - * be null. - */ -@Documented -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER, ElementType.LOCAL_VARIABLE}) -public @interface Nullable {} From a2e7768c449b358e7d59de1a7238ec5c50a741ba Mon Sep 17 00:00:00 2001 From: Scott Sandre Date: Fri, 1 Nov 2024 14:28:15 -0700 Subject: [PATCH 11/12] Update AbstractCommitCoordinatorBuilder javadocs --- .../AbstractCommitCoordinatorBuilder.java | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/AbstractCommitCoordinatorBuilder.java b/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/AbstractCommitCoordinatorBuilder.java index 035e3dfc2c..4376ff0820 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/AbstractCommitCoordinatorBuilder.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/AbstractCommitCoordinatorBuilder.java @@ -24,7 +24,8 @@ import java.util.NoSuchElementException; /** - * A builder class to create a {@link CommitCoordinatorClient} (CCC). + * A builder class to create a {@link CommitCoordinatorClient} (CCC). Engines may choose not to use + * this builder and to instantiate CCCs directly. * *

Subclasses of this class must provide a no-argument constructor. * @@ -36,9 +37,17 @@ * #buildCommitCoordinatorClient} to (1) instantiate your builder, and then (2) build a new CCC via * reflection. * - *

In short, this builder provides the ability for users to specify at runtime which builder - * implementation (e.g. x.y.z.FooCCBuilder) to use for a given commit coordinator name (e.g. "foo"), - * without imposing any restrictions on how to construct CCCs (i.e. these builders are optional). + *

From a user perspective, this means users just need to: + * + *

    + *
  1. specify the commit coordinator name -> commit coordinator builder mapping in their Engine's + * configuration (e.g. "foo" -> "x.y.z.FooCCBuilder"). Note that Engine is then expected to + * include that mapping in the session-level configuration that is passed in to the {@link + * #buildCommitCoordinatorClient} API call. + *
  2. include the x.y.z.FooCCBuilder and respective client implementation on their classpath + *
+ * + * and then the client can be instantiated at runtime using reflection. * * @since 3.3.0 */ @@ -68,8 +77,9 @@ public abstract class AbstractCommitCoordinatorBuilder { * name. It can also be used by builders to lookup per-session configuration values unique to * that builder. * @param commitCoordinatorConf the parsed value of the Delta table property {@link - * io.delta.kernel.internal.TableConfig#COORDINATED_COMMITS_TABLE_CONF} and represents the - * configuration properties for describing the Delta table to the commit-coordinator. + * io.delta.kernel.internal.TableConfig#COORDINATED_COMMITS_COORDINATOR_CONF} which represents + * the configuration properties the commit coordinator client needs to be correctly + * instantiated and to communicate to the commit coordinator. * @return the {@link CommitCoordinatorClient} corresponding to the given commit coordinator name */ public static CommitCoordinatorClient buildCommitCoordinatorClient( From 2e89c294192dadad01e48b06be26a8e7b9810b42 Mon Sep 17 00:00:00 2001 From: Scott Sandre Date: Fri, 1 Nov 2024 14:57:41 -0700 Subject: [PATCH 12/12] Fix javadoc javafmt error --- .../coordinatedcommits/AbstractCommitCoordinatorBuilder.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/AbstractCommitCoordinatorBuilder.java b/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/AbstractCommitCoordinatorBuilder.java index 4376ff0820..fb05a8fe6f 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/AbstractCommitCoordinatorBuilder.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/coordinatedcommits/AbstractCommitCoordinatorBuilder.java @@ -40,8 +40,8 @@ *

From a user perspective, this means users just need to: * *

    - *
  1. specify the commit coordinator name -> commit coordinator builder mapping in their Engine's - * configuration (e.g. "foo" -> "x.y.z.FooCCBuilder"). Note that Engine is then expected to + *
  2. specify the commit coordinator name to commit coordinator builder mapping in their Engine's + * configuration (e.g. "foo" to "x.y.z.FooCCBuilder"). Note that Engine is then expected to * include that mapping in the session-level configuration that is passed in to the {@link * #buildCommitCoordinatorClient} API call. *
  3. include the x.y.z.FooCCBuilder and respective client implementation on their classpath