Skip to content

Commit

Permalink
[Kernel] Add Default client implementations
Browse files Browse the repository at this point in the history 
Following client implementations for the default module are added:

 * `JsonHandler`
 * `ExpressionHandler`
 * `FileSystemClient`

and the supporting classes.
  • Loading branch information
@vkorukanti
vkorukanti committed Jun 21, 2023
1 parent cb89436 commit 0667019
Show file tree
Hide file tree
Showing 29 changed files with 2,548 additions and 46 deletions.
10 changes: 2 additions & 8 deletions kernel/build.sbt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,17 +78,11 @@ lazy val kernelDefault = (project in file("kernel-default"))
scalaStyleSettings,
releaseSettings,
libraryDependencies ++= Seq(
"org.apache.hadoop" % "hadoop-client-api" % hadoopVersion, // Configuration, Path
"io.delta" % "delta-storage" % deltaStorageVersion, // LogStore
"com.fasterxml.jackson.core" % "jackson-databind" % "2.13.5", // ObjectMapper
"org.apache.hadoop" % "hadoop-client-runtime" % hadoopVersion,
"com.fasterxml.jackson.core" % "jackson-databind" % "2.13.5",
"org.apache.parquet" % "parquet-hadoop" % "1.12.3",

"org.scalatest" %% "scalatest" % scalaTestVersion % "test",
"io.delta" %% "delta-core" % deltaSparkVersion % "test",
"org.apache.spark" %% "spark-sql" % sparkVersion % "test", // SparkSession
"org.apache.spark" %% "spark-sql" % sparkVersion % "test" classifier "tests",
"org.apache.spark" %% "spark-core" % sparkVersion % "test" classifier "tests",
"org.apache.spark" %% "spark-catalyst" % sparkVersion % "test" classifier "tests",
"junit" % "junit" % "4.11" % "test",
"com.novocode" % "junit-interface" % "0.11" % "test"
)
Expand Down
30 changes: 27 additions & 3 deletions kernel/kernel-api/src/main/java/io/delta/kernel/data/ColumnarBatch.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
import io.delta.kernel.types.StructType;
import io.delta.kernel.utils.CloseableIterator;

import io.delta.kernel.internal.ColumnarBatchRow;

/**
* Represents zero or more rows of records with same schema type.
*/
Expand Down Expand Up @@ -55,8 +57,30 @@ default ColumnarBatch slice(int start, int end) {
/**
* @return iterator of {@link Row}s in this batch
*/
default CloseableIterator<Row> getRows() {
// TODO needs io.delta.kernel.internal.ColumnarBatchRow
throw new UnsupportedOperationException("Not yet implemented!");
default CloseableIterator<Row> getRows()
{
final ColumnarBatch batch = this;
return new CloseableIterator<Row>()
{
int rowId = 0;
int maxRowId = getSize();

@Override
public boolean hasNext()
{
return rowId < maxRowId;
}

@Override
public Row next()
{
Row row = new ColumnarBatchRow(batch, rowId);
rowId += 1;
return row;
}

@Override
public void close() {}
};
}
}
33 changes: 32 additions & 1 deletion kernel/kernel-api/src/main/java/io/delta/kernel/data/Row.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@
/**
* Represent a single record
*/
public interface Row {
public interface Row
{

/**
* @return Schema of the record.
Expand All @@ -43,6 +44,18 @@ public interface Row {
*/
boolean getBoolean(int ordinal);

/**
* Return byte value of the column located at the given ordinal.
* Throws error if the column at given ordinal is not of boolean type,
*/
byte getByte(int ordinal);

/**
* Return short value of the column located at the given ordinal.
* Throws error if the column at given ordinal is not of boolean type,
*/
short getShort(int ordinal);

/**
* Return integer value of the column located at the given ordinal.
* Throws error if the column at given ordinal is not of integer type,
Expand All @@ -55,12 +68,30 @@ public interface Row {
*/
long getLong(int ordinal);

/**
* Return float value of the column located at the given ordinal.
* Throws error if the column at given ordinal is not of long type,
*/
float getFloat(int ordinal);

/**
* Return double value of the column located at the given ordinal.
* Throws error if the column at given ordinal is not of long type,
*/
double getDouble(int ordinal);

/**
* Return string value of the column located at the given ordinal.
* Throws error if the column at given ordinal is not of varchar type,
*/
String getString(int ordinal);

/**
* Return binary value of the column located at the given ordinal.
* Throws error if the column at given ordinal is not of varchar type,
*/
byte[] getBinary(int ordinal);

/**
* Return struct value of the column located at the given ordinal.
* Throws error if the column at given ordinal is not of struct type,
Expand Down
153 changes: 123 additions & 30 deletions kernel/kernel-api/src/main/java/io/delta/kernel/expressions/Literal.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,108 +16,201 @@

package io.delta.kernel.expressions;

import java.sql.Date;
import java.sql.Timestamp;
import java.util.Objects;

import io.delta.kernel.data.Row;
import io.delta.kernel.types.ArrayType;
import io.delta.kernel.types.BinaryType;
import io.delta.kernel.types.BooleanType;
import io.delta.kernel.types.ByteType;
import io.delta.kernel.types.DataType;
import io.delta.kernel.types.DateType;
import io.delta.kernel.types.DoubleType;
import io.delta.kernel.types.FloatType;
import io.delta.kernel.types.IntegerType;
import io.delta.kernel.types.LongType;
import io.delta.kernel.types.MapType;
import io.delta.kernel.types.ShortType;
import io.delta.kernel.types.StringType;
import io.delta.kernel.types.StructType;
import io.delta.kernel.types.TimestampType;

/**
* A literal value.
* <p>
* Only supports primitive data types, see
* <a href="https://github.com/delta-io/delta/blob/master/PROTOCOL.md#primitive-types">Delta Transaction Log Protocol: Primitive Types</a>.
*/
public final class Literal extends LeafExpression {

////////////////////////////////////////////////////////////////////////////////
// Static Fields / Methods
////////////////////////////////////////////////////////////////////////////////

public final class Literal extends LeafExpression
{
public static final Literal TRUE = Literal.of(true);
public static final Literal FALSE = Literal.of(false);

/**
* Create an integer {@link Literal} object
* @param value integer value
* @return a {@link Literal} with data type {@link IntegerType}
*/
public static Literal of(int value) {
return new Literal(value, IntegerType.INSTANCE);
}

/**
* Create a boolean {@link Literal} object
*
* @param value boolean value
* @return a {@link Literal} with data type {@link BooleanType}
*/
public static Literal of(boolean value) {
public static Literal of(boolean value)
{
return new Literal(value, BooleanType.INSTANCE);
}

/**
* @return a {@link Literal} with data type {@link ByteType}
*/
public static Literal of(byte value)
{
return new Literal(value, ByteType.INSTANCE);
}

/**
* @return a {@link Literal} with data type {@link ShortType}
*/
public static Literal of(short value)
{
return new Literal(value, ShortType.INSTANCE);
}

/**
* Create an integer {@link Literal} object
*
* @param value integer value
* @return a {@link Literal} with data type {@link IntegerType}
*/
public static Literal of(int value)
{
return new Literal(value, IntegerType.INSTANCE);
}

/**
* Create a long {@link Literal} object
*
* @param value long value
* @return a {@link Literal} with data type {@link LongType}
*/
public static Literal of(long value) {
public static Literal of(long value)
{
return new Literal(value, LongType.INSTANCE);
}

/**
* @return a {@link Literal} with data type {@link FloatType}
*/
public static Literal of(float value)
{
return new Literal(value, FloatType.INSTANCE);
}

/**
* @return a {@link Literal} with data type {@link DoubleType}
*/
public static Literal of(double value)
{
return new Literal(value, DoubleType.INSTANCE);
}

/**
* Create a string {@link Literal} object
*
* @param value string value
* @return a {@link Literal} with data type {@link StringType}
*/
public static Literal of(String value) {
public static Literal of(String value)
{
return new Literal(value, StringType.INSTANCE);
}

////////////////////////////////////////////////////////////////////////////////
// Instance Fields / Methods
////////////////////////////////////////////////////////////////////////////////
/**
* @return a {@link Literal} with data type {@link BinaryType}
*/
public static Literal of(byte[] value)
{
return new Literal(value, BinaryType.INSTANCE);
}

/**
* @return a {@link Literal} with data type {@link DateType}
*/
public static Literal of(Date value)
{
return new Literal(value, DateType.INSTANCE);
}

/**
* @return a {@link Literal} with data type {@link TimestampType}
*/
public static Literal of(Timestamp value)
{
return new Literal(value, TimestampType.INSTANCE);
}

/**
* @return a null {@link Literal} with the given data type
*/
public static Literal ofNull(DataType dataType)
{
if (dataType instanceof ArrayType
|| dataType instanceof MapType
|| dataType instanceof StructType) {
throw new IllegalArgumentException(
dataType + " is an invalid data type for Literal.");
}
return new Literal(null, dataType);
}

private final Object value;
private final DataType dataType;

private Literal(Object value, DataType dataType) {
private Literal(Object value, DataType dataType)
{
this.value = value;
this.dataType = dataType;
}

public Object value() {
public Object value()
{
return value;
}

@Override
public Object eval(Row record) {
public Object eval(Row record)
{
return value;
}

@Override
public DataType dataType() {
public DataType dataType()
{
return dataType;
}

@Override
public String toString() {
public String toString()
{
return String.valueOf(value);
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
public boolean equals(Object o)
{
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
Literal literal = (Literal) o;
return Objects.equals(value, literal.value) &&
Objects.equals(dataType, literal.dataType);
}

@Override
public int hashCode() {
public int hashCode()
{
return Objects.hash(value, dataType);
}
}
Loading

0 comments on commit 0667019

Please sign in to comment.