Skip to content

Commit

Permalink
Update StructField.java
Browse files Browse the repository at this point in the history
  • Loading branch information
vkorukanti committed Sep 26, 2024
1 parent c5f41b9 commit 7b8c844
Showing 1 changed file with 25 additions and 25 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -108,31 +108,6 @@ public String toString() {
"StructField(name=%s,type=%s,nullable=%s,metadata=%s)", name, dataType, nullable, metadata);
}

private List<Tuple2<String, String>> getNestedCollatedFields(DataType parent, String path) {
List<Tuple2<String, String>> nestedCollatedFields = new ArrayList<>();
if (parent instanceof StringType) {
StringType stringType = (StringType) parent;
if (!stringType
.getCollationIdentifier()
.equals(CollationIdentifier.fromString("SPARK.UTF8_BINARY"))) {
nestedCollatedFields.add(
new Tuple2<>(
path, ((StringType) parent).getCollationIdentifier().toStringWithoutVersion()));
}
} else if (parent instanceof MapType) {
nestedCollatedFields.addAll(
getNestedCollatedFields(((MapType) parent).getKeyType(), path + ".key"));
nestedCollatedFields.addAll(
getNestedCollatedFields(((MapType) parent).getValueType(), path + ".value"));
} else if (parent instanceof ArrayType) {
nestedCollatedFields.addAll(
getNestedCollatedFields(((ArrayType) parent).getElementType(), path + ".element"));
}
// We didn't check for StructType because we store the StringType's
// collation information in the nearest ancestor StructField's metadata when serializing.
return nestedCollatedFields;
}

@Override
public boolean equals(Object o) {
if (this == o) {
Expand All @@ -157,6 +132,31 @@ public StructField withNewMetadata(FieldMetadata metadata) {
return new StructField(name, dataType, nullable, metadata);
}

private List<Tuple2<String, String>> getNestedCollatedFields(DataType parent, String path) {
List<Tuple2<String, String>> nestedCollatedFields = new ArrayList<>();
if (parent instanceof StringType) {
StringType stringType = (StringType) parent;
if (!stringType
.getCollationIdentifier()
.equals(CollationIdentifier.fromString("SPARK.UTF8_BINARY"))) {
nestedCollatedFields.add(
new Tuple2<>(
path, ((StringType) parent).getCollationIdentifier().toStringWithoutVersion()));
}
} else if (parent instanceof MapType) {
nestedCollatedFields.addAll(
getNestedCollatedFields(((MapType) parent).getKeyType(), path + ".key"));
nestedCollatedFields.addAll(
getNestedCollatedFields(((MapType) parent).getValueType(), path + ".value"));
} else if (parent instanceof ArrayType) {
nestedCollatedFields.addAll(
getNestedCollatedFields(((ArrayType) parent).getElementType(), path + ".element"));
}
// We didn't check for StructType because we store the StringType's
// collation information in the nearest ancestor StructField's metadata when serializing.
return nestedCollatedFields;
}

/** Fetches collation metadata from nested collated fields. */
private FieldMetadata fetchCollationMetadata() {
List<Tuple2<String, String>> nestedCollatedFields = getNestedCollatedFields(dataType, name);
Expand Down

0 comments on commit 7b8c844

Please sign in to comment.