Skip to content

Commit 49da3a4

Browse files
[SPARK-48340][PYTHON][FOLLOWUP] Support TimestampNTZ infer schema miss prefer_timestamp_ntz
### What changes were proposed in this pull request? Add UT ### Why are the changes needed? ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Added UT ### Was this patch authored or co-authored using generative AI tooling? No Closes apache#46750 from AngersZhuuuu/SPARK-48340-FOLLOWUP. Lead-authored-by: Angerszhuuuu <[email protected]> Co-authored-by: Hyukjin Kwon <[email protected]> Signed-off-by: Hyukjin Kwon <[email protected]>
1 parent 90c1637 commit 49da3a4

File tree

1 file changed

+14
-1
lines changed

1 file changed

+14
-1
lines changed

python/pyspark/sql/tests/test_types.py

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,7 @@ def __init__(self):
192192
Row(a=1),
193193
Row("a")(1),
194194
A(),
195+
Row(b=Row(c=datetime.datetime(1970, 1, 1, 0, 0))),
195196
]
196197

197198
df = self.spark.createDataFrame([data])
@@ -214,6 +215,7 @@ def __init__(self):
214215
"struct<a:bigint>",
215216
"struct<a:bigint>",
216217
"struct<a:bigint>",
218+
"struct<b:struct<c:timestamp>>",
217219
]
218220
self.assertEqual(actual, expected)
219221

@@ -236,14 +238,25 @@ def __init__(self):
236238
Row(a=1),
237239
Row(a=1),
238240
Row(a=1),
241+
Row(b=Row(c=datetime.datetime(1970, 1, 1, 0, 0))),
239242
]
240243
self.assertEqual(actual, expected)
241244

242245
with self.sql_conf({"spark.sql.timestampType": "TIMESTAMP_NTZ"}):
243246
with self.sql_conf({"spark.sql.session.timeZone": "America/Sao_Paulo"}):
244-
df = self.spark.createDataFrame([(datetime.datetime(1970, 1, 1, 0, 0),)])
247+
data = [
248+
(
249+
datetime.datetime(1970, 1, 1, 0, 0),
250+
Row(a=Row(a=datetime.datetime(1970, 1, 1, 0, 0))),
251+
)
252+
]
253+
df = self.spark.createDataFrame(data)
245254
self.assertEqual(list(df.schema)[0].dataType.simpleString(), "timestamp_ntz")
246255
self.assertEqual(df.first()[0], datetime.datetime(1970, 1, 1, 0, 0))
256+
self.assertEqual(
257+
list(df.schema)[1].dataType.simpleString(), "struct<a:struct<a:timestamp_ntz>>"
258+
)
259+
self.assertEqual(df.first()[1], Row(a=Row(a=datetime.datetime(1970, 1, 1, 0, 0))))
247260

248261
df = self.spark.createDataFrame(
249262
[

0 commit comments

Comments
 (0)