Skip to content

Commit f84c829

Browse files
committed
feat: add example of reading parquet from s3
1 parent 217ede8 commit f84c829

File tree

1 file changed

+22
-0
lines changed

1 file changed

+22
-0
lines changed

examples/sql-parquet-s3.py

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import os
2+
import datafusion
3+
from datafusion.object_store import AmazonS3
4+
5+
region = "us-east-1"
6+
bucket_name = "yellow-trips"
7+
8+
s3 = AmazonS3(
9+
bucket_name=bucket_name,
10+
region=region,
11+
access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),
12+
secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"),
13+
)
14+
15+
ctx = datafusion.SessionContext()
16+
path = f"s3://{bucket_name}/"
17+
ctx.register_object_store(path, s3)
18+
19+
ctx.register_parquet("trips", path)
20+
21+
df = ctx.sql("select count(passenger_count) from trips")
22+
df.show()

0 commit comments

Comments
 (0)