increase chunk-size to 10000 (#142)

CartoDB · Jun 24, 2024 · 211f029 · 211f029
1 parent 035fe4f
commit 211f029
Show file tree

Hide file tree

Showing 7 changed files with 12 additions and 12 deletions.
diff --git a/docs/source/user_guide/cli.rst b/docs/source/user_guide/cli.rst
@@ -163,10 +163,10 @@ due to excessive operations in the destination table:
 Exceeded rate limits: too many table update operations for this table. For more information, see https://cloud.google.com/bigquery/troubleshooting-errors
 ```
 
-The default chunk size is 1000 rows.
+The default chunk size is 10000 rows.
 
 For example, the following command uploads the raster in chunks
-of 2000 rows:
+of 20000 rows:
 
 .. code-block:: bash
 
@@ -175,7 +175,7 @@ of 2000 rows:
      --project my-gcp-project \
      --dataset my-bigquery-dataset \
      --table my-bigquery-table \
-     --chunk_size 1000
+     --chunk_size 20000
 
 
 

diff --git a/raster_loader/cli/bigquery.py b/raster_loader/cli/bigquery.py
@@ -57,7 +57,7 @@ def bigquery(args=None):
     multiple=True,
 )
 @click.option(
-    "--chunk_size", help="The number of blocks to upload in each chunk.", default=1000
+    "--chunk_size", help="The number of blocks to upload in each chunk.", default=10000
 )
 @click.option(
     "--overwrite",

diff --git a/raster_loader/cli/snowflake.py b/raster_loader/cli/snowflake.py
@@ -66,7 +66,7 @@ def snowflake(args=None):
     multiple=True,
 )
 @click.option(
-    "--chunk_size", help="The number of blocks to upload in each chunk.", default=1000
+    "--chunk_size", help="The number of blocks to upload in each chunk.", default=10000
 )
 @click.option(
     "--overwrite",

diff --git a/raster_loader/io/datawarehouse.py b/raster_loader/io/datawarehouse.py
@@ -64,7 +64,7 @@ def upload_raster(
         fqn: str,
         band: int = 1,
         band_name: str = None,
-        chunk_size: int = 1000,
+        chunk_size: int = 10000,
         overwrite: bool = False,
         append: bool = False,
     ):
@@ -80,7 +80,7 @@ def upload_raster(
         band_name : str, optional
             Name of the band
         chunk_size : int, optional
-            Number of blocks to upload in each chunk, by default 1000
+            Number of blocks to upload in each chunk, by default 10000
         overwrite : bool, optional
             Overwrite existing data in the table if it already exists, by default False
         append : bool, optional

diff --git a/raster_loader/io/snowflake.py b/raster_loader/io/snowflake.py
@@ -129,7 +129,7 @@ def upload_records(
             table_name=table,
             database=database,
             schema=schema,
-            chunk_size=1000,
+            chunk_size=10000,
             auto_create_table=True,
             overwrite=overwrite,
         )[0]

diff --git a/raster_loader/tests/bigquery/test_io.py b/raster_loader/tests/bigquery/test_io.py
@@ -568,7 +568,7 @@ def test_rasterio_to_table_with_chunk_size(*args, **kwargs):
     success = connector.upload_raster(
         os.path.join(fixtures_dir, "mosaic_cog.tif"),
         f"{BQ_PROJECT_ID}.{BQ_DATASET_ID}.{table_name}",
-        chunk_size=1000,
+        chunk_size=10000,
     )
 
     assert success
@@ -604,7 +604,7 @@ def test_rasterio_to_table_invalid_raster(*args, **kwargs):
         connector.upload_raster(
             os.path.join(fixtures_dir, "mosaic.tif"),
             f"{BQ_PROJECT_ID}.{BQ_DATASET_ID}.{table_name}",
-            chunk_size=1000,
+            chunk_size=10000,
         )
 
 

diff --git a/raster_loader/tests/snowflake/test_io.py b/raster_loader/tests/snowflake/test_io.py
@@ -546,7 +546,7 @@ def test_rasterio_to_table_with_chunk_size(*args, **kwargs):
     success = connector.upload_raster(
         os.path.join(fixtures_dir, "mosaic_cog.tif"),
         f"{SF_DATABASE}.{SF_SCHEMA}.{table_name}",
-        chunk_size=1000,
+        chunk_size=10000,
     )
 
     assert success
@@ -582,7 +582,7 @@ def test_rasterio_to_table_invalid_raster(*args, **kwargs):
         connector.upload_raster(
             os.path.join(fixtures_dir, "mosaic.tif"),
             f"{SF_DATABASE}.{SF_SCHEMA}.{table_name}",
-            chunk_size=1000,
+            chunk_size=10000,
         )