adjust join for sqlite, prefilter csv and gpkg

get_zensus.sh

@@ -1,17 +1,15 @@
 #!/bin/bash
 
 ###################################################################################################
-# script to create 100m raster cells of Zensus 2011 data using the BKG 100m geogitter
-# required: postgres db, psql, shp2pgsql, GrassGIS (e.g. 7.6)
+# script to create 100m raster cells of Zensus 2011 data using the BKG 100m geogitter (GPKG)
+# required: gdal, awk, sqlite3 (cmd tools), GrassGIS (e.g. 7.8)
 # author: Hannes Blitza, [email protected]
 ###################################################################################################
 
-#zensus2011 datasource "Bevölkerung im 100 Meter-Gitter"
-#https://www.zensus2011.de/SharedDocs/Downloads/DE/Pressemitteilung/DemografischeGrunddaten/csv_Bevoelkerung_100m_Gitter.zip?__blob=publicationFile&v=3
-
 grass=grass78
+cwd=$(pwd)
 
-#download and unzip csv
+# #download and unzip csv
 zensus='https://www.zensus2011.de/SharedDocs/Downloads/DE/Pressemitteilung/DemografischeGrunddaten/csv_Bevoelkerung_100m_Gitter.zip?__blob=publicationFile&v=3'
 TMPFILE='zensus.zip'
 wget -c $zensus -O $TMPFILE
@@ -21,28 +19,42 @@ sed -i '1s/.*/\L&/' Zensus_Bevoelkerung_100m-Gitter.csv
 
 rm zensus.zip
 
-#download and unzip geogitter100m LAEA from BKG
+# extract entries for the grid N307E411 (Bonn) (10000 rows)
+awk -F ";" '$1 ~ /N307/' Zensus_Bevoelkerung_100m-Gitter.csv | awk -F ";" '$1 ~ /E411/' > zensus_subset_bonn.csv
+
+# download and unzip geogitter100m LAEA from BKG
 geogitter='https://daten.gdz.bkg.bund.de/produkte/sonstige/geogitter/aktuell/DE_Grid_ETRS89-LAEA_100m.gpkg.zip'
 TMPFILE=geogitter.zip
 wget -c $geogitter -O $TMPFILE
 unzip geogitter.zip -d .
+rm gegitter.zip
 
+# create new gpkg with filtered features
+ogr2ogr \
+    where id LIKE '%N307%E411%' \
+    -f geogitter_subset \
+    DE_Grid_ETRS89-LAEA_100m.gpkg
 
+# import csv to gpkg using sqlite3 https://sqlite.org/cli.html
+sqlite3
+# in sqlite 3
+.open geogitter_subset
+.separator ";"
+.import zensus_subset_Bonn.csv
+.quit
 
-# perform attribute join
-sudo -u postgres psql -f "join.sql";
+# perform table join
+ogrinfo geogitter_subset.gpkg -sql @join.sql
 
 # grass
 # create location
 $grass -c epsg:3035 -e ~/grassdata/3035_zensus/
 $grass ~/grassdata/3035_zensus/PERMANENT/
 # link postgres layer to grass
-# limited to 10000 polygons using where clause
-v.external input=/home/hannes/geodata/geogitter/DE_Grid_ETRS89-LAEA_100m.gpkg layer=de_grid_laea_100m output=geogitter2 where="id LIKE '%N307%E411%'"
-
+v.external input=$pwd/geogitter_subset.gpkg layer=de_grid_laea_100m output=geogitter_subset --overwrite
 # set region to layer
-g.region vector=gegitter -p
+g.region vector=gegitter_subset -p
 # rasterize
-v.to.rast input=geogitter type=area output=zensusraster use=attr attribute_column=einwohner where="einwohner > 20" memory=4000 --verbose
+v.to.rast input=geogitter_subset type=area output=zensusraster use=attr attribute_column=einwohner memory=4000 --verbose
 # export as GeoTiff
 r.out.gdal -m -v input=zensusraster output=zensusraster.tif format=GTiff createopt="COMPRESS=LZW" overviews=4

join.sql

@@ -1,7 +1,14 @@
-ALTER TABLE geogitter
-ADD COLUMN "einwohner" integer;
+ALTER TABLE de_grid_laea_100m ADD einwohner integer;
+
+UPDATE de_grid_laea_100m
+SET
+    einwohner = (SELECT zensusdata.einwohner
+                                FROM zensusdata
+                                WHERE zensusdata.gitter_id_100m = de_grid_laea_100m.id )
+WHERE
+    EXISTS (
+       SELECT *
+       FROM zensusdata
+       WHERE zensusdata.gitter_id_100m = de_grid_laea_100m.id
+   )
 
-UPDATE geogitter a
-SET einwohner = b.einwohner
-FROM zensusdata b
-WHERE a.id = b.gitter_id_100m;