diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e71e5972b..54d0422d0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,4 +11,4 @@ jobs: - name: Copy To Branches Action uses: planetoftheweb/copy-to-branches@v1.2 env: - key: main + key: main \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 236981053..c200c1b5c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,7 +3,7 @@ "editor.cursorBlinking": "solid", "editor.fontFamily": "ui-monospace, Menlo, Monaco, 'Cascadia Mono', 'Segoe UI Mono', 'Roboto Mono', 'Oxygen Mono', 'Ubuntu Monospace', 'Source Code Pro', 'Fira Mono', 'Droid Sans Mono', 'Courier New', monospace", "editor.fontLigatures": false, - "editor.fontSize": 22, + "editor.fontSize": 14, "editor.formatOnPaste": true, "editor.formatOnSave": true, "editor.lineNumbers": "on", @@ -16,9 +16,8 @@ "explorer.openEditors.visible": 0, "files.autoSave": "afterDelay", "screencastMode.onlyKeyboardShortcuts": true, - "terminal.integrated.fontSize": 18, - "workbench.activityBar.visible": true, + "terminal.integrated.fontSize": 14, "workbench.colorTheme": "Visual Studio Dark", "workbench.fontAliasing": "antialiased", "workbench.statusBar.visible": true -} +} \ No newline at end of file diff --git a/data/nyc_parking_violations.db b/data/nyc_parking_violations.db index 19e587528..72b099b19 100644 Binary files a/data/nyc_parking_violations.db and b/data/nyc_parking_violations.db differ diff --git a/data/prod_nyc_parking_violations.db b/data/prod_nyc_parking_violations.db index 9a2bd4ad7..e6e19c690 100644 Binary files a/data/prod_nyc_parking_violations.db and b/data/prod_nyc_parking_violations.db differ diff --git a/nyc_parking_violations/.gitignore b/nyc_parking_violations/.gitignore index 9a7943d2d..49f147cb9 100644 --- a/nyc_parking_violations/.gitignore +++ b/nyc_parking_violations/.gitignore @@ -1,4 +1,4 @@ -.user.yml + target/ dbt_packages/ logs/ diff --git a/nyc_parking_violations/.user.yml b/nyc_parking_violations/.user.yml new file mode 100644 index 000000000..54b06290f --- /dev/null +++ b/nyc_parking_violations/.user.yml @@ -0,0 +1 @@ +id: 334930ae-2627-4a6c-af90-c9826c0f4e5d diff --git a/nyc_parking_violations/dbt_project.yml b/nyc_parking_violations/dbt_project.yml index ec48605d3..1118ac748 100644 --- a/nyc_parking_violations/dbt_project.yml +++ b/nyc_parking_violations/dbt_project.yml @@ -49,4 +49,4 @@ models: gold: +materialized: table tests: - +store_failures: true + +store_failures: true \ No newline at end of file diff --git a/nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql b/nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql index 217a5f263..40fad7ffc 100644 --- a/nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql +++ b/nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql @@ -1,7 +1,3 @@ -SELECT - code AS violation_code, - definition, - manhattan_96th_st_below, - all_other_areas -FROM - parking_violation_codes +SELECT code AS violation_code, + definition, manhattan_96th_st_below, all_other_areas +FROM parking_violation_codes \ No newline at end of file diff --git a/nyc_parking_violations/models/bronze/bronze_parking_violations.sql b/nyc_parking_violations/models/bronze/bronze_parking_violations.sql index 46525da2a..63106dfc2 100644 --- a/nyc_parking_violations/models/bronze/bronze_parking_violations.sql +++ b/nyc_parking_violations/models/bronze/bronze_parking_violations.sql @@ -1,23 +1,8 @@ -SELECT - summons_number, - registration_state, - plate_type, - issue_date, - violation_code, - vehicle_body_type, - vehicle_make, - issuing_agency, - vehicle_expiration_date, - violation_location, - violation_precinct, - issuer_precinct, - issuer_code, - issuer_command, - issuer_squad, - violation_time, - violation_county, - violation_legal_code, - vehicle_color, - vehicle_year, -FROM - parking_violations_2023 \ No newline at end of file +SELECT summons_number, registration_state, plate_type, issue_date, violation_code, vehicle_body_type, + vehicle_make, issuing_agency, vehicle_expiration_date, violation_location, violation_precinct, + issuer_precinct, issuer_code, issuer_command, issuer_squad, violation_time, time_first_observed, + violation_county, violation_in_front_of_or_opposite, date_first_observed, law_section, sub_division, + violation_legal_code, days_parking_in_effect, from_hours_in_effect, to_hours_in_effect, vehicle_color, + unregistered_vehicle, vehicle_year, meter_number, feet_from_curb, no_standing_or_stopping_violation, + hydrant_violation, double_parking_violation +FROM parking_violations_2023 \ No newline at end of file diff --git a/nyc_parking_violations/models/example/first_model.sql b/nyc_parking_violations/models/example/first_model.sql index 942e472df..2370069ae 100644 --- a/nyc_parking_violations/models/example/first_model.sql +++ b/nyc_parking_violations/models/example/first_model.sql @@ -1 +1,2 @@ -SELECT * FROM parking_violation_codes \ No newline at end of file +SELECT * +FROM parking_violation_codes \ No newline at end of file diff --git a/nyc_parking_violations/models/example/ref_model.sql b/nyc_parking_violations/models/example/ref_model.sql index 9bb4f51c6..64b53a444 100644 --- a/nyc_parking_violations/models/example/ref_model.sql +++ b/nyc_parking_violations/models/example/ref_model.sql @@ -1,4 +1,2 @@ -SELECT - COUNT(*) -FROM - {{ref('first_model')}} +SELECT COUNT(*) +FROM {{ref('first_model')}} \ No newline at end of file diff --git a/nyc_parking_violations/models/gold/gold_ticket_metrics.sql b/nyc_parking_violations/models/gold/gold_ticket_metrics.sql index 861de4d61..f08e76396 100644 --- a/nyc_parking_violations/models/gold/gold_ticket_metrics.sql +++ b/nyc_parking_violations/models/gold/gold_ticket_metrics.sql @@ -1,10 +1,6 @@ -SELECT - violation_code, +SELECT violation_code, COUNT(summons_number) AS ticket_count, SUM(fee_usd) AS total_revenue_usd -FROM - {{ref('silver_violation_tickets')}} -GROUP BY - violation_code -ORDER BY - total_revenue_usd DESC \ No newline at end of file +FROM {{ref('silver_violation_tickets')}} +GROUP BY violation_code +ORDER BY total_revenue_usd DESC \ No newline at end of file diff --git a/nyc_parking_violations/models/gold/gold_vehicles_metrics.sql b/nyc_parking_violations/models/gold/gold_vehicles_metrics.sql index af796bfa9..17debeb87 100644 --- a/nyc_parking_violations/models/gold/gold_vehicles_metrics.sql +++ b/nyc_parking_violations/models/gold/gold_vehicles_metrics.sql @@ -1,11 +1,6 @@ -SELECT - registration_state, - COUNT(summons_number) AS ticket_count, -FROM - {{ref('silver_violation_vehicles')}} -WHERE - registration_state != 'NY' -GROUP BY - registration_state -ORDER BY - ticket_count DESC \ No newline at end of file +SELECT registration_state, + COUNT(summons_number) AS ticket_count +FROM {{ref('silver_violation_vehicles')}} +WHERE registration_state != 'NY' +GROUP BY registration_state +ORDER BY ticket_count DESC \ No newline at end of file diff --git a/nyc_parking_violations/models/silver/silver_parking_violation_codes.sql b/nyc_parking_violations/models/silver/silver_parking_violation_codes.sql index 44144d9dd..6fb8882f0 100644 --- a/nyc_parking_violations/models/silver/silver_parking_violation_codes.sql +++ b/nyc_parking_violations/models/silver/silver_parking_violation_codes.sql @@ -1,24 +1,20 @@ -WITH manhattan_violation_codes AS ( - SELECT - violation_code, - definition, +WITH manhattan_violation_codes +AS ( + SELECT violation_code, definition, TRUE AS is_manhattan_96th_st_below, - manhattan_96th_st_below AS fee_usd, - FROM - {{ref('bronze_parking_violation_codes')}} + manhattan_96th_st_below AS fee_usd + FROM {{ref('bronze_parking_violation_codes')}} ), - -all_other_violation_codes AS ( - SELECT - violation_code, - definition, +all_other_violation_codes +AS ( + SELECT violation_code, definition, FALSE AS is_manhattan_96th_st_below, - all_other_areas AS fee_usd, - FROM - {{ref('bronze_parking_violation_codes')}} + all_other_areas AS fee_usd + FROM {{ref('bronze_parking_violation_codes')}} ) - -SELECT * FROM manhattan_violation_codes +SELECT * +FROM manhattan_violation_codes UNION ALL -SELECT * FROM all_other_violation_codes -ORDER BY violation_code ASC \ No newline at end of file +SELECT * +FROM all_other_violation_codes +ORDER BY violation_code \ No newline at end of file diff --git a/nyc_parking_violations/models/silver/silver_parking_violations.sql b/nyc_parking_violations/models/silver/silver_parking_violations.sql index 3a6e112cb..663732590 100644 --- a/nyc_parking_violations/models/silver/silver_parking_violations.sql +++ b/nyc_parking_violations/models/silver/silver_parking_violations.sql @@ -1,28 +1,9 @@ -SELECT - summons_number, - registration_state, - plate_type, - issue_date, - violation_code, - vehicle_body_type, - vehicle_make, - issuing_agency, - vehicle_expiration_date, - violation_location, - violation_precinct, - issuer_precinct, - issuer_code, - issuer_command, - issuer_squad, - violation_time, - violation_county, - violation_legal_code, - vehicle_color, - vehicle_year, - CASE WHEN - violation_county == 'MN' - THEN TRUE +SELECT summons_number, registration_state, plate_type, issue_date, violation_code, vehicle_body_type, + vehicle_make, issuing_agency, vehicle_expiration_date, violation_location, violation_precinct, + issuer_precinct, issuer_code, issuer_command, issuer_squad, violation_time, violation_county, + violation_legal_code, vehicle_color, vehicle_year, + CASE + WHEN violation_county == 'MN' THEN TRUE ELSE FALSE - END AS is_manhattan_96th_st_below -FROM - {{ref('bronze_parking_violations')}} \ No newline at end of file + END AS is_manhattan_96th_st_below +FROM {{ref('bronze_parking_violations')}} \ No newline at end of file diff --git a/nyc_parking_violations/models/silver/silver_violation_tickets.sql b/nyc_parking_violations/models/silver/silver_violation_tickets.sql index 66b5ce0d4..22ff573bc 100644 --- a/nyc_parking_violations/models/silver/silver_violation_tickets.sql +++ b/nyc_parking_violations/models/silver/silver_violation_tickets.sql @@ -1,22 +1,9 @@ -SELECT - violations.summons_number, - violations.issue_date, - violations.violation_code, - violations.is_manhattan_96th_st_below, - violations.issuing_agency, - violations.violation_location, - violations.violation_precinct, - violations.issuer_precinct, - violations.issuer_code, - violations.issuer_command, - violations.issuer_squad, - violations.violation_time, - violations.violation_county, - violations.violation_legal_code, - codes.fee_usd -FROM - {{ref('silver_parking_violations')}} AS violations -LEFT JOIN - {{ref('silver_parking_violation_codes')}} AS codes ON - violations.violation_code = codes.violation_code AND - violations.is_manhattan_96th_st_below = codes.is_manhattan_96th_st_below \ No newline at end of file +SELECT violations.summons_number, violations.issue_date, violations.violation_code, + violations.is_manhattan_96th_st_below, violations.issuing_agency, violations.violation_location, + violations.violation_precinct, violations.issuer_precinct, violations.issuer_code, + violations.issuer_command, violations.issuer_squad, violations.violation_time, + violations.violation_county, violations.violation_legal_code, codes.fee_usd +FROM {{ref('silver_parking_violations')}} AS violations + LEFT JOIN {{ref('silver_parking_violation_codes')}} AS codes ON + codes.violation_code = violations.violation_code + AND codes.is_manhattan_96th_st_below = violations.is_manhattan_96th_st_below \ No newline at end of file diff --git a/nyc_parking_violations/models/silver/silver_violation_vehicles.sql b/nyc_parking_violations/models/silver/silver_violation_vehicles.sql index e2b8537a2..0918c62a8 100644 --- a/nyc_parking_violations/models/silver/silver_violation_vehicles.sql +++ b/nyc_parking_violations/models/silver/silver_violation_vehicles.sql @@ -1,11 +1,3 @@ -SELECT - summons_number, - registration_state, - plate_type, - vehicle_body_type, - vehicle_make, - vehicle_expiration_date, - vehicle_color, - vehicle_year -FROM - {{ref('silver_parking_violations')}} \ No newline at end of file +SELECT summons_number, registration_state, plate_type, vehicle_body_type, + vehicle_make, vehicle_expiration_date, vehicle_color, vehicle_year +FROM {{ref('silver_parking_violations')}} \ No newline at end of file diff --git a/nyc_parking_violations/profiles.yml b/nyc_parking_violations/profiles.yml index cabe18c54..8fe04e9a1 100644 --- a/nyc_parking_violations/profiles.yml +++ b/nyc_parking_violations/profiles.yml @@ -1,12 +1,12 @@ nyc_parking_violations: outputs: - dev: - type: duckdb - path: '../data/nyc_parking_violations.db' - prod: - type: duckdb - # note that path is slightly different as GitHub actions - # start in the root directory and not in the - # nyc_parking_violations directory - path: './data/prod_nyc_parking_violations.db' + dev: + type: duckdb + path: '../data/nyc_parking_violations.db' + prod: + type: duckdb + # note that path is slightly different as GitHub actions + # start in the root directory and not in the + # nyc_parking_violations directory + path: '../data/prod_nyc_parking_violations.db' target: dev \ No newline at end of file diff --git a/nyc_parking_violations/tests/generic/generic_not_null.sql b/nyc_parking_violations/tests/generic/generic_not_null.sql index e49c2b087..186ce09c9 100644 --- a/nyc_parking_violations/tests/generic/generic_not_null.sql +++ b/nyc_parking_violations/tests/generic/generic_not_null.sql @@ -1,8 +1,6 @@ -- source: https://docs.getdbt.com/guides/best-practices/writing-custom-generic-tests#generic-tests-with-default-config-values {% test generic_not_null(model, column_name) %} - - select * - from {{ model }} - where {{ column_name }} is null - + SELECT * + FROM {{model}} + WHERE {{column_name}} IS NULL {% endtest %} \ No newline at end of file diff --git a/nyc_parking_violations/tests/violation_codes_revenue.sql b/nyc_parking_violations/tests/violation_codes_revenue.sql index 091ddfdd4..916729c73 100644 --- a/nyc_parking_violations/tests/violation_codes_revenue.sql +++ b/nyc_parking_violations/tests/violation_codes_revenue.sql @@ -1,11 +1,6 @@ -{{ config(severity = 'warn') }} - -SELECT - violation_code, +{{config(severity = 'warn')}} +SELECT violation_code, SUM(fee_usd) AS total_revenue_usd -FROM - {{ref('silver_parking_violation_codes')}} -GROUP BY - violation_code -HAVING - NOT(total_revenue_usd >= 1) +FROM {{ref('silver_parking_violation_codes')}} +GROUP BY violation_code +HAVING NOT (total_revenue_usd >= 1) \ No newline at end of file diff --git a/run_sql_queries_here.ipynb b/run_sql_queries_here.ipynb index f7c54e6c9..3787552ca 100644 --- a/run_sql_queries_here.ipynb +++ b/run_sql_queries_here.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -12,17 +12,128 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
name
0bronze_parking_violation_codes
1bronze_parking_violations
2gold_ticket_metrics
3gold_vehicles_metrics
4parking_violation_code
5parking_violation_codes
6parking_violations_2023
7silver_violation_tickets
8silver_violation_vehicles
\n", + "
" + ], + "text/plain": [ + " name\n", + "0 bronze_parking_violation_codes\n", + "1 bronze_parking_violations\n", + "2 gold_ticket_metrics\n", + "3 gold_vehicles_metrics\n", + "4 parking_violation_code\n", + "5 parking_violation_codes\n", + "6 parking_violations_2023\n", + "7 silver_violation_tickets\n", + "8 silver_violation_vehicles" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "sql_query = '''\n", "show tables\n", "'''\n", "\n", - "with duckdb.connect('data/nyc_parking_violations.db') as con:\n", + "with duckdb.connect('data/prod_nyc_parking_violations.db') as con:\n", " display(con.sql(sql_query).df())" ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "sql_query_import_1 = '''\n", + "CREATE OR REPLACE TABLE parking_violation_code AS\n", + "SELECT *\n", + "FROM read_csv_auto(\n", + " 'data/dof_parking_violation_codes.csv',\n", + " normalize_names=True\n", + ")\n", + "'''\n", + "\n", + "sql_query_import_2 = '''\n", + "CREATE OR REPLACE TABLE parking_violations_2023 AS\n", + "SELECT *\n", + "FROM read_csv_auto(\n", + " 'data/parking_violations_issued_fiscal_year_2023_sample.csv',\n", + " normalize_names=True\n", + ")\n", + "'''\n", + "\n", + "with duckdb.connect('data/prod_nyc_parking_violations.db') as con:\n", + " con.sql(sql_query_import_1)\n", + " con.sql(sql_query_import_2)" + ] } ], "metadata": {