diff --git a/CHANGELOG.md b/CHANGELOG.md index de83826eb..c26527e4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 6.19.3 - Jul 8, 2024 + +* Add instructions on DB migrations to docs +* Change schema to fix two DB issues: + * Event Document `note` changed to Text type to avoid character limit errors + * Vote Event `dedupe_key` index added to improve performance on Vote Event lookups during import + ## 6.19.2 - Jun 7, 2024 * Turn off archiving processed realtime bills by default. @@ -53,11 +60,11 @@ ## 6.17.3 - August 1, 2023 * update pyyaml dependency - + ## 6.17.2 - August 1, 2023 * Support additional territories in db init - + ## 6.17.0 - July 14, 2023 * Add additional US territory support#113 @@ -156,7 +163,6 @@ * add SQS for batching - ## 6.13.8 - January 9, 2023 * update us legacy districts @@ -349,7 +355,7 @@ * fix os-us-to-yaml phone numbers * added latest_bill_update and latest_people_update to Jurisdiction table (these fields are now automatically updated when os-update or os-people - update the relevant tables) + update the relevant tables) * moved openstates.reports into openstates.data ## 6.3.4 - August 2 2021 @@ -460,8 +466,8 @@ ## 5.8.0 - April 5 2021 * remaining implementation of OSEP #5 - * remove pupa_id backwards compatibility - * migration: drop reports.Identifier + * remove pupa_id backwards compatibility + * migration: drop reports.Identifier * Add veto and veto-override vote classifications ## 5.7.1 - April 5 2021 @@ -471,9 +477,9 @@ ## 5.7.0 - April 2 2021 * Implementation of OSEP #5 - * migration: added dedupe_key to replace pupa_id - * added dedupe_key to replace pupa_id in scrape as well - * alter import logic to stop using reports.Identifier + * migration: added dedupe_key to replace pupa_id + * added dedupe_key to replace pupa_id in scrape as well + * alter import logic to stop using reports.Identifier ## 5.6.0 - March 23 2021 @@ -542,26 +548,25 @@ * merge openstates_metadata into openstates.metadata * large database cleanup: - * remove unused fields: - * BillAbstract.date - * BillAction.extras - * BillIdentifier.scheme & note - * Jurisdiction.feature_flags - * Membership.label - * Organization.image, founding_date, dissolution_date - * Person.summary - * PersonContactDetail.label - * Post.start_date & end_date - * VoteEvent.end_date - * BillDocumentLink & BillVersionLink.text - * *.locked_fields - * add VoteEvent.order + * remove unused fields: + * BillAbstract.date + * BillAction.extras + * BillIdentifier.scheme & note + * Jurisdiction.feature_flags + * Membership.label + * Organization.image, founding_date, dissolution_date + * Person.summary + * PersonContactDetail.label + * Post.start_date & end_date + * VoteEvent.end_date + * BillDocumentLink & BillVersionLink.text + * *.locked_fields + * add VoteEvent.order ## 4.8.0 - August 21 2020 * remove GIS dependency, alter (unused) EventLocation model accordingly - ## 4.7.1 - August 5 2020 * remove person_role_division_id diff --git a/README.md b/README.md index 60c69e6d2..5dc8f3b1a 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,30 @@ This repository contains the Open States data model and scraper backend. See [RELASE.md](./RELEASE.md) +## Database Migrations + +openstates-core data models may occasionally change, requiring changes to the Open States database in production. Here +are the steps to create and execute a new migration: + +* Follow the steps in [Running a Local Database doc](https://docs.openstates.org/contributing/local-database/) to get + a copy of the database running locally with the prior schema. It is important to develop and test your migration + locally before executing it on the production DB. +* In most cases, you will auto-generate a migration by first modifying the model file. Make a change to one or more of + the files in `openstates/data/models`. +* Test the migration locally: + * Identify the DB connection URL that is accurate to your local database. In most cases, it should + be: `postgis://openstates:openstates@localhost:5405/openstatesorg` + * In the repo's root folder, run the `os-dbmakemigrations` command to generate a migration file based on your + changes: `DATABASE_URL=postgis://openstates:openstates@localhost:5405/openstatesorg poetry run os-dbmakemigrations` + * Look at the generated migration file in `openstates/data/migrations` and ensure that it looks correct. + * Execute the migration by running the `os-initdb` + command: `DATABASE_URL=postgis://openstates:openstates@localhost:5405/openstatesorg poetry run os-initdb` +* Once the migration is verified by local testing, you can execute it against the production DB + * Identify the DB connection URL that is accurate for the PROD database. This typically should use the same Postgres + user that owns the tables you want to change. Contact an admin if you need help. + * Run the `os-initdb` command to + migrate: `DATABASE_URL=postgis://USERNAME_HERE:PASSWORD_HERE@PROD_DB_HOSTNAME_HERE:5432/openstatesorg poetry run os-initdb` + ## Debugging openstates-core code ### Commands that do not integrate with openstates-scrapers diff --git a/openstates/cli/makemigrations.py b/openstates/cli/makemigrations.py new file mode 100644 index 000000000..70e291e4d --- /dev/null +++ b/openstates/cli/makemigrations.py @@ -0,0 +1,8 @@ +# type: ignore +from ..utils.django import init_django +from django.core import management + + +def main() -> None: + init_django() + management.call_command("makemigrations") diff --git a/openstates/data/migrations/0045_auto_20240705_1812.py b/openstates/data/migrations/0045_auto_20240705_1812.py new file mode 100644 index 000000000..4aeff27aa --- /dev/null +++ b/openstates/data/migrations/0045_auto_20240705_1812.py @@ -0,0 +1,22 @@ +# Generated by Django 3.2.14 on 2024-07-05 18:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('data', '0044_bill_citations'), + ] + + operations = [ + migrations.AlterField( + model_name='eventdocument', + name='note', + field=models.TextField(), + ), + migrations.AddIndex( + model_name='voteevent', + index=models.Index(fields=['dedupe_key'], name='opencivicdata_voteevent_dedupe__75a90b_idx'), + ), + ] diff --git a/openstates/data/models/event.py b/openstates/data/models/event.py index 88b7f3436..ca9f56639 100644 --- a/openstates/data/models/event.py +++ b/openstates/data/models/event.py @@ -95,7 +95,7 @@ class Meta: class EventDocument(RelatedBase): event = models.ForeignKey(Event, related_name="documents", on_delete=models.CASCADE) - note = models.CharField(max_length=300) + note = models.TextField() date = models.CharField(max_length=25, blank=True) # YYYY-MM-DD HH:MM:SS+HH:MM classification = models.CharField( max_length=50, choices=EVENT_DOCUMENT_CLASSIFICATION_CHOICES, blank=True diff --git a/openstates/data/models/vote.py b/openstates/data/models/vote.py index fb977e15b..0ee436e93 100644 --- a/openstates/data/models/vote.py +++ b/openstates/data/models/vote.py @@ -63,6 +63,9 @@ class Meta: ["legislative_session", "identifier", "bill"], ["legislative_session", "bill"], ] + indexes = [ + models.Index(fields=["dedupe_key"]) + ] ordering = ["start_date", "order"] diff --git a/poetry.lock b/poetry.lock index 9f96ab463..fc3fb6944 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1548,6 +1548,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, diff --git a/pyproject.toml b/pyproject.toml index f8c2f5fe6..db614b140 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "openstates" -version = "6.19.2" +version = "6.19.3" description = "core infrastructure for the openstates project" authors = ["James Turk "] license = "MIT" @@ -8,6 +8,7 @@ license = "MIT" [tool.poetry.scripts] os-update = 'openstates.cli.update:main' os-initdb = 'openstates.cli.initdb:main' +os-dbmakemigrations = 'openstates.cli.makemigrations:main' os-update-computed = 'openstates.cli.update_computed:main' os-text-extract = 'openstates.cli.text_extract:main' os-people = 'openstates.cli.people:main'