WIP - initial involved and accident tables, load, and simple query

alembic/versions/99364b16374f_add_safety_data_tables.py

@@ -0,0 +1,78 @@
+"""Add safety-data tables
+
+Revision ID: 99364b16374f
+Revises: e962054e4422
+Create Date: 2024-11-15 11:03:26.435210
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '99364b16374f'
+down_revision = 'e962054e4422'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+sd_involved_table = "safety_data_involved"
+sd_accident_table = "safety_data_accident"
+
+
+def downgrade():
+    op.drop_table(sd_involved_table) # pylint: disable=no-member
+    op.drop_table(sd_accident_table) # pylint: disable=no-member
+
+
+def upgrade():
+    op.create_table( # pylint: disable=no-member
+        sd_involved_table,
+        sa.Column('id', sa.Integer(), autoincrement=True, nullable=True),
+        sa.Column('involve_id', sa.Integer(), primary_key=True, autoincrement=False, nullable=False),
+        sa.Column('accident_id', sa.BigInteger(), primary_key=True, autoincrement=False, nullable=False),
+        sa.Column('accident_year', sa.Integer(), primary_key=True, autoincrement=False, nullable=False),
+        sa.Column('provider_code', sa.Integer(), primary_key=True, autoincrement=False, nullable=False),
+        sa.Column('injury_severity', sa.Integer(), nullable=True),
+        sa.Column('injured_type', sa.Integer(), nullable=True),
+        sa.Column('age_group', sa.Integer(), nullable=True),
+        sa.Column('sex', sa.Integer(), nullable=True),
+        sa.Column('population_type', sa.Integer(), nullable=True),
+    )
+    op.create_index(op.f(f'ix_{sd_involved_table}_inv_acc'), sd_involved_table,
+                    ['involve_id', 'accident_id', 'accident_year', 'provider_code'], unique=True)
+    for field in ['injury_severity', 'injured_type', 'age_group', 'sex', 'population_type']:
+         # pylint: disable=no-member
+        op.create_index(op.f(f'ix_{sd_involved_table}_{field}'), sd_involved_table, [field], unique=False)
+
+    op.create_table( # pylint: disable=no-member
+        sd_accident_table,
+        sa.Column('accident_id', sa.BigInteger(), primary_key=True, autoincrement=False, nullable=False),
+        sa.Column('accident_year', sa.Integer(), primary_key=True, autoincrement=False, nullable=False),
+        sa.Column('provider_code', sa.Integer(), primary_key=True, autoincrement=False, nullable=False),
+        sa.Column('accident_month', sa.Integer()),
+        sa.Column('accident_timestamp', sa.TIMESTAMP()),
+        sa.Column('road_type', sa.Integer(), nullable=True),
+        sa.Column('road_width', sa.Integer(), nullable=True),
+        sa.Column('day_night', sa.Integer(), nullable=True),
+        sa.Column('one_lane_type', sa.Integer(), nullable=True),
+        sa.Column('multi_lane_type', sa.Integer(), nullable=True),
+        sa.Column('speed_limit_type', sa.Integer(), nullable=True),
+        sa.Column('yishuv_symbol', sa.Integer(), nullable=True),
+        sa.Column('street1', sa.Integer(), nullable=True),
+        sa.Column('street2', sa.Integer(), nullable=True),
+        sa.Column('road', sa.Integer(), nullable=True),
+        sa.Column('road_segment', sa.Integer(), nullable=True),
+        sa.Column('vehicle_types', sa.Integer(), nullable=True), # bit map
+        sa.Column('lat', sa.Float(), nullable=True),
+        sa.Column('lon', sa.Float(), nullable=True),
+    )
+    op.create_index(op.f(f'ix_{sd_accident_table}_acc_ids'), sd_accident_table,
+                    ['accident_id', 'accident_year', 'provider_code'], unique=True)
+    for field in ['accident_year', 'accident_month', 'accident_timestamp',
+                  'road_type', 'road_width', 'day_night',
+                  'one_lane_type', 'multi_lane_type', 'speed_limit_type',
+                  'yishuv_symbol', 'street1', 'street2', 'road', 'road_segment', 'vehicle_types',
+                  'lat', 'lon',
+                  ]:
+         # pylint: disable=no-member
+        op.create_index(op.f(f'ix_{sd_accident_table}_{field}'), sd_accident_table, [field], unique=False)

anyway/flask_app.py

@@ -1551,3 +1551,18 @@ def test_roles_func():
 
 
 app.add_url_rule("/api/test_roles", endpoint=None, view_func=test_roles, methods=["GET"])
+
+
+@app.route("/safety-data-test", methods=["GET"])
+def safety_data_test():
+    from anyway.views.safety_data import sd_utils as sdu
+    from anyway.views.safety_data import involved_query as ac
+
+    load_data = request.values.get("load-data")
+    if load_data:
+        return sdu.load_data()
+
+    iq = ac.InvolvedQuery()
+    res = iq.get_data()
+    return Response(json.dumps(res, default=str), mimetype="application/json")
+

anyway/models.py

@@ -3081,3 +3081,40 @@ class TelegramForwardedMessages(Base):
     message_id = Column(String(), primary_key=True)
     newsflash_id = Column(BigInteger(), nullable=False)
     group_sent = Column(String(), nullable=False)
+
+
+class SDInvolved(Base):
+    __tablename__ = "safety_data_involved"
+    id = Column(Integer(), autoincrement=True)
+    involve_id = Column(Integer(), primary_key=True)
+    accident_id = Column(BigInteger())
+    accident_year = Column(Integer(), primary_key=True)
+    provider_code = Column(Integer(), primary_key=True)
+    injury_severity = Column(Integer(), nullable=True)
+    injured_type = Column(Integer(), nullable=True)
+    age_group = Column(Integer(), nullable=True)
+    sex = Column(Integer(), nullable=True)
+    population_type = Column(Integer(), nullable=True)
+
+
+class SDAccident(Base):
+    __tablename__ = "safety_data_accident"
+    accident_id = Column(Integer(), primary_key=True, autoincrement=False, nullable=False)
+    accident_year = Column(Integer(), primary_key=True, autoincrement=False, nullable=False)
+    provider_code = Column(Integer(), primary_key=True)
+    accident_month = Column(Integer())
+    accident_timestamp = Column(DateTime, default=None, index=True)
+    road_type = Column(Integer(), nullable=True)
+    road_width = Column(Integer(), nullable=True)
+    day_night = Column(Integer(), nullable=True)
+    one_lane_type = Column(Integer(), nullable=True)
+    multi_lane_type = Column(Integer(), nullable=True)
+    speed_limit_type = Column(Integer(), nullable=True)
+    yishuv_symbol = Column(Integer(), nullable=True)
+    street1 = Column(Integer(), nullable=True)
+    street2 = Column(Integer(), nullable=True)
+    road = Column(Integer(), nullable=True)
+    road_segment = Column(Integer(), nullable=True)
+    vehicle_types = Column(Integer(), nullable=True) # bit map
+    lat = Column(Float(), nullable=True)
+    lon = Column(Float(), nullable=True)

anyway/views/safety_data/involved_query.py

@@ -0,0 +1,72 @@
+import json
+from typing import List
+import pandas as pd
+from sqlalchemy.orm import aliased
+from sqlalchemy import and_
+from flask import request, Response
+from anyway.models import (
+    SDAccident, SDInvolved,
+    City, InjurySeverity, Streets, Sex, RoadLight, RoadType, AgeGroup,
+)
+from anyway.app_and_db import app, db
+
+class InvolvedQuery:
+    def __init__(self):
+        self.S1: Streets = aliased(Streets)
+        self.S2: Streets = aliased(Streets)
+        self.fill_text_tables()
+
+
+    def get_data(self):
+        data = self.get_data_from_db()
+        [self.add_text(d) for d in data]
+        return data
+
+    def get_data_from_db(self):
+        query = (
+            db.session.query(SDInvolved, SDAccident, )
+            .join(SDAccident,
+                and_(SDInvolved.provider_code == SDAccident.provider_code,
+                    SDInvolved.accident_id == SDAccident.accident_id,
+                    SDInvolved.accident_year == SDAccident.accident_year))
+            # .filter(SDAccident.yishuv_symbol == 5000)
+            # .filter(SDInvolved.id == 26833652)
+            .with_entities(
+                SDInvolved.involve_id,
+                SDInvolved.injury_severity,
+                SDInvolved.injured_type,
+                SDInvolved.age_group,
+                SDInvolved.sex,
+                SDInvolved.population_type,
+                SDAccident.accident_year,
+                SDAccident.accident_timestamp,
+                SDAccident.lat,
+                SDAccident.lon,
+            )
+            .limit(50)
+        )
+        # pylint: disable=no-member
+        results = pd.read_sql_query(query.statement, query.session.bind).to_dict(
+            orient="records"
+        )  # pylint: disable=no-member
+        return results
+
+    def add_text(self, d: dict) -> None:
+        n = d["injury_severity"]
+        d["injury_severity"] = self.injury_severity[n] if n else None
+        n = d["sex"]
+        d["sex"] = self.sex[n] if n else None
+        n = d["age_group"]
+        d["age_group"] = ("85+" if n == 99 else self.age_group[n]) if n else None
+        n = d["injured_type"]
+        d["injured_type"] = self.injured_type[n] if n else None
+        n = d["population_type"]
+        d["population_type"] = self.population_type[n] if n else None
+
+    def fill_text_tables(self):
+        self.injury_severity = ["0", "הרוג", "פצוע קשה", "פצוע בינוני"]
+        self.sex = ["0", "זכר", "נקבה"]
+        self.age_group = ["0", "0-4", "5-9", "10-14", "15-19", "20-24", "25-29", "30-34", "35-39", "40-44", "45-49", "50-54", "55-59", "60-64", "65-69", "70-74", "75-79", "80-84", "85+"]
+        self.injured_type = ["0", "הולך רגל", "נהג - רכב בעל 4 גלגלים ויותר", "נוסע - רכב בעל 4 גלגלים ויותר", "נהג - אופנוע", "נוסע - אופנוע (לא נהג)", "נהג - אופניים", "נוסע - אופניים (לא נהג)", "נהג - רכב לא ידוע", "נוסע - רכב לא ידוע"]
+        self.population_type = ["0", "יהודים", "ערבים", "אחרים", "זרים"]
+

anyway/views/safety_data/sd_utils.py

@@ -0,0 +1,174 @@
+import json
+from sqlalchemy.orm import aliased
+from sqlalchemy import and_
+from flask import request, Response
+from anyway.models import (
+    InvolvedMarkerView, SDAccident, SDInvolved, AccidentMarkerView,
+    City, InjurySeverity, Streets, Sex, RoadLight, RoadType, AgeGroup,
+    Involved,
+)
+from anyway.app_and_db import app, db
+
+
+def load_data():
+    load_data = request.values.get("load-data")
+    if load_data:
+        return sd_load_data()
+
+
+def get_data():
+    S1: Streets = aliased(Streets)
+    S2: Streets = aliased(Streets)
+    res = (
+        db.session.query(Involved,
+                         AccidentMarkerView,
+                         City,
+                         InjurySeverity,
+                         Streets,
+                         Sex,
+                         RoadLight,
+                         RoadType,
+                         AgeGroup,
+                         )
+        .join(AccidentMarkerView,
+              and_(Involved.provider_code == AccidentMarkerView.provider_code,
+                   Involved.accident_id == AccidentMarkerView.id,
+                   Involved.accident_year == AccidentMarkerView.accident_year))
+        .join(City, AccidentMarkerView.yishuv_symbol == City.yishuv_symbol)
+        .join(InjurySeverity, and_(Involved.injury_severity == InjurySeverity.id,
+                                   Involved.accident_year == InjurySeverity.year,
+                                   Involved.provider_code == InjurySeverity.provider_code))
+        .join(S1, and_(AccidentMarkerView.street1 == S1.street,
+                       AccidentMarkerView.yishuv_symbol == S1.yishuv_symbol))
+        .join(S2, and_(AccidentMarkerView.street2 == S2.street,
+                       AccidentMarkerView.yishuv_symbol == S2.yishuv_symbol))
+        .join(Sex, and_(Involved.sex == Sex.id,
+                        Involved.accident_year == Sex.year,
+                        Involved.provider_code == Sex.provider_code))
+        .join(RoadLight, and_(AccidentMarkerView.road_light == RoadLight.id,
+                        AccidentMarkerView.accident_year == RoadLight.year,
+                        AccidentMarkerView.provider_code == RoadLight.provider_code))
+        .join(RoadType, and_(AccidentMarkerView.road_type == RoadType.id,
+                        AccidentMarkerView.accident_year == RoadType.year,
+                        AccidentMarkerView.provider_code == RoadType.provider_code))
+        .join(AgeGroup, and_(Involved.age_group == AgeGroup.id,
+                        Involved.accident_year == AgeGroup.year,
+                        Involved.provider_code == AgeGroup.provider_code))
+        # .filter(AccidentMarkerView.yishuv_symbol == 5000)
+        # .filter(Involved.id == 26833652)
+        .with_entities(
+            Involved.id,
+            AccidentMarkerView.accident_year,
+            AccidentMarkerView.accident_timestamp,
+            City.heb_name,
+            S1.street_hebrew,
+            S2.street_hebrew,
+            RoadLight.road_light_hebrew,
+            RoadType.road_type_hebrew,
+            AccidentMarkerView.latitude,
+            AccidentMarkerView.longitude,
+            InjurySeverity.injury_severity_hebrew,
+            Sex.sex_hebrew,
+            AgeGroup.age_group_hebrew,
+        )
+        .limit(50).all()
+    )
+    return Response(json.dumps(res, default=str), mimetype="application/json")
+
+def sd_load_data():
+    sd_load_accident()
+    sd_load_involved()
+    return Response(json.dumps("Tables loaded", default=str), mimetype="application/json")
+
+def sd_load_involved():
+    db.session.query(SDInvolved).delete()
+    db.session.commit()
+    db.get_engine().execute(
+        SDInvolved.__table__.insert(),
+          [d for d in get_involved_data()]
+          )
+    db.session.commit()
+
+def get_involved_data():
+    for d in db.session.query(InvolvedMarkerView, SDAccident)\
+        .join(SDAccident,
+            and_(SDAccident.provider_code == InvolvedMarkerView.provider_code,
+                 SDAccident.accident_id == InvolvedMarkerView.accident_id,
+                 SDAccident.accident_year == InvolvedMarkerView.accident_year))\
+        .with_entities(InvolvedMarkerView.involve_id,
+                        InvolvedMarkerView.accident_id,
+                        InvolvedMarkerView.accident_year,
+                        InvolvedMarkerView.provider_code,
+                        InvolvedMarkerView.injury_severity,
+                        InvolvedMarkerView.injured_type,
+                        InvolvedMarkerView.age_group,
+                        InvolvedMarkerView.sex,
+                        InvolvedMarkerView.population_type,
+                        )\
+        .limit(1000):
+        yield{
+                "involve_id": d.involve_id,
+                "accident_id": d.accident_id,
+                "accident_year": d.accident_year,
+                "provider_code": d.provider_code,
+                "injury_severity": d.injury_severity,
+                "injured_type": d.injured_type,
+                "age_group": d.age_group,
+                "sex": d.sex,
+                "population_type": d.population_type,
+            }
+
+def sd_load_accident():
+    from anyway.models import SDAccident, AccidentMarkerView, SDInvolved
+    db.session.query(SDAccident).delete()
+    db.session.commit()
+    db.get_engine().execute(
+        SDAccident.__table__.insert(),
+        [
+            {
+                "accident_id": d.id,
+                "accident_year": d.accident_year,
+                "provider_code": d.provider_code,
+                "accident_month": d.accident_month,
+                "accident_timestamp": d.accident_timestamp,
+                "road_type": d.road_type,
+                "road_width": d.road_width,
+                "day_night": d.day_night,
+                "one_lane_type": d.one_lane,
+                "multi_lane_type": d.multi_lane,
+                "speed_limit_type": d.speed_limit,
+                "yishuv_symbol": d.yishuv_symbol,
+                "street1": d.street1,
+                "street2": d.street2,
+                "road": d.road1,
+                "road_segment": d.road_segment_number,
+                "lat": d.latitude,
+                "lon": d.longitude,
+            }
+            for d in db.session.query(AccidentMarkerView)
+            .with_entities(AccidentMarkerView.id,
+                           AccidentMarkerView.accident_year,
+                           AccidentMarkerView.provider_code,
+                           AccidentMarkerView.accident_year,
+                           AccidentMarkerView.accident_month,
+                           AccidentMarkerView.accident_timestamp,
+                           AccidentMarkerView.road_type,
+                           AccidentMarkerView.road_width,
+                           AccidentMarkerView.day_night,
+                           AccidentMarkerView.one_lane,
+                           AccidentMarkerView.multi_lane,
+                           AccidentMarkerView.speed_limit,
+                           AccidentMarkerView.yishuv_symbol,
+                           AccidentMarkerView.street1,
+                           AccidentMarkerView.street2,
+                           AccidentMarkerView.road1,
+                           AccidentMarkerView.road_segment_number,
+                           AccidentMarkerView.latitude,
+                           AccidentMarkerView.longitude,
+                           )
+                           .limit(1000)
+        ]
+    )
+    db.session.commit()
+
+