fix(import): correct count displayed juste before import data to dest…

…ination (#3336)

* fix(import): correct count displayed juste before import data to destination
* change name of occhab geonature template
* add message when no entity data ready to import
* add test backend for preview valid data in occhab

backend/geonature/core/imports/checks/sql/extra.py

@@ -640,7 +640,7 @@ def check_entity_data_consistency(imprt, entity, fields, grouping_field):
         select(hashedRows.c.grouping_col.label("grouping_col"))
         .group_by(hashedRows.c.grouping_col)
         .having(func.count(func.distinct(hashedRows.c.hashed)) > 1)
-    )
+    ).cte()
 
     # note: rows are unidentified (None) instead of being marked as invalid (False) in order to avoid running checks
     report_erroneous_rows(

backend/geonature/core/imports/routes/imports.py

@@ -476,7 +476,7 @@ def preview_valid_data(scope, imprt):
 
     # Retrieve data for each entity from entries in the transient table which are related to the import
     transient_table = imprt.destination.get_transient_table()
-    entities = db.session.scalars(
+    entities: list[Entity] = db.session.scalars(
         select(Entity).filter_by(destination=imprt.destination).order_by(Entity.order)
     ).all()
 
@@ -493,46 +493,54 @@ def preview_valid_data(scope, imprt):
             .all()
         )
         columns = [{"prop": field.dest_column, "name": field.name_field} for field in fields]
-        columns_to_count_unique_entities = [
-            transient_table.c[field.dest_column] for field in fields
-        ]
 
-        valid_data = db.session.execute(
-            select(*[transient_table.c[field.dest_field] for field in fields])
-            .distinct()
-            .where(
-                transient_table.c.id_import == imprt.id_import,
-                transient_table.c[entity.validity_column] == True,
-            )
-            .limit(100)
-        ).all()
+        id_field = (
+            entity.unique_column.dest_field if entity.unique_column.dest_field in fields else None
+        )
+        data_fields_query = [transient_table.c[field.dest_field] for field in fields]
 
-        n_valid_data = db.session.execute(
-            select(func.count(func.distinct(*columns_to_count_unique_entities)))
-            .select_from(transient_table)
-            .where(
-                transient_table.c.id_import == imprt.id_import,
-                transient_table.c[entity.validity_column] == True,
-            )
-        ).scalar()
+        query = select(*data_fields_query).where(
+            transient_table.c.id_import == imprt.id_import,
+            transient_table.c[entity.validity_column] == True,
+        )
+        valid_data = db.session.execute(query.limit(100)).all()
 
-        n_invalid_data = db.session.execute(
-            select(func.count(func.distinct(*columns_to_count_unique_entities)))
-            .select_from(transient_table)
+        def count_select(query_cte):
+            count_ = "*"
+            # if multiple entities and the entity has a unique column we base the count on the unique column
+
+            if entity.unique_column and len(entities) > 1 and id_field:
+                count_ = func.distinct(query_cte.c[id_field])
+            return count_
+
+        valid_data_cte = query.cte()
+        n_valid_data = db.session.scalar(
+            select(func.count(count_select(valid_data_cte))).select_from(valid_data_cte)
+        )
+
+        invalid_data_cte = (
+            select(data_fields_query)
             .where(
                 transient_table.c.id_import == imprt.id_import,
                 transient_table.c[entity.validity_column] == False,
             )
-        ).scalar()
+            .cte()
+        )
+
+        n_invalid_data = db.session.scalar(
+            select(func.count(count_select(invalid_data_cte))).select_from(invalid_data_cte)
+        )
+
         data["entities"].append(
             {
                 "entity": entity.as_dict(),
                 "columns": columns,
                 "valid_data": valid_data,
                 "n_valid_data": n_valid_data,
-                "n_invalid_data": n_invalid_data,
+                "n_invalid_data": n_invalid_data,  # NOTE: Not used in the frontend ...
             }
         )
+
     return jsonify(data)
 
 

backend/geonature/tests/imports/test_imports_occhab.py

@@ -633,3 +633,27 @@ def test_bbox_computation_transient(
                 ]
             ],
         }
+
+    @pytest.mark.parametrize("import_file_name", ["valid_file.csv"])
+    def test_preview_data(self, client, prepared_import):
+        valid_numbers = {
+            "station_valid": 7,
+            "station_invalid": 8,
+            "habitat_valid": 11,
+            "habitat_invalid": 23,
+        }
+        imprt = prepared_import
+        with logged_user(client, imprt.authors[0]):
+            response = client.get(url_for("import.preview_valid_data", import_id=imprt.id_import))
+        assert response.status_code == 200
+        data = response.json
+
+        index_data_station = 0 if data["entities"][0]["entity"]["code"] == "station" else 1
+        data_station = data["entities"][index_data_station]
+        data_habitat = data["entities"][0 if index_data_station == 1 else 1]
+
+        assert data_station["n_valid_data"] == valid_numbers["station_valid"]
+        assert data_station["n_invalid_data"] == valid_numbers["station_invalid"]
+
+        assert data_habitat["n_valid_data"] == valid_numbers["habitat_valid"]
+        assert data_habitat["n_invalid_data"] == valid_numbers["habitat_invalid"]

contrib/gn_module_occhab/backend/gn_module_occhab/migrations/9c3e1f98361f_fix_typo.py

@@ -0,0 +1,76 @@
+"""fix_typo
+
+Revision ID: 9c3e1f98361f
+Revises: c1a6b0793360
+Create Date: 2025-01-20 16:09:12.490217
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "9c3e1f98361f"
+down_revision = "c1a6b0793360"
+branch_labels = None
+depends_on = None
+
+OLD_NAME_MAPPING = "Occhab"
+NEW_NAME_MAPPING = "Occhab GeoNature"
+
+
+def get_table():
+    conn = op.get_bind()
+    metadata = sa.MetaData(bind=conn)
+    bib_fields = sa.Table("bib_fields", metadata, schema="gn_imports", autoload_with=op.get_bind())
+    destinations = sa.Table(
+        "bib_destinations", metadata, schema="gn_imports", autoload_with=op.get_bind()
+    )
+    t_mappings = sa.Table("t_mappings", metadata, schema="gn_imports", autoload_with=op.get_bind())
+    return bib_fields, destinations, t_mappings
+
+
+def get_id_dest_occhab():
+    _, destinations, _ = get_table()
+    id_destination_occhab = (
+        op.get_bind()
+        .execute(sa.select(destinations.c.id_destination).where(destinations.c.code == "occhab"))
+        .scalar()
+    )
+    return id_destination_occhab
+
+
+def upgrade():
+    bib_fields, destinations, t_mappings = get_table()
+    op.execute(
+        sa.update(bib_fields)
+        .where(
+            bib_fields.c.name_field == "depth_max",
+            bib_fields.c.id_destination == get_id_dest_occhab(),
+        )
+        .values(dest_field="depth_max")
+    )
+
+    op.execute(
+        sa.update(t_mappings)
+        .where(t_mappings.c.label == OLD_NAME_MAPPING)
+        .values(label=NEW_NAME_MAPPING)
+    )
+
+
+def downgrade():
+    bib_fields, _, t_mappings = get_table()
+    op.execute(
+        sa.update(bib_fields)
+        .where(
+            bib_fields.c.name_field == "depth_max",
+            bib_fields.c.id_destination == get_id_dest_occhab(),
+        )
+        .values(dest_field="depth_min")
+    )
+    op.execute(
+        sa.update(t_mappings)
+        .where(t_mappings.c.label == NEW_NAME_MAPPING)
+        .values(label=OLD_NAME_MAPPING)
+    )

frontend/src/app/modules/imports/components/import_process/import-step/import-step.component.html

@@ -111,7 +111,7 @@ <h5 class="card-title mb-0">
           label="{{ entityPreview.entity.label }}"
         >
           <ngx-datatable
-            *ngIf="tableReady"
+            *ngIf="tableReady && entityPreview.valid_data.length > 0"
             #table
             class="material striped"
             [rows]="entityPreview.valid_data"
@@ -124,6 +124,15 @@ <h5 class="card-title mb-0">
             [limit]="15"
             [selectionType]="single"
           ></ngx-datatable>
+          <ng-container *ngIf="entityPreview.valid_data.length === 0">
+            <div
+              class="alert alert-info mt-3"
+              role="alert"
+            >
+              Aucun(e) {{ entityPreview.entity.label.toLowerCase() }}
+              valide n'a été trouvée
+            </div>
+          </ng-container>
         </mat-tab>
       </mat-tab-group>
 

frontend/src/app/modules/imports/services/mappings/field-mapping.service.ts

@@ -300,6 +300,7 @@ export class FieldMappingService {
 
     this.currentFieldMappingSubscription = this.currentFieldMapping.subscribe((fieldMapping) => {
       this.fieldMappingStatus.unmapped = new Set(this.sourceFields);
+      this.fieldMappingStatus.mapped = new Set();
 
       if (fieldMapping === null) {
         this.mappingFormGroup.reset();