diff --git a/lib/arrow/shuttles/definition_upload.ex b/lib/arrow/shuttles/definition_upload.ex new file mode 100644 index 00000000..2991284f --- /dev/null +++ b/lib/arrow/shuttles/definition_upload.ex @@ -0,0 +1,85 @@ +defmodule Arrow.Shuttles.DefinitionUpload do + @moduledoc "functions for extracting shuttle defintions from xlsx uploads" + alias Arrow.Shuttles.Stop + + @doc """ + Parses a shuttle definition xlsx worksheet and returns a list of two stop_id lists + """ + @spec extract_stop_ids_from_upload(%{path: String.t()}) :: + {:ok, {list(Stop.id()), list(Stop.id())} | {:error, [String.t(), ...]}} + def extract_stop_ids_from_upload(%{path: xlsx_path}) do + with tids when is_list(tids) <- Xlsxir.multi_extract(xlsx_path), + {:ok, + %{ + "Direction 0 STOPS" => direction_0_tab_tid, + "Direction 1 STOPS" => direction_1_tab_tid + }} <- get_xlsx_tab_tids(tids), + {:ok, direction_0_stop_ids} <- parse_direction_tab(direction_0_tab_tid), + {:ok, direction_1_stop_ids} <- parse_direction_tab(direction_1_tab_tid) do + {:ok, {direction_0_stop_ids, direction_1_stop_ids}} + else + {:error, error} -> {:ok, {:error, [error]}} + {:errors, errors} -> {:ok, {:error, errors}} + end + end + + defp get_xlsx_tab_tids(tab_tids) do + tab_map = + tab_tids + |> Enum.map(fn {:ok, tid} -> + name = Xlsxir.get_info(tid, :name) + + if name in ["Direction 0 STOPS", "Direction 1 STOPS"] do + {name, tid} + else + Xlsxir.close(tid) + nil + end + end) + |> Enum.reject(&is_nil/1) + |> Map.new() + + case {tab_map["Direction 0 STOPS"], tab_map["Direction 1 STOPS"]} do + {nil, nil} -> {:error, "Missing tabs for both directions"} + {nil, _} -> {:error, "Missing Direction 0 STOPS tab"} + {_, nil} -> {:error, "Missing Direction 1 STOPS tab"} + _ -> {:ok, tab_map} + end + end + + def parse_direction_tab(table_id) do + tab_data = + table_id + |> Xlsxir.get_list() + # Cells that have been touched but are empty can return nil + |> Enum.reject(fn list -> Enum.all?(list, &is_nil/1) end) + |> tap(fn _ -> Xlsxir.close(table_id) end) + + parse_stop_ids(tab_data) + end + + defp parse_stop_ids([headers | _] = data) do + if stop_id_col_index = Enum.find_index(headers, &(&1 === "Stop ID")) do + stop_ids = data |> Enum.drop(1) |> Enum.map(&Enum.at(&1, stop_id_col_index)) + + errors = + stop_ids + |> Enum.with_index(1) + |> Enum.reduce([], fn {stop_id, i}, acc -> + append_if( + acc, + is_nil(stop_id) or not is_integer(stop_id), + "Missing/invalid stop ID on row #{i + 1}" + ) + end) + + if Enum.empty?(errors), do: {:ok, stop_ids}, else: {:errors, errors} + else + {:errors, ["Unable to parse Stop ID column"]} + end + end + + defp append_if(list, condition, item) do + if condition, do: [item | list], else: list + end +end diff --git a/lib/arrow_web/live/shuttle_live/shuttle_live.ex b/lib/arrow_web/live/shuttle_live/shuttle_live.ex index 752a854f..0abf2c2d 100644 --- a/lib/arrow_web/live/shuttle_live/shuttle_live.ex +++ b/lib/arrow_web/live/shuttle_live/shuttle_live.ex @@ -3,7 +3,7 @@ defmodule ArrowWeb.ShuttleViewLive do import Phoenix.HTML.Form alias Arrow.Shuttles - alias Arrow.Shuttles.{Route, RouteStop, Shape, Shuttle} + alias Arrow.Shuttles.{DefinitionUpload, Route, RouteStop, Shape, Shuttle} alias ArrowWeb.ShapeView embed_templates "shuttle_live/*" @@ -18,6 +18,7 @@ defmodule ArrowWeb.ShuttleViewLive do attr :shapes, :list, required: true attr :map_props, :map, required: false, default: %{} attr :errors, :map, required: false, default: %{route_stops: %{}} + attr :uploads, :any def shuttle_form(assigns) do ~H""" @@ -64,6 +65,18 @@ defmodule ArrowWeb.ShuttleViewLive do /> +
+
+ <.link_button + class="btn-primary" + phx-click={JS.dispatch("click", to: "##{@uploads.definition.ref}")} + target="_blank" + > + <.live_file_input upload={@uploads.definition} class="hidden" /> + Upload Shuttle Definition XLSX + +
+
<%= live_react_component("Components.ShapeStopViewMap", @map_props, id: "shuttle-view-map") %>

define route

@@ -337,6 +350,11 @@ defmodule ArrowWeb.ShuttleViewLive do |> assign(:shapes, shapes) |> assign(:map_props, %{layers: routes_to_layers(shuttle.routes)}) |> assign(:errors, %{route_stops: %{}}) + |> allow_upload(:definition, + accept: ~w(.xlsx), + progress: &handle_progress/3, + auto_upload: true + ) {:ok, socket} end @@ -362,6 +380,11 @@ defmodule ArrowWeb.ShuttleViewLive do |> assign(:shapes, shapes) |> assign(:map_props, %{layers: routes_to_layers(shuttle.routes)}) |> assign(:errors, %{route_stops: %{}}) + |> allow_upload(:definition, + accept: ~w(.xlsx), + progress: &handle_progress/3, + auto_upload: true + ) {:ok, socket} end @@ -522,6 +545,32 @@ defmodule ArrowWeb.ShuttleViewLive do } end + defp update_route_changeset_with_uploaded_stops(route_changeset, stop_ids, direction_id) do + if Ecto.Changeset.get_field(route_changeset, :direction_id) == direction_id do + new_route_stops = + stop_ids + |> Enum.with_index() + |> Enum.map(fn {stop_id, i} -> + Arrow.Shuttles.RouteStop.changeset( + %Arrow.Shuttles.RouteStop{}, + %{ + direction_id: direction_id, + stop_sequence: i, + display_stop_id: Integer.to_string(stop_id) + } + ) + end) + + Ecto.Changeset.put_assoc( + route_changeset, + :route_stops, + new_route_stops + ) + else + route_changeset + end + end + @spec get_stop_travel_times(list({:ok, any()})) :: {:ok, list(number())} | {:error, any()} defp get_stop_travel_times(stop_coordinates) do @@ -630,4 +679,42 @@ defmodule ArrowWeb.ShuttleViewLive do {:noreply, socket |> assign(form: form) |> update_map(change)} end + + defp handle_progress(:definition, entry, socket) do + socket = clear_flash(socket) + + if entry.done? do + case consume_uploaded_entry(socket, entry, &DefinitionUpload.extract_stop_ids_from_upload/1) do + {:error, errors} -> + {:noreply, put_flash(socket, :errors, {"Failed to upload definition:", errors})} + + stop_ids -> + socket = populate_stop_ids(socket, stop_ids) + + {:noreply, socket} + end + else + {:noreply, socket} + end + end + + defp populate_stop_ids(socket, stop_ids) do + changeset = socket.assigns.form.source + existing_routes = Ecto.Changeset.get_assoc(changeset, :routes) + + new_routes = + Enum.map(existing_routes, fn route_changeset -> + direction_id = Ecto.Changeset.get_field(route_changeset, :direction_id) + + update_route_changeset_with_uploaded_stops( + route_changeset, + elem(stop_ids, direction_id |> Atom.to_string() |> String.to_integer()), + direction_id + ) + end) + + changeset = Ecto.Changeset.put_assoc(changeset, :routes, new_routes) + + socket |> assign(:form, to_form(changeset)) |> update_map(changeset) + end end diff --git a/lib/arrow_web/live/shuttle_live/shuttle_view_live.html.heex b/lib/arrow_web/live/shuttle_live/shuttle_view_live.html.heex index af2458b4..936b50d3 100644 --- a/lib/arrow_web/live/shuttle_live/shuttle_view_live.html.heex +++ b/lib/arrow_web/live/shuttle_live/shuttle_view_live.html.heex @@ -10,6 +10,7 @@ shapes={@shapes} map_props={@map_props} errors={@errors} + uploads={@uploads} /> <.back navigate={~p"/shuttles"}>Back to shuttles diff --git a/mix.exs b/mix.exs index f903a231..91be2cf0 100644 --- a/mix.exs +++ b/mix.exs @@ -96,7 +96,8 @@ defmodule Arrow.MixProject do compile: false, depth: 1}, {:sax_map, "~> 1.2"}, - {:unzip, "~> 0.12.0"} + {:unzip, "~> 0.12.0"}, + {:xlsxir, "~> 1.6"} ] end diff --git a/mix.lock b/mix.lock index 19e290f2..1ab175ee 100644 --- a/mix.lock +++ b/mix.lock @@ -14,6 +14,7 @@ "ecto_psql_extras": {:hex, :ecto_psql_extras, "0.8.2", "79350a53246ac5ec27326d208496aebceb77fa82a91744f66a9154560f0759d3", [:mix], [{:ecto_sql, "~> 3.7", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:postgrex, "> 0.16.0 and < 0.20.0", [hex: :postgrex, repo: "hexpm", optional: false]}, {:table_rex, "~> 3.1.1 or ~> 4.0.0", [hex: :table_rex, repo: "hexpm", optional: false]}], "hexpm", "6149c1c4a5ba6602a76cb09ee7a269eb60dab9694a1dbbb797f032555212de75"}, "ecto_sql": {:hex, :ecto_sql, "3.12.1", "c0d0d60e85d9ff4631f12bafa454bc392ce8b9ec83531a412c12a0d415a3a4d0", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.12", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.19 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "aff5b958a899762c5f09028c847569f7dfb9cc9d63bdb8133bff8a5546de6bf5"}, "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"}, + "erlsom": {:hex, :erlsom, "1.5.1", "c8fe2babd33ff0846403f6522328b8ab676f896b793634cfe7ef181c05316c03", [:rebar3], [], "hexpm", "7965485494c5844dd127656ac40f141aadfa174839ec1be1074e7edf5b4239eb"}, "esbuild": {:hex, :esbuild, "0.8.2", "5f379dfa383ef482b738e7771daf238b2d1cfb0222bef9d3b20d4c8f06c7a7ac", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "558a8a08ed78eb820efbfda1de196569d8bfa9b51e8371a1934fbb31345feda7"}, "ex_aws": {:hex, :ex_aws, "2.5.6", "6f642e0f82eff10a9b470044f084b81a791cf15b393d647ea5f3e65da2794e3d", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:req, "~> 0.3", [hex: :req, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c69eec59e31fdd89d0beeb1d97e16518dd1b23ad95b3d5c9f1dcfec23d97f960"}, "ex_aws_rds": {:hex, :ex_aws_rds, "2.0.2", "38dd8e83d57cf4b7286c4f6f5c978f700c40c207ffcdd6ca5d738e5eba933f9a", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}], "hexpm", "9e5b5cc168077874cbd0d29ba65d01caf1877e705fb5cecacf0667dd19bfa75c"}, @@ -81,4 +82,5 @@ "web_driver_client": {:hex, :web_driver_client, "0.2.0", "63b76cd9eb3b0716ec5467a0f8bead73d3d9612e63f7560d21357f03ad86e31a", [:mix], [{:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:tesla, "~> 1.3", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm", "83cc6092bc3e74926d1c8455f0ce927d5d1d36707b74d9a65e38c084aab0350f"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, "websock_adapter": {:hex, :websock_adapter, "0.5.7", "65fa74042530064ef0570b75b43f5c49bb8b235d6515671b3d250022cb8a1f9e", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "d0f478ee64deddfec64b800673fd6e0c8888b079d9f3444dd96d2a98383bdbd1"}, + "xlsxir": {:hex, :xlsxir, "1.6.4", "d1e69439cbd9edc1190950f9f883ac364e1f31641e0395ccdb27761791b169a3", [:mix], [{:erlsom, "~> 1.5", [hex: :erlsom, repo: "hexpm", optional: false]}], "hexpm", "38e91f65eb8a4c8dea07d941c8b7e21baf8c8d4938232395c9ffd19d2eb071f2"}, } diff --git a/test/arrow_web/live/shuttle_live/shuttle_live_test.exs b/test/arrow_web/live/shuttle_live/shuttle_live_test.exs index 1564203d..a7977bdb 100644 --- a/test/arrow_web/live/shuttle_live/shuttle_live_test.exs +++ b/test/arrow_web/live/shuttle_live/shuttle_live_test.exs @@ -401,6 +401,98 @@ defmodule ArrowWeb.ShuttleLiveTest do end end + describe "upload definition" do + @tag :authenticated_admin + test "sets route_stops using uploaded stop IDs", %{conn: conn} do + {:ok, new_live, _html} = live(conn, ~p"/shuttles/new") + + definition = + file_input(new_live, "#shuttle-form", :definition, [ + %{ + name: "valid.xlsx", + content: File.read!("test/support/fixtures/xlsx/valid.xlsx") + } + ]) + + direction_0_stop_sequence = ~w(9328 5327 5271) + direction_1_stop_sequence = ~w(5271 5072 9328) + html = render_upload(definition, "valid.xlsx") + + direction_0_stop_rows = Floki.find(html, "#stops-dir-0 > .row") + direction_1_stop_rows = Floki.find(html, "#stops-dir-1 > .row") + + for {stop_id, index} <- Enum.with_index(direction_0_stop_sequence) do + assert [^stop_id] = + Floki.attribute( + direction_0_stop_rows, + "[data-stop_sequence=#{index}] > div.form-group > input[type=text]", + "value" + ) + end + + for {stop_id, index} <- Enum.with_index(direction_1_stop_sequence) do + assert [^stop_id] = + Floki.attribute( + direction_1_stop_rows, + "[data-stop_sequence=#{index}] > div.form-group > input[type=text]", + "value" + ) + end + end + + @tag :authenticated_admin + test "displays error for missing/invalid tabs", %{conn: conn} do + {:ok, new_live, _html} = live(conn, ~p"/shuttles/new") + + definition = + file_input(new_live, "#shuttle-form", :definition, [ + %{ + name: "invalid_missing_tab.xlsx", + content: File.read!("test/support/fixtures/xlsx/invalid_missing_tab.xlsx") + } + ]) + + page = render_upload(definition, "invalid_missing_tab.xlsx") + assert page =~ "Failed to upload definition:" + assert page =~ "Missing Direction 0 STOPS tab" + end + + @tag :authenticated_admin + test "displays error for missing/invalid data", %{conn: conn} do + {:ok, new_live, _html} = live(conn, ~p"/shuttles/new") + + definition = + file_input(new_live, "#shuttle-form", :definition, [ + %{ + name: "invalid_missing_data.xlsx", + content: File.read!("test/support/fixtures/xlsx/invalid_missing_data.xlsx") + } + ]) + + page = render_upload(definition, "invalid_missing_data.xlsx") + assert page =~ "Failed to upload definition:" + assert page =~ "Missing/invalid stop ID on row 3" + end + + @tag :authenticated_admin + + test "displays error for missing headers", %{conn: conn} do + {:ok, new_live, _html} = live(conn, ~p"/shuttles/new") + + definition = + file_input(new_live, "#shuttle-form", :definition, [ + %{ + name: "invalid_missing_headers.xlsx", + content: File.read!("test/support/fixtures/xlsx/invalid_missing_headers.xlsx") + } + ]) + + page = render_upload(definition, "invalid_missing_headers.xlsx") + assert page =~ "Failed to upload definition:" + assert page =~ "Unable to parse Stop ID column" + end + end + defp create_shuttle_with_stops(_) do shuttle = shuttle_fixture(%{}, true) %{shuttle: shuttle} diff --git a/test/support/fixtures/xlsx/invalid_missing_data.xlsx b/test/support/fixtures/xlsx/invalid_missing_data.xlsx new file mode 100644 index 00000000..db0d14e6 Binary files /dev/null and b/test/support/fixtures/xlsx/invalid_missing_data.xlsx differ diff --git a/test/support/fixtures/xlsx/invalid_missing_headers.xlsx b/test/support/fixtures/xlsx/invalid_missing_headers.xlsx new file mode 100644 index 00000000..05b613d1 Binary files /dev/null and b/test/support/fixtures/xlsx/invalid_missing_headers.xlsx differ diff --git a/test/support/fixtures/xlsx/invalid_missing_tab.xlsx b/test/support/fixtures/xlsx/invalid_missing_tab.xlsx new file mode 100644 index 00000000..b4fb96c0 Binary files /dev/null and b/test/support/fixtures/xlsx/invalid_missing_tab.xlsx differ diff --git a/test/support/fixtures/xlsx/valid.xlsx b/test/support/fixtures/xlsx/valid.xlsx new file mode 100644 index 00000000..17040851 Binary files /dev/null and b/test/support/fixtures/xlsx/valid.xlsx differ