diff --git a/.credo.exs b/.credo.exs index 0edcce4878..354c44d44a 100644 --- a/.credo.exs +++ b/.credo.exs @@ -129,10 +129,15 @@ # Deprecated checks (these will be deleted after a grace period) # - {Credo.Check.Readability.Specs, false} + {Credo.Check.Readability.Specs, false}, # Custom checks can be created using `mix credo.gen.check`. # + + {Credo.Check.Warning.ForbiddenModule, + [ + modules: [Timex] + ]} ] } ] diff --git a/apps/shared/lib/date_time_display.ex b/apps/shared/lib/date_time_display.ex index 2b39f18931..6da044c163 100644 --- a/apps/shared/lib/date_time_display.ex +++ b/apps/shared/lib/date_time_display.ex @@ -34,7 +34,7 @@ defmodule Shared.DateTimeDisplay do def format_date(nil, _), do: "" def format_date(date, locale, iso_extended: true) do - date |> Timex.parse!("{ISO:Extended}") |> format_date(locale) + date |> TimeWrapper.parse!("{ISO:Extended}") |> format_date(locale) end @doc """ @@ -100,7 +100,7 @@ defmodule Shared.DateTimeDisplay do def format_datetime_to_paris(datetime, locale, options) when is_binary(datetime) do datetime - |> Timex.parse!("{ISO:Extended}") + |> TimeWrapper.parse!("{ISO:Extended}") |> format_datetime_to_paris(locale, options) end @@ -157,7 +157,7 @@ defmodule Shared.DateTimeDisplay do def format_time_to_paris(datetime, locale, options) when is_binary(datetime) do datetime - |> Timex.parse!("{ISO:Extended}") + |> TimeWrapper.parse!("{ISO:Extended}") |> format_time_to_paris(locale, options) end @@ -225,14 +225,11 @@ defmodule Shared.DateTimeDisplay do @spec convert_to_paris_time(DateTime.t() | NaiveDateTime.t()) :: DateTime.t() def convert_to_paris_time(%DateTime{} = dt) do - case Timex.Timezone.convert(dt, "Europe/Paris") do - %Timex.AmbiguousDateTime{after: dt} -> dt - %DateTime{} = dt -> dt - end + TimeWrapper.convert_to_paris_time(dt) end def convert_to_paris_time(%NaiveDateTime{} = ndt) do - ndt |> Timex.Timezone.convert("UTC") |> convert_to_paris_time() + ndt |> TimeWrapper.convert("UTC") |> convert_to_paris_time() end defp get_localized_datetime_format("en" = locale, options) do diff --git a/apps/shared/lib/time_wrapper.ex b/apps/shared/lib/time_wrapper.ex new file mode 100644 index 0000000000..c19d9b2385 --- /dev/null +++ b/apps/shared/lib/time_wrapper.ex @@ -0,0 +1,48 @@ +defmodule TimeWrapper do + @moduledoc """ + This module concentrates all the calls to `Timex` in a single place. + + The idea behind this module is 1. to reduce our dependency on `Timex`, and + 2. to ideally gradually replace calls by built-in Elixir `DateTime` calls, since + `Timex` filled a void in the language that has been partially filled now. + """ + + # credo:disable-for-this-file Credo.Check.Warning.ForbiddenModule + + def parse!(date_as_string, "{ISO:Extended}" = param) do + Timex.parse!(date_as_string, param) + end + + def parse!(date_as_string, "{YYYY}{0M}{0D}" = param) do + Timex.parse!(date_as_string, param) + end + + # NOTE: try not to use this, we will remove it. This is rfc2822 ; + # Plug encodes it, but there is no built-in decoder. + def parse!(datetime_as_string, "{WDshort}, {D} {Mshort} {YYYY} {h24}:{m}:{s} GMT" = param) do + Timex.parse!(datetime_as_string, param) + end + + def diff(first, second, :hours = param) do + Timex.diff(first, second, param) + end + + def now do + Timex.now() + end + + def shift(dt, months: months) do + Timex.shift(dt, months: months) + end + + def convert(dt, "UTC") do + Timex.Timezone.convert(dt, "UTC") + end + + def convert_to_paris_time(dt) do + case Timex.Timezone.convert(dt, "Europe/Paris") do + %Timex.AmbiguousDateTime{after: dt} -> dt + %DateTime{} = dt -> dt + end + end +end diff --git a/apps/shared/test/time_wrapper_test.exs b/apps/shared/test/time_wrapper_test.exs new file mode 100644 index 0000000000..9a6eff0309 --- /dev/null +++ b/apps/shared/test/time_wrapper_test.exs @@ -0,0 +1,4 @@ +defmodule TimeWrapperTest do + use ExUnit.Case, async: true + doctest TimeWrapper +end diff --git a/apps/transport/lib/db/dataset.ex b/apps/transport/lib/db/dataset.ex index 02ad3a5d8f..75a6731c67 100644 --- a/apps/transport/lib/db/dataset.ex +++ b/apps/transport/lib/db/dataset.ex @@ -154,7 +154,7 @@ defmodule DB.Dataset do @spec type_to_str_map() :: %{binary() => binary()} def type_to_str_map, do: %{ - "public-transit" => dgettext("db-dataset", "Public transit - static schedules"), + "public-transit" => dgettext("db-dataset", "Public transit"), "carpooling-areas" => dgettext("db-dataset", "Carpooling areas"), "carpooling-lines" => dgettext("db-dataset", "Carpooling lines"), "carpooling-offers" => dgettext("db-dataset", "Carpooling offers"), diff --git a/apps/transport/lib/irve/extractor.ex b/apps/transport/lib/irve/extractor.ex index 262f7b903e..ba50b62618 100644 --- a/apps/transport/lib/irve/extractor.ex +++ b/apps/transport/lib/irve/extractor.ex @@ -114,6 +114,7 @@ defmodule Transport.IRVE.Extractor do |> Enum.map(fn x -> Map.take(x, [ :dataset_id, + :http_status, :dataset_title, :dataset_organisation_name, :dataset_organisation_url, @@ -136,7 +137,7 @@ defmodule Transport.IRVE.Extractor do Transport.IRVE.Fetcher.get!(row[:url], compressed: false, decode_body: false) row - |> Map.put(:status, status) + |> Map.put(:http_status, status) |> Map.put(:index, index) |> then(fn x -> process_resource_body(x, body) end) end @@ -145,7 +146,7 @@ defmodule Transport.IRVE.Extractor do For a given resource and corresponding body, enrich data with extra stuff like estimated number of charge points. """ - def process_resource_body(%{status: 200} = row, body) do + def process_resource_body(%{http_status: 200} = row, body) do body = body |> String.split("\n") first_line = body |> hd() line_count = (body |> length) - 1 @@ -160,7 +161,7 @@ defmodule Transport.IRVE.Extractor do |> Map.put(:line_count, line_count) end - def process_body(row), do: row + def process_resource_body(row, _body), do: row @doc """ Save the outcome in the database for reporting. diff --git a/apps/transport/lib/jobs/gtfs_to_db.ex b/apps/transport/lib/jobs/gtfs_to_db.ex index 238e8a7207..2b64b38c9f 100644 --- a/apps/transport/lib/jobs/gtfs_to_db.ex +++ b/apps/transport/lib/jobs/gtfs_to_db.ex @@ -120,8 +120,8 @@ defmodule Transport.Jobs.GtfsToDB do friday: friday = r |> Map.fetch!("friday") |> String.to_integer(), saturday: saturday = r |> Map.fetch!("saturday") |> String.to_integer(), sunday: sunday = r |> Map.fetch!("sunday") |> String.to_integer(), - start_date: r |> Map.fetch!("start_date") |> Timex.parse!("{YYYY}{0M}{0D}") |> NaiveDateTime.to_date(), - end_date: r |> Map.fetch!("end_date") |> Timex.parse!("{YYYY}{0M}{0D}") |> NaiveDateTime.to_date() + start_date: r |> Map.fetch!("start_date") |> TimeWrapper.parse!("{YYYY}{0M}{0D}") |> NaiveDateTime.to_date(), + end_date: r |> Map.fetch!("end_date") |> TimeWrapper.parse!("{YYYY}{0M}{0D}") |> NaiveDateTime.to_date() } res @@ -214,7 +214,7 @@ defmodule Transport.Jobs.GtfsToDB do %{ data_import_id: data_import_id, service_id: r |> Map.fetch!("service_id"), - date: r |> Map.fetch!("date") |> Timex.parse!("{YYYY}{0M}{0D}") |> NaiveDateTime.to_date(), + date: r |> Map.fetch!("date") |> TimeWrapper.parse!("{YYYY}{0M}{0D}") |> NaiveDateTime.to_date(), exception_type: r |> Map.fetch!("exception_type") |> String.to_integer() } end) diff --git a/apps/transport/lib/jobs/oban_logger.ex b/apps/transport/lib/jobs/oban_logger.ex index ce1b3c059a..817b930954 100644 --- a/apps/transport/lib/jobs/oban_logger.ex +++ b/apps/transport/lib/jobs/oban_logger.ex @@ -1,8 +1,14 @@ defmodule Transport.Jobs.ObanLogger do @moduledoc """ - Logs the Oban job exceptions as warnings + Setup telemetry/logging for Oban. + + We: + - log job exceptions as warnings + - log Oban events related to the orchestration (notifier, queues, plugins etc.) + - we send an email when a job failed after its maximum attempt for jobs with a specific tag """ require Logger + @tag_email_on_failure "email_on_failure" @doc """ @@ -35,5 +41,22 @@ defmodule Transport.Jobs.ObanLogger do ) end - def setup, do: :telemetry.attach("oban-logger", [:oban, :job, :exception], &handle_event/4, nil) + def setup do + :telemetry.attach("oban-logger", [:oban, :job, :exception], &handle_event/4, nil) + + # Log recommended events for production. + # We leave out `job` events because job start/end can be quite noisy. + # https://hexdocs.pm/oban/preparing_for_production.html#logging + # https://hexdocs.pm/oban/Oban.Telemetry.html + # We may simplify this when + # https://github.com/oban-bg/oban/commit/13eabe3f8019e350ef979369a26f186bdf7be63e + # will be released. + events = [ + [:oban, :notifier, :switch], + [:oban, :queue, :shutdown], + [:oban, :stager, :switch] + ] + + :telemetry.attach_many("oban-default-logger", events, &Oban.Telemetry.handle_event/4, encode: true, level: :info) + end end diff --git a/apps/transport/lib/jobs/workflow.ex b/apps/transport/lib/jobs/workflow.ex index 906eec5abc..07a813af92 100644 --- a/apps/transport/lib/jobs/workflow.ex +++ b/apps/transport/lib/jobs/workflow.ex @@ -82,7 +82,7 @@ defmodule Transport.Jobs.Workflow do {:notification, :gossip, %{"success" => false, "job_id" => ^job_id} = notif} -> reason = notif |> Map.get("reason", "unknown reason") - {:error, "Job #{job_id} has failed: #{inspect(reason)}. Workflow is stopping here"} + {:error, "Job #{job_id} has failed: #{reason}. Workflow is stopping here"} end end end @@ -173,10 +173,17 @@ defmodule Transport.Jobs.Workflow do }, nil ) do + # `error` can be an error message or an `Oban.TimeoutError` exception. + # ```` + # %Oban.TimeoutError{ + # message: "Transport.Jobs.ResourceHistoryJob timed out after 1000ms", + # reason: :timeout + # } + # ``` Notifier.notify_workflow(%{meta: %{"workflow" => true}}, %{ "success" => false, "job_id" => job_id, - "reason" => error + "reason" => inspect(error) }) end diff --git a/apps/transport/lib/transport_web/live/backoffice/irve_dashboard_live.ex b/apps/transport/lib/transport_web/live/backoffice/irve_dashboard_live.ex index 25623eae92..4088708b62 100644 --- a/apps/transport/lib/transport_web/live/backoffice/irve_dashboard_live.ex +++ b/apps/transport/lib/transport_web/live/backoffice/irve_dashboard_live.ex @@ -3,7 +3,7 @@ defmodule TransportWeb.Backoffice.IRVEDashboardLive do use Phoenix.HTML import TransportWeb.Backoffice.JobsLive, only: [ensure_admin_auth_or_redirect: 3] import TransportWeb.Router.Helpers - import Helpers, only: [format_number: 1] + import Helpers, only: [format_number_maybe_nil: 2] import Ecto.Query, only: [last: 2] @impl true @@ -25,7 +25,12 @@ defmodule TransportWeb.Backoffice.IRVEDashboardLive do filtering_expression == "" || String.contains?(resource["dataset_organisation_name"] |> String.downcase(), filtering_expression) || - String.contains?(format_validity(resource["valid"]) |> inspect |> String.downcase(), filtering_expression) + String.contains?( + format_validity(resource["valid"], resource["http_status"]) + |> inspect + |> String.downcase(), + filtering_expression + ) end def assign_data(socket) do @@ -119,7 +124,9 @@ defmodule TransportWeb.Backoffice.IRVEDashboardLive do |> Enum.sort_by(fn x -> x["line_count"] end, :desc) end - def format_validity(false), do: {:safe, "KO"} - def format_validity(true), do: "OK" - def format_validity(nil), do: "Non testé" + def format_validity(false, 200), do: {:safe, "KO"} + def format_validity(true, 200), do: "OK" + def format_validity(nil, 200), do: "Non testé" + # mostly there to handle 404/500. Ignore validity and assume the resource is not reachable at all + def format_validity(_validity, http_status), do: {:safe, "KO (#{http_status})"} end diff --git a/apps/transport/lib/transport_web/live/backoffice/irve_dashboard_live.html.heex b/apps/transport/lib/transport_web/live/backoffice/irve_dashboard_live.html.heex index eef1805a15..717a4ed912 100644 --- a/apps/transport/lib/transport_web/live/backoffice/irve_dashboard_live.html.heex +++ b/apps/transport/lib/transport_web/live/backoffice/irve_dashboard_live.html.heex @@ -43,9 +43,9 @@ — <%= resource["last_modified"] |> String.slice(0..15) %> - <%= format_validity(resource["valid"]) %> + <%= format_validity(resource["valid"], resource["http_status"]) %> - <%= format_number(resource["line_count"]) %> + <%= format_number_maybe_nil(resource["line_count"], nil_result: "???") %> <% end %> diff --git a/apps/transport/lib/transport_web/live/discussions_live.ex b/apps/transport/lib/transport_web/live/discussions_live.ex index 476ecb28fe..577ef4569f 100644 --- a/apps/transport/lib/transport_web/live/discussions_live.ex +++ b/apps/transport/lib/transport_web/live/discussions_live.ex @@ -115,7 +115,7 @@ defmodule TransportWeb.DiscussionsLive do end) |> Enum.max(DateTime) - two_months_ago = DateTime.utc_now() |> Timex.shift(months: -2) + two_months_ago = DateTime.utc_now() |> TimeWrapper.shift(months: -2) DateTime.compare(two_months_ago, latest_comment_datetime) == :gt end diff --git a/apps/transport/lib/transport_web/plugs/router.ex b/apps/transport/lib/transport_web/plugs/router.ex index d1cbfb54a9..4ee9e686e6 100644 --- a/apps/transport/lib/transport_web/plugs/router.ex +++ b/apps/transport/lib/transport_web/plugs/router.ex @@ -2,7 +2,7 @@ defmodule TransportWeb.Plugs.Router do use Plug.Router plug(TransportWeb.Plugs.HealthCheck, at: "/health-check") - plug(TransportWeb.Plugs.Halt, if: {Transport.Application, :worker_only?}, message: "UP (WORKER-ONLY)") + plug(TransportWeb.Plugs.WorkerHealthcheck, if: {Transport.Application, :worker_only?}) plug(:match) plug(:dispatch) diff --git a/apps/transport/lib/transport_web/plugs/worker_healthcheck.ex b/apps/transport/lib/transport_web/plugs/worker_healthcheck.ex new file mode 100644 index 0000000000..3066e88e5f --- /dev/null +++ b/apps/transport/lib/transport_web/plugs/worker_healthcheck.ex @@ -0,0 +1,94 @@ +defmodule TransportWeb.Plugs.WorkerHealthcheck do + @moduledoc """ + A plug for the worker. + It can be conditionally enabled by passing an `:if` condition that will be evaluated. + + It displays: + - when the app was started + - the last attempt for Oban jobs + - if the system is healthy + + The system is considered healthy if the app was started recently or + if Oban attempted jobs recently. + """ + import Plug.Conn + require Logger + + @app_start_waiting_delay {20, :minute} + @oban_max_delay_since_last_attempt {60, :minute} + + def init(options), do: options + + def call(conn, opts) do + {mod, fun} = opts[:if] + + if apply(mod, fun, []) do + store_last_attempted_at_delay_metric() + status_code = if healthy_state?(), do: 200, else: 503 + + conn = + conn + |> put_resp_content_type("text/plain") + |> send_resp(status_code, """ + UP (WORKER-ONLY) + App start time: #{app_start_datetime()} + App started recently?: #{app_started_recently?()} + Oban last attempt: #{oban_last_attempted_at()} + Oban attempted jobs recently?: #{oban_attempted_jobs_recently?()} + Healthy state?: #{healthy_state?()} + """) + |> halt() + + # NOTE: Clever Cloud monitoring will better pick stuff back up + # if the app is completely down. + if !healthy_state?() do + Logger.info("Hot-fix: shutting down!!!") + # "Asynchronously and carefully stops the Erlang runtime system." + System.stop() + end + + conn + else + conn + end + end + + def store_last_attempted_at_delay_metric do + value = DateTime.diff(oban_last_attempted_at(), DateTime.utc_now(), :second) + Appsignal.add_distribution_value("oban.last_attempted_at_delay", value) + end + + def healthy_state? do + app_started_recently?() or oban_attempted_jobs_recently?() + end + + def app_started_recently? do + {delay, unit} = @app_start_waiting_delay + DateTime.diff(DateTime.utc_now(), app_start_datetime(), unit) < delay + end + + def app_start_datetime do + Transport.Cache.fetch(app_start_datetime_cache_key_name(), fn -> DateTime.utc_now() end, expire: nil) + end + + def app_start_datetime_cache_key_name, do: "#{__MODULE__}::app_start_datetime" + + def oban_attempted_jobs_recently? do + {delay, unit} = @oban_max_delay_since_last_attempt + DateTime.after?(oban_last_attempted_at(), DateTime.add(DateTime.utc_now(), -delay, unit)) + end + + def oban_last_attempted_at do + %Postgrex.Result{rows: [[delay]]} = + DB.Repo.query!(""" + SELECT MAX(attempted_at) + FROM oban_jobs + WHERE state = 'completed' + """) + + case delay do + nil -> DateTime.new!(~D[1970-01-01], ~T[00:00:00.000], "Etc/UTC") + %NaiveDateTime{} = nt -> DateTime.from_naive!(nt, "Etc/UTC") + end + end +end diff --git a/apps/transport/lib/transport_web/router.ex b/apps/transport/lib/transport_web/router.ex index 39ec8b7c28..cb9f5dfeb8 100644 --- a/apps/transport/lib/transport_web/router.ex +++ b/apps/transport/lib/transport_web/router.ex @@ -56,7 +56,6 @@ defmodule TransportWeb.Router do pipeline :reuser_space do plug(:browser) plug(:authentication_required, destination_path: "/infos_reutilisateurs") - plug(:check_reuser_space_enabled) end scope "/", OpenApiSpex.Plug do @@ -387,17 +386,6 @@ defmodule TransportWeb.Router do end end - def check_reuser_space_enabled(%Plug.Conn{} = conn, _) do - if TransportWeb.Session.display_reuser_space?(conn) do - conn - else - conn - |> put_flash(:info, dgettext("alert", "This feature is currently not available.")) - |> redirect(to: "/") - |> halt() - end - end - # Check that a secret key is passed in the URL in the `export_key` query parameter defp check_export_secret_key(%Plug.Conn{params: params} = conn, _) do export_key_value = Map.get(params, "export_key", "") diff --git a/apps/transport/lib/transport_web/session.ex b/apps/transport/lib/transport_web/session.ex index 27aebfad97..e8f73874a2 100644 --- a/apps/transport/lib/transport_web/session.ex +++ b/apps/transport/lib/transport_web/session.ex @@ -58,23 +58,6 @@ defmodule TransportWeb.Session do DB.Dataset.base_query() |> where([dataset: d], d.organization_id in ^org_ids) |> DB.Repo.exists?() end - @doc """ - A temporary helper method to determine if we should display "reuser space features". - Convenient method to find various entrypoints in the codebase: - - links and buttons to the reuser space - - follow dataset hearts (search results, dataset pages) - - reuser space - - Enable it for everybody but keep a "kill switch" to disable it quickly - by setting an environment variable and rebooting the app. - - transport.data.gouv.fr admins get access no matter what. - """ - def display_reuser_space?(%Plug.Conn{} = conn) do - feature_disabled = Application.fetch_env!(:transport, :disable_reuser_space) - admin?(conn) or not feature_disabled - end - @spec set_session_attribute_attribute(Plug.Conn.t(), binary(), boolean()) :: Plug.Conn.t() defp set_session_attribute_attribute(%Plug.Conn{} = conn, key, value) do current_user = current_user(conn) diff --git a/apps/transport/lib/transport_web/templates/dataset/_header_links.html.heex b/apps/transport/lib/transport_web/templates/dataset/_header_links.html.heex index 0515fd571d..25a3964251 100644 --- a/apps/transport/lib/transport_web/templates/dataset/_header_links.html.heex +++ b/apps/transport/lib/transport_web/templates/dataset/_header_links.html.heex @@ -3,32 +3,30 @@ <%= link("Backoffice", to: backoffice_page_path(@conn, :edit, @dataset.id)) %> · <% end %> - <%= if TransportWeb.Session.display_reuser_space?(@conn) do %> - - <%= if @current_user do %> - <%= if @is_producer do %> - <%= link(dgettext("default", "Producer space"), - to: espace_producteur_path(@conn, :edit_dataset, @dataset.id, utm_campaign: "dataset_details"), + + <%= if @current_user do %> + <%= if @is_producer do %> + <%= link(dgettext("default", "Producer space"), + to: espace_producteur_path(@conn, :edit_dataset, @dataset.id, utm_campaign: "dataset_details"), + target: "_blank" + ) %> + <% else %> + <%= if @follows_dataset do %> + <%= link(dgettext("default", "Reuser space"), + to: reuser_space_path(@conn, :datasets_edit, @dataset.id, utm_campaign: "dataset_details"), target: "_blank" ) %> <% else %> - <%= if @follows_dataset do %> - <%= link(dgettext("default", "Reuser space"), - to: reuser_space_path(@conn, :datasets_edit, @dataset.id, utm_campaign: "dataset_details"), - target: "_blank" - ) %> - <% else %> - <%= link(dgettext("default", "Reuser space"), - to: reuser_space_path(@conn, :espace_reutilisateur, utm_campaign: "dataset_details"), - target: "_blank" - ) %> - <% end %> + <%= link(dgettext("default", "Reuser space"), + to: reuser_space_path(@conn, :espace_reutilisateur, utm_campaign: "dataset_details"), + target: "_blank" + ) %> <% end %> - <% else %> - <%= link(dgettext("default", "Reuser space"), - to: page_path(@conn, :infos_reutilisateurs, utm_campaign: "dataset_details"), - target: "_blank" - ) %> <% end %> + <% else %> + <%= link(dgettext("default", "Reuser space"), + to: page_path(@conn, :infos_reutilisateurs, utm_campaign: "dataset_details"), + target: "_blank" + ) %> <% end %> diff --git a/apps/transport/lib/transport_web/templates/dataset/details.html.heex b/apps/transport/lib/transport_web/templates/dataset/details.html.heex index 10de3427b1..686432d3a2 100644 --- a/apps/transport/lib/transport_web/templates/dataset/details.html.heex +++ b/apps/transport/lib/transport_web/templates/dataset/details.html.heex @@ -239,11 +239,9 @@
- <%= if TransportWeb.Session.display_reuser_space?(@conn) do %> - <%= live_render(@conn, TransportWeb.Live.FollowDatasetLive, - session: %{"current_user" => @current_user, "dataset_id" => @dataset.id} - ) %> - <% end %> + <%= live_render(@conn, TransportWeb.Live.FollowDatasetLive, + session: %{"current_user" => @current_user, "dataset_id" => @dataset.id} + ) %> @@ -273,14 +271,6 @@ text: Dataset.type_to_str(@dataset.type), icon: icon_type_path(@dataset) ) %> - <%= if real_time_public_transit?(@dataset) do %> - <%= render("_dataset_type.html", - conn: @conn, - link: dataset_path(@conn, :index, type: "public-transit", filter: "has_realtime"), - text: dgettext("page-index", "Public transport - realtime traffic"), - icon: icon_type_path("real-time-public-transit") - ) %> - <% end %>
<%= render("_licence.html", conn: @conn, dataset: @dataset) %> diff --git a/apps/transport/lib/transport_web/templates/dataset/index.html.heex b/apps/transport/lib/transport_web/templates/dataset/index.html.heex index 4e301295fe..86a7d9e53e 100644 --- a/apps/transport/lib/transport_web/templates/dataset/index.html.heex +++ b/apps/transport/lib/transport_web/templates/dataset/index.html.heex @@ -196,9 +196,7 @@
- <%= if not is_nil(@current_user) and TransportWeb.Session.display_reuser_space?(@conn) do %> - - <% end %> + <%= unless is_nil(icon_type_path(dataset)) do %> <%= img_tag(icon_type_path(dataset), alt: dataset.type) %> <% end %> diff --git a/apps/transport/lib/transport_web/templates/layout/_header.html.heex b/apps/transport/lib/transport_web/templates/layout/_header.html.heex index 6e222369e7..3950024dfb 100644 --- a/apps/transport/lib/transport_web/templates/layout/_header.html.heex +++ b/apps/transport/lib/transport_web/templates/layout/_header.html.heex @@ -97,11 +97,9 @@ to: page_path(@conn, :espace_producteur, utm_campaign: "menu_dropdown") ) %> <% end %> - <%= if TransportWeb.Session.display_reuser_space?(@conn) do %> - <%= link(gettext("Reuser space"), - to: reuser_space_path(@conn, :espace_reutilisateur, utm_campaign: "menu_dropdown") - ) %> - <% end %> + <%= link(gettext("Reuser space"), + to: reuser_space_path(@conn, :espace_reutilisateur, utm_campaign: "menu_dropdown") + ) %> <%= dgettext("page-index", "Producer space") %> - <%= if TransportWeb.Session.display_reuser_space?(@conn) do %> - - <%= dgettext("page-index", "Reuser space") %> - - <% end %> + + <%= dgettext("page-index", "Reuser space") %> +
diff --git a/apps/transport/lib/transport_web/views/dataset_view.ex b/apps/transport/lib/transport_web/views/dataset_view.ex index ecf07a04db..54919eee7f 100644 --- a/apps/transport/lib/transport_web/views/dataset_view.ex +++ b/apps/transport/lib/transport_web/views/dataset_view.ex @@ -367,12 +367,6 @@ defmodule TransportWeb.DatasetView do |> Enum.filter(&Resource.documentation?/1) end - def real_time_public_transit?(%Dataset{type: "public-transit"} = dataset) do - not Enum.empty?(real_time_official_resources(dataset)) - end - - def real_time_public_transit?(%Dataset{}), do: false - def community_resources(dataset), do: Dataset.community_resources(dataset) def licence_url(licence) when licence in ["fr-lo", "lov2"], @@ -426,7 +420,7 @@ defmodule TransportWeb.DatasetView do resources |> Enum.filter(fn r -> r.format == "csv" end) |> Enum.reject(fn r -> Resource.community_resource?(r) or Resource.documentation?(r) end) - |> Enum.max_by(& &1.last_update, DateTime, fn -> nil end) + |> Enum.max_by(&{&1.type, &1.last_update}, TransportWeb.DatasetView.ResourceTypeSortKey, fn -> nil end) end def get_resource_to_display(%Dataset{type: type, resources: resources}) @@ -585,3 +579,13 @@ defmodule TransportWeb.DatasetView do "fa fa-heart #{value}" |> String.trim() end end + +defmodule TransportWeb.DatasetView.ResourceTypeSortKey do + def compare({left_type, left_last_update}, {right_type, right_last_update}) do + cond do + left_type == right_type -> DateTime.compare(left_last_update, right_last_update) + left_type == "main" -> :gt + true -> :lt + end + end +end diff --git a/apps/transport/lib/validators/validator_selection.ex b/apps/transport/lib/validators/validator_selection.ex index a5ffd7fc6b..b786fd9c1e 100644 --- a/apps/transport/lib/validators/validator_selection.ex +++ b/apps/transport/lib/validators/validator_selection.ex @@ -33,7 +33,14 @@ defmodule Transport.ValidatorsSelection.Impl do def validators(%{format: "GTFS"}), do: [Validators.GTFSTransport] def validators(%{format: "gtfs-rt"}), do: [Validators.GTFSRT] def validators(%{format: "gbfs"}), do: [Validators.GBFSValidator] - def validators(%{format: "NeTEx"}), do: [Validators.NeTEx] + + def validators(%{format: "NeTEx"}) do + if netex_validator_enabled?() do + [Validators.NeTEx] + else + [] + end + end def validators(%{schema_name: schema_name}) when not is_nil(schema_name) do cond do @@ -49,4 +56,6 @@ defmodule Transport.ValidatorsSelection.Impl do end def validators(_), do: [] + + defp netex_validator_enabled?, do: !Application.fetch_env!(:transport, :disable_netex_validator) end diff --git a/apps/transport/priv/gettext/alert.pot b/apps/transport/priv/gettext/alert.pot index 6d63c9f33e..bb5c2fb819 100644 --- a/apps/transport/priv/gettext/alert.pot +++ b/apps/transport/priv/gettext/alert.pot @@ -29,7 +29,3 @@ msgstr "" #, elixir-autogen, elixir-format msgid "Unable to get this dataset for the moment" msgstr "" - -#, elixir-autogen, elixir-format -msgid "This feature is currently not available." -msgstr "" diff --git a/apps/transport/priv/gettext/db-dataset.pot b/apps/transport/priv/gettext/db-dataset.pot index 5fbb0754fc..38a0347f40 100644 --- a/apps/transport/priv/gettext/db-dataset.pot +++ b/apps/transport/priv/gettext/db-dataset.pot @@ -75,7 +75,7 @@ msgid "Charging & refuelling stations" msgstr "" #, elixir-autogen, elixir-format -msgid "Public transit - static schedules" +msgid "Public transit" msgstr "" #, elixir-autogen, elixir-format diff --git a/apps/transport/priv/gettext/en/LC_MESSAGES/alert.po b/apps/transport/priv/gettext/en/LC_MESSAGES/alert.po index 7d7f0874ab..aacda5c257 100644 --- a/apps/transport/priv/gettext/en/LC_MESSAGES/alert.po +++ b/apps/transport/priv/gettext/en/LC_MESSAGES/alert.po @@ -29,7 +29,3 @@ msgstr "" #, elixir-autogen, elixir-format msgid "Unable to get this dataset for the moment" msgstr "" - -#, elixir-autogen, elixir-format -msgid "This feature is currently not available." -msgstr "" diff --git a/apps/transport/priv/gettext/en/LC_MESSAGES/db-dataset.po b/apps/transport/priv/gettext/en/LC_MESSAGES/db-dataset.po index fe4d7b66cc..4e092e1180 100644 --- a/apps/transport/priv/gettext/en/LC_MESSAGES/db-dataset.po +++ b/apps/transport/priv/gettext/en/LC_MESSAGES/db-dataset.po @@ -76,7 +76,7 @@ msgid "Charging & refuelling stations" msgstr "" #, elixir-autogen, elixir-format, fuzzy -msgid "Public transit - static schedules" +msgid "Public transit" msgstr "" #, elixir-autogen, elixir-format diff --git a/apps/transport/priv/gettext/fr/LC_MESSAGES/alert.po b/apps/transport/priv/gettext/fr/LC_MESSAGES/alert.po index fef9ff23a2..c704b97474 100644 --- a/apps/transport/priv/gettext/fr/LC_MESSAGES/alert.po +++ b/apps/transport/priv/gettext/fr/LC_MESSAGES/alert.po @@ -29,7 +29,3 @@ msgstr "Une erreur a eu lieu lors de la récupération de vos ressources" #, elixir-autogen, elixir-format msgid "Unable to get this dataset for the moment" msgstr "Impossible de récupérer ce jeu de données pour le moment" - -#, elixir-autogen, elixir-format -msgid "This feature is currently not available." -msgstr "La fonctionnalité n'est pas disponible pour le moment." diff --git a/apps/transport/priv/gettext/fr/LC_MESSAGES/db-dataset.po b/apps/transport/priv/gettext/fr/LC_MESSAGES/db-dataset.po index bc96cfb20f..f605986ae6 100644 --- a/apps/transport/priv/gettext/fr/LC_MESSAGES/db-dataset.po +++ b/apps/transport/priv/gettext/fr/LC_MESSAGES/db-dataset.po @@ -76,8 +76,8 @@ msgid "Charging & refuelling stations" msgstr "Stations de réapprovisionnement de véhicules" #, elixir-autogen, elixir-format, fuzzy -msgid "Public transit - static schedules" -msgstr "Transport public collectif - horaires théoriques" +msgid "Public transit" +msgstr "Transport public collectif" #, elixir-autogen, elixir-format msgid "Road data" diff --git a/apps/transport/test/transport/data_checker_test.exs b/apps/transport/test/transport/data_checker_test.exs index 44b9be7ced..509c0f3143 100644 --- a/apps/transport/test/transport/data_checker_test.exs +++ b/apps/transport/test/transport/data_checker_test.exs @@ -340,7 +340,7 @@ defmodule Transport.DataCheckerTest do subject: "Nouveaux jeux de données référencés", text_body: nil, html_body: - ~r|
  • #{dataset.custom_title} - \(Transport public collectif - horaires théoriques\)
    • | + ~r|
  • #{dataset.custom_title} - \(Transport public collectif\)
    • | ) assert [ diff --git a/apps/transport/test/transport/irve/irve_extractor_test.exs b/apps/transport/test/transport/irve/irve_extractor_test.exs index 56fe4e97d4..c13b37fa5c 100644 --- a/apps/transport/test/transport/irve/irve_extractor_test.exs +++ b/apps/transport/test/transport/irve/irve_extractor_test.exs @@ -117,6 +117,38 @@ defmodule Transport.IRVE.ExtractorTest do orig_resource |> Map.put(:index, 0) |> Map.put(:line_count, 3) + |> Map.put(:http_status, 200) + |> Map.delete(:url) + ] + end + + test "handles non-200 response" do + resources = [ + orig_resource = %{ + url: expected_url = "https://static.data.gouv.fr/resources/something/something.csv", + dataset_id: "the-dataset-id", + dataset_title: "the-dataset-title", + resource_id: "the-resource-id", + resource_title: "the-resource-title", + valid: true + } + ] + + Transport.Req.Mock + |> expect(:get!, fn _request, options -> + assert options[:url] == expected_url + + %Req.Response{ + status: 404, + body: "there is nothing here" + } + end) + + # parsed resources must be enriched with line count & index, and url removed + assert Transport.IRVE.Extractor.download_and_parse_all(resources) == [ + orig_resource + |> Map.put(:index, 0) + |> Map.put(:http_status, 404) |> Map.delete(:url) ] end diff --git a/apps/transport/test/transport/jobs/import_gbfs_feed_contact_point_job_test.exs b/apps/transport/test/transport/jobs/import_gbfs_feed_contact_point_job_test.exs index 25489f5f9d..be33a379f0 100644 --- a/apps/transport/test/transport/jobs/import_gbfs_feed_contact_point_job_test.exs +++ b/apps/transport/test/transport/jobs/import_gbfs_feed_contact_point_job_test.exs @@ -209,7 +209,7 @@ defmodule Transport.Test.Transport.Jobs.ImportGBFSFeedContactEmailJobTest do assert :ok == perform_job(ImportGBFSFeedContactEmailJob, %{}) - assert [first_contact, new_contact] = DB.Contact |> DB.Repo.all() |> Enum.sort_by(& &1.inserted_at) + assert [first_contact, new_contact] = DB.Contact |> DB.Repo.all() |> Enum.sort_by(& &1.id) assert %DB.Contact{id: ^existing_gbfs_contact_id, email: ^gbfs_2_email} = first_contact assert "Example" == Transport.GBFSMetadata.operator(gbfs_1.url) diff --git a/apps/transport/test/transport/jobs/oban_logger_test.exs b/apps/transport/test/transport/jobs/oban_logger_test.exs index f33c27ccce..4b4d449aaa 100644 --- a/apps/transport/test/transport/jobs/oban_logger_test.exs +++ b/apps/transport/test/transport/jobs/oban_logger_test.exs @@ -44,4 +44,17 @@ defmodule Transport.Test.Transport.Jobs.ObanLoggerTest do "Un job Oban Transport.Test.Transport.Jobs.ObanLoggerJobTag vient d'échouer, il serait bien d'investiguer." ) end + + test "oban default logger is set up for important components" do + registered_handlers = Enum.filter(:telemetry.list_handlers([]), &(&1.id == "oban-default-logger")) + + assert Enum.count(registered_handlers) > 0 + + components = + registered_handlers + |> Enum.map(fn %{event_name: [:oban, component, _]} -> component end) + |> MapSet.new() + + assert MapSet.new([:notifier, :queue, :stager]) == MapSet.new(components) + end end diff --git a/apps/transport/test/transport_web/controllers/dataset_controller_test.exs b/apps/transport/test/transport_web/controllers/dataset_controller_test.exs index f84498b915..c57436d36e 100644 --- a/apps/transport/test/transport_web/controllers/dataset_controller_test.exs +++ b/apps/transport/test/transport_web/controllers/dataset_controller_test.exs @@ -850,7 +850,7 @@ defmodule TransportWeb.DatasetControllerTest do test "dataset-page-title", %{conn: conn} do [ - {%{"type" => "public-transit"}, "Transport public collectif - horaires théoriques"}, + {%{"type" => "public-transit"}, "Transport public collectif"}, {%{"type" => "public-transit", "filter" => "has_realtime"}, "Transport public collectif - horaires temps réel"}, {%{"modes" => ["rail"]}, "Transport ferroviaire"} ] diff --git a/apps/transport/test/transport_web/controllers/dataset_search_test.exs b/apps/transport/test/transport_web/controllers/dataset_search_test.exs index 29fdd8b474..7c3d8c0349 100644 --- a/apps/transport/test/transport_web/controllers/dataset_search_test.exs +++ b/apps/transport/test/transport_web/controllers/dataset_search_test.exs @@ -62,7 +62,7 @@ defmodule TransportWeb.DatasetSearchControllerTest do test "GET /datasets filter", %{conn: conn} do conn = conn |> get(dataset_path(conn, :index)) # info dans les filtres à gauche des datasets - assert html_response(conn, 200) =~ "Transport public collectif - horaires théoriques (2)" + assert html_response(conn, 200) =~ "Transport public collectif (2)" end describe "list datasets" do @@ -156,7 +156,7 @@ defmodule TransportWeb.DatasetSearchControllerTest do test "GET /datasets?type=public-transit&licence=odc-odbl", %{conn: conn} do conn = conn |> get(dataset_path(conn, :index), %{type: "public-transit", licence: "odc-odbl"}) - assert html_response(conn, 200) =~ "Transport public collectif - horaires théoriques (1)" + assert html_response(conn, 200) =~ "Transport public collectif (1)" assert html_response(conn, 200) =~ "Horaires Angers" refute html_response(conn, 200) =~ "Horaires Laval" diff --git a/apps/transport/test/transport_web/controllers/reuser_space_controller_test.exs b/apps/transport/test/transport_web/controllers/reuser_space_controller_test.exs index 92f760790f..f6ab217596 100644 --- a/apps/transport/test/transport_web/controllers/reuser_space_controller_test.exs +++ b/apps/transport/test/transport_web/controllers/reuser_space_controller_test.exs @@ -28,17 +28,6 @@ defmodule TransportWeb.ReuserSpaceControllerTest do # Feedback form is displayed refute content |> Floki.parse_document!() |> Floki.find("form.feedback-form") |> Enum.empty?() end - - test "reuser space disabled by killswitch", %{conn: conn} do - old_value = Application.fetch_env!(:transport, :disable_reuser_space) - Application.put_env(:transport, :disable_reuser_space, true) - conn = Plug.Test.init_test_session(conn, %{current_user: %{}}) - refute TransportWeb.Session.display_reuser_space?(conn) - conn = conn |> get(@home_url) - assert redirected_to(conn, 302) == "/" - assert Phoenix.Flash.get(conn.assigns.flash, :info) =~ "La fonctionnalité n'est pas disponible pour le moment" - Application.put_env(:transport, :disable_reuser_space, old_value) - end end describe "datasets_edit" do diff --git a/apps/transport/test/transport_web/live_views/discussions_live_test.exs b/apps/transport/test/transport_web/live_views/discussions_live_test.exs index 5f5e7fc469..92e47f0a31 100644 --- a/apps/transport/test/transport_web/live_views/discussions_live_test.exs +++ b/apps/transport/test/transport_web/live_views/discussions_live_test.exs @@ -209,7 +209,7 @@ defmodule Transport.TransportWeb.DiscussionsLiveTest do end defp iso8601_string_x_months_ago(x) do - DateTime.utc_now() |> Timex.shift(months: -x) |> DateTime.to_iso8601() + DateTime.utc_now() |> TimeWrapper.shift(months: -x) |> DateTime.to_iso8601() end defp discussions do diff --git a/apps/transport/test/transport_web/live_views/followed_datasets_live_test.exs b/apps/transport/test/transport_web/live_views/followed_datasets_live_test.exs index 023e8f7fe1..8ec358c8ab 100644 --- a/apps/transport/test/transport_web/live_views/followed_datasets_live_test.exs +++ b/apps/transport/test/transport_web/live_views/followed_datasets_live_test.exs @@ -27,7 +27,7 @@ defmodule TransportWeb.Live.FollowedDatasetsLiveTest do {"select", [{"id", "type"}, {"name", "type"}], [ {"option", [{"selected", "selected"}, {"value", ""}], ["Tout"]}, - {"option", [{"value", "public-transit"}], ["Transport public collectif - horaires théoriques"]}, + {"option", [{"value", "public-transit"}], ["Transport public collectif"]}, {"option", [{"value", "bike-scooter-sharing"}], ["Vélos et trottinettes en libre-service"]} ]} ] == view |> element("form select") |> render() |> Floki.parse_document!() diff --git a/apps/transport/test/transport_web/plugs/worker_healthcheck_test.exs b/apps/transport/test/transport_web/plugs/worker_healthcheck_test.exs new file mode 100644 index 0000000000..ce6da59f9d --- /dev/null +++ b/apps/transport/test/transport_web/plugs/worker_healthcheck_test.exs @@ -0,0 +1,129 @@ +defmodule TransportWeb.Plugs.WorkerHealthcheckTest do + # async: false is required because we use real in-memory caching in these tests, + # and we swap application config (shared state) + use TransportWeb.ConnCase, async: false + alias TransportWeb.Plugs.WorkerHealthcheck + + @cache_name Transport.Cache.Cachex.cache_name() + @cache_key WorkerHealthcheck.app_start_datetime_cache_key_name() + + setup do + # Use a real in-memory cache for these tests to test the caching mecanism + old_value = Application.fetch_env!(:transport, :cache_impl) + Application.put_env(:transport, :cache_impl, Transport.Cache.Cachex) + + on_exit(fn -> + Application.put_env(:transport, :cache_impl, old_value) + Cachex.reset(@cache_name) + end) + + Ecto.Adapters.SQL.Sandbox.checkout(DB.Repo) + end + + describe "healthy_state?" do + test "app was started recently, no Oban jobs" do + assert WorkerHealthcheck.app_started_recently?() + refute WorkerHealthcheck.oban_attempted_jobs_recently?() + assert WorkerHealthcheck.healthy_state?() + end + + test "app was not started recently, Oban jobs have not been attempted recently" do + datetime = DateTime.add(DateTime.utc_now(), -30, :minute) + Cachex.put(@cache_name, @cache_key, datetime) + + refute WorkerHealthcheck.app_started_recently?() + refute WorkerHealthcheck.oban_attempted_jobs_recently?() + refute WorkerHealthcheck.healthy_state?() + end + + test "app was not started recently, Oban jobs have been attempted recently" do + datetime = DateTime.add(DateTime.utc_now(), -30, :minute) + Cachex.put(@cache_name, @cache_key, datetime) + + # A completed job was attempted 55 minutes ago + Transport.Jobs.ResourceUnavailableJob.new(%{resource_id: 1}) + |> Oban.insert!() + |> Ecto.Changeset.change(attempted_at: DateTime.add(DateTime.utc_now(), -55, :minute), state: "completed") + |> DB.Repo.update!() + + refute WorkerHealthcheck.app_started_recently?() + assert WorkerHealthcheck.oban_attempted_jobs_recently?() + assert WorkerHealthcheck.healthy_state?() + end + end + + describe "app_started_recently?" do + test "value is set when executed for the first time" do + assert {:ok, false} == Cachex.exists?(@cache_name, @cache_key) + # Calling for the first time creates the key + assert WorkerHealthcheck.app_started_recently?() + assert {:ok, true} == Cachex.exists?(@cache_name, @cache_key) + + # Calling again does not refresh the initial value + start_datetime = WorkerHealthcheck.app_start_datetime() + WorkerHealthcheck.app_started_recently?() + assert start_datetime == WorkerHealthcheck.app_start_datetime() + + # Key does not expire + assert {:ok, nil} == Cachex.ttl(@cache_name, @cache_key) + end + + test "acceptable delay is 20 minutes" do + # Just right + datetime = DateTime.add(DateTime.utc_now(), -19, :minute) + Cachex.put(@cache_name, @cache_key, datetime) + + assert WorkerHealthcheck.app_started_recently?() + + # Too long ago + datetime = DateTime.add(DateTime.utc_now(), -21, :minute) + Cachex.put(@cache_name, @cache_key, datetime) + refute WorkerHealthcheck.app_started_recently?() + end + end + + describe "oban_attempted_jobs_recently?" do + test "job attempted recently" do + # Attempted less than 60 minutes ago + Transport.Jobs.ResourceUnavailableJob.new(%{resource_id: 1}) + |> Oban.insert!() + |> Ecto.Changeset.change(attempted_at: DateTime.add(DateTime.utc_now(), -59, :minute), state: "completed") + |> DB.Repo.update!() + + assert WorkerHealthcheck.oban_attempted_jobs_recently?() + end + + test "job attempted too long ago" do + # Attempted more than 60 minutes ago + Transport.Jobs.ResourceUnavailableJob.new(%{resource_id: 1}) + |> Oban.insert!() + |> Ecto.Changeset.change(attempted_at: DateTime.add(DateTime.utc_now(), -61, :minute), state: "completed") + |> DB.Repo.update!() + + refute WorkerHealthcheck.oban_attempted_jobs_recently?() + end + end + + describe "call" do + test "healthy system", %{conn: conn} do + assert WorkerHealthcheck.app_started_recently?() + refute WorkerHealthcheck.oban_attempted_jobs_recently?() + assert WorkerHealthcheck.healthy_state?() + + assert conn |> WorkerHealthcheck.call(if: {__MODULE__, :plug_enabled?}) |> text_response(200) + end + + test "unhealthy system", %{conn: conn} do + datetime = DateTime.add(DateTime.utc_now(), -30, :minute) + Cachex.put(@cache_name, @cache_key, datetime) + + refute WorkerHealthcheck.app_started_recently?() + refute WorkerHealthcheck.oban_attempted_jobs_recently?() + refute WorkerHealthcheck.healthy_state?() + + assert conn |> WorkerHealthcheck.call(if: {__MODULE__, :plug_enabled?}) |> text_response(503) + end + end + + def plug_enabled?, do: true +end diff --git a/apps/transport/test/transport_web/routing/headers_and_cookies_test.exs b/apps/transport/test/transport_web/routing/headers_and_cookies_test.exs index 2b1d7bffe5..49d566d0a1 100644 --- a/apps/transport/test/transport_web/routing/headers_and_cookies_test.exs +++ b/apps/transport/test/transport_web/routing/headers_and_cookies_test.exs @@ -24,7 +24,7 @@ defmodule TransportWeb.HeadersAndCookiesTest do "expires" => datetime } = Plug.Conn.Cookies.decode(header) - datetime = Timex.parse!(datetime, "{WDshort}, {D} {Mshort} {YYYY} {h24}:{m}:{s} GMT") - assert_in_delta Timex.diff(datetime, Timex.now(), :hours), 15 * 24, 1 + datetime = TimeWrapper.parse!(datetime, "{WDshort}, {D} {Mshort} {YYYY} {h24}:{m}:{s} GMT") + assert_in_delta TimeWrapper.diff(datetime, TimeWrapper.now(), :hours), 15 * 24, 1 end end diff --git a/apps/transport/test/transport_web/session_test.exs b/apps/transport/test/transport_web/session_test.exs index 65b92205a7..7a969102b6 100644 --- a/apps/transport/test/transport_web/session_test.exs +++ b/apps/transport/test/transport_web/session_test.exs @@ -55,24 +55,6 @@ defmodule TransportWeb.SessionTest do end end - describe "display_reuser_space?" do - test "killswitch can disable the reuser space" do - old_value = Application.fetch_env!(:transport, :disable_reuser_space) - Application.put_env(:transport, :disable_reuser_space, true) - conn = Plug.Test.init_test_session(%Plug.Conn{}, %{}) - refute TransportWeb.Session.display_reuser_space?(conn) - Application.put_env(:transport, :disable_reuser_space, old_value) - end - - test "admins get access when killswitch is enabled" do - old_value = Application.fetch_env!(:transport, :disable_reuser_space) - Application.put_env(:transport, :disable_reuser_space, true) - conn = Plug.Test.init_test_session(%Plug.Conn{}, %{current_user: %{"is_admin" => true}}) - assert TransportWeb.Session.display_reuser_space?(conn) - Application.put_env(:transport, :disable_reuser_space, old_value) - end - end - def pan_org do %{"slug" => "equipe-transport-data-gouv-fr", "name" => "PAN", "id" => @pan_org_id} end diff --git a/apps/transport/test/transport_web/views/dataset_view_test.exs b/apps/transport/test/transport_web/views/dataset_view_test.exs index 4247f881bd..567ca5bf59 100644 --- a/apps/transport/test/transport_web/views/dataset_view_test.exs +++ b/apps/transport/test/transport_web/views/dataset_view_test.exs @@ -67,6 +67,68 @@ defmodule TransportWeb.DatasetViewTest do assert get_resource_to_display(dataset_only_roads) == nil end + test "resource to display for BNLC" do + dataset_two_bnlcs = %DB.Dataset{ + type: "carpooling-areas", + resources: [ + %DB.Resource{ + id: 1, + url: "https://example.com/bnlc.csv", + format: "csv", + schema_name: "etalab/schema-lieux-covoiturage", + last_update: ~U[2016-05-24 13:26:08Z], + type: "other" + }, + %DB.Resource{ + id: 2, + url: "https://example.com/bnlc-consolidated.csv", + format: "csv", + schema_name: "etalab/schema-lieux-covoiturage", + last_update: ~U[2016-05-24 13:25:08Z], + type: "other" + } + ] + } + + dataset_two_bnlcs_with_the_main = %DB.Dataset{ + type: "carpooling-areas", + resources: [ + %DB.Resource{ + id: 1, + url: "https://example.com/bnlc.csv", + format: "csv", + schema_name: "etalab/schema-lieux-covoiturage", + last_update: ~U[2016-05-24 13:26:08Z], + type: "other" + }, + %DB.Resource{ + id: 2, + url: "https://example.com/bnlc-consolidated-1.csv", + format: "csv", + schema_name: "etalab/schema-lieux-covoiturage", + # not the latest, but we prefer the main one + last_update: ~U[2016-05-24 13:24:08Z], + type: "main" + }, + %DB.Resource{ + id: 3, + url: "https://example.com/bnlc-consolidated-2.csv", + format: "csv", + schema_name: "etalab/schema-lieux-covoiturage", + # not the latest, but we prefer the main one + last_update: ~U[2016-05-24 13:25:08Z], + type: "main" + } + ] + } + + # Display the latest + assert get_resource_to_display(dataset_two_bnlcs).id == 1 + + # Display the latest main one + assert get_resource_to_display(dataset_two_bnlcs_with_the_main).id == 3 + end + test "test data is up to date" do assert "tipi.bison-fute.gouv.fr" == Application.fetch_env!(:transport, :bison_fute_host) end diff --git a/config/runtime.exs b/config/runtime.exs index 3c72728c3e..af99631d5f 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -38,8 +38,8 @@ config :transport, worker: worker, webserver: webserver, # kill switches: set specific variable environments to disable features - disable_reuser_space: System.get_env("DISABLE_REUSER_SPACE") in ["1", "true"], - disable_national_gtfs_map: System.get_env("DISABLE_NATIONAL_GTFS_MAP") in ["1", "true"] + disable_national_gtfs_map: System.get_env("DISABLE_NATIONAL_GTFS_MAP") in ["1", "true"], + disable_netex_validator: System.get_env("DISABLE_NETEX_VALIDATOR") in ["1", "true"] config :unlock, enforce_ttl: webserver diff --git a/mix.lock b/mix.lock index 63db11d2fe..0f7718d356 100644 --- a/mix.lock +++ b/mix.lock @@ -23,7 +23,7 @@ "crontab": {:hex, :crontab, "1.1.13", "3bad04f050b9f7f1c237809e42223999c150656a6b2afbbfef597d56df2144c5", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "d67441bec989640e3afb94e123f45a2bc42d76e02988c9613885dc3d01cf7085"}, "csv": {:hex, :csv, "3.2.1", "6d401f1ed33acb2627682a9ab6021e96d33ca6c1c6bccc243d8f7e2197d032f5", [:mix], [], "hexpm", "8f55a0524923ae49e97ff2642122a2ce7c61e159e7fe1184670b2ce847aee6c8"}, "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"}, - "decimal": {:hex, :decimal, "2.2.0", "df3d06bb9517e302b1bd265c1e7f16cda51547ad9d99892049340841f3e15836", [:mix], [], "hexpm", "af8daf87384b51b7e611fb1a1f2c4d4876b65ef968fa8bd3adf44cff401c7f21"}, + "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"}, "decorator": {:hex, :decorator, "1.4.0", "a57ac32c823ea7e4e67f5af56412d12b33274661bb7640ec7fc882f8d23ac419", [:mix], [], "hexpm", "0a07cedd9083da875c7418dea95b78361197cf2bf3211d743f6f7ce39656597f"}, "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"}, "digital_token": {:hex, :digital_token, "0.6.0", "13e6de581f0b1f6c686f7c7d12ab11a84a7b22fa79adeb4b50eec1a2d278d258", [:mix], [{:cldr_utils, "~> 2.17", [hex: :cldr_utils, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "2455d626e7c61a128b02a4a8caddb092548c3eb613ac6f6a85e4cbb6caddc4d1"}, @@ -88,7 +88,7 @@ "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, "nx": {:hex, :nx, "0.7.3", "51ff45d9f9ff58b616f4221fa54ccddda98f30319bb8caaf86695234a469017a", [:mix], [{:complex, "~> 0.5", [hex: :complex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5ff29af84f08db9bda66b8ef7ce92ab583ab4f983629fe00b479f1e5c7c705a6"}, "oauth2": {:hex, :oauth2, "2.1.0", "beb657f393814a3a7a8a15bd5e5776ecae341fd344df425342a3b6f1904c2989", [:mix], [{:tesla, "~> 1.5", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm", "8ac07f85b3307dd1acfeb0ec852f64161b22f57d0ce0c15e616a1dfc8ebe2b41"}, - "oban": {:hex, :oban, "2.18.0", "092d20bfd3d70c7ecb70960f8548d300b54bb9937c7f2e56b388f3a9ed02ec68", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "aace1eff6f8227ae38d4274af967d96f051c2f0a5152f2ef9809dd1f97866745"}, + "oban": {:hex, :oban, "2.18.3", "1608c04f8856c108555c379f2f56bc0759149d35fa9d3b825cb8a6769f8ae926", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "36ca6ca84ef6518f9c2c759ea88efd438a3c81d667ba23b02b062a0aa785475e"}, "open_api_spex": {:hex, :open_api_spex, "3.20.0", "d4fcf1ee297aa94a673cddb92734eb0bc7cac698be93949a223a50f724e3af89", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0 or ~> 6.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "2e9beea71142ff09f8f935579b39406e2c6b5a3978e7235978d7faf2f90cd081"}, "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, "phoenix": {:hex, :phoenix, "1.7.14", "a7d0b3f1bc95987044ddada111e77bd7f75646a08518942c72a8440278ae7825", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "c7859bc56cc5dfef19ecfc240775dae358cbaa530231118a9e014df392ace61a"},