Merge branch 'master' into remove-ods-comms

README.md

@@ -223,9 +223,11 @@ Some custom one shot tasks are available.
 
 To run a custom task: `mix <custom task>`
 
-* `Transport.ImportAom`: import the aom data from the cerema
-* `Transport.ImportEPCI`: import the french EPCI from data.gouv
-* `Transport.OpenApiSpec`: generate an OpenAPI specification file
+* `mix Transport.ImportAOMs`: import the aom data from the cerema
+* `mix Transport.ImportCommunes`: import the french communes from data.gouv
+* `mix Transport.ImportEPCI`: import the french EPCI from data.gouv
+* `mix Transport.ImportDepartements`: import the french Départements from data.gouv
+* `mix Transport.OpenApiSpec`: generate an OpenAPI specification file
 
 ## Testing emails
 

apps/gbfs/mix.exs

@@ -11,7 +11,6 @@ defmodule GBFS.MixProject do
       lockfile: "../../mix.lock",
       elixir: "~> 1.8",
       elixirc_paths: elixirc_paths(Mix.env()),
-      compilers: [:phoenix] ++ Mix.compilers(),
       start_permanent: Mix.env() == :prod,
       deps: deps(),
       test_coverage: [tool: ExCoveralls],
@@ -38,7 +37,7 @@ defmodule GBFS.MixProject do
     [
       {:cachex, "~> 3.5"},
       {:httpoison, "~> 2.1"},
-      {:phoenix, "~> 1.6.2"},
+      {:phoenix, "~> 1.7.0"},
       {:sweet_xml, ">= 0.0.0"},
       {:jason, ">= 0.0.0"},
       {:cors_plug, "~> 3.0"},

apps/shared/lib/helpers.ex

@@ -57,13 +57,4 @@ defmodule Helpers do
       dates -> dates |> Enum.max(DateTime) |> DateTime.to_iso8601()
     end
   end
-
-  @spec admin?(map | nil) :: boolean
-  def admin?(%{} = user) do
-    user
-    |> Map.get("organizations", [])
-    |> Enum.any?(fn org -> org["slug"] == "equipe-transport-data-gouv-fr" end)
-  end
-
-  def admin?(nil), do: false
 end

apps/shared/mix.exs

@@ -46,7 +46,7 @@ defmodule Shared.MixProject do
       {:finch, "~> 0.8"},
       # Required for the ConditionalJSONEncoder shared component, but
       # there is probably a way to avoid that?
-      {:phoenix, "~> 1.6.2"},
+      {:phoenix, "~> 1.7.0"},
       # The global app config references Sentry.LoggerBackend. We add it in "shared"
       # as an implicit dependency, to ensure `Sentry.LoggerBackend` is always defined,
       # otherwise running tests for an individual umbrella sub-app will raise error.

apps/transport/client/yarn.lock

@@ -4041,7 +4041,7 @@ path-type@^4.0.0:
   integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==
 
 "phoenix@file:../../../deps/phoenix":
-  version "1.6.16"
+  version "1.7.10"
 
 "phoenix_html@file:../../../deps/phoenix_html":
   version "3.3.3"

apps/transport/lib/db/commune.ex

@@ -4,7 +4,7 @@ defmodule DB.Commune do
   """
   use Ecto.Schema
   use TypedEctoSchema
-  alias DB.{AOM, Departement, Region}
+  alias DB.{AOM, Departement, EPCI, Region}
   alias Geo.MultiPolygon
 
   typed_schema "commune" do
@@ -20,5 +20,6 @@ defmodule DB.Commune do
     belongs_to(:aom_res, AOM, references: :composition_res_id)
     belongs_to(:region, Region)
     belongs_to(:departement, Departement, foreign_key: :departement_insee, references: :insee, type: :string)
+    belongs_to(:epci, EPCI, foreign_key: :epci_insee, references: :insee, type: :string)
   end
 end

apps/transport/lib/db/epci.ex

@@ -1,19 +1,34 @@
 defmodule DB.EPCI do
   @moduledoc """
-  EPCI schema
+  EPCI schema.
 
-  Link the EPCI to some Communes.
-  The EPCI are loaded once and for all by the task transport/lib/transport/import_epci.ex
+  The EPCI are loaded by the task transport/lib/transport/import_epci.ex.
+  The EPCI imported are only "à fiscalité propre". This excludes Etablissements Publics Territoriaux.
+  This allows to have a 1 to 1 relation between a commune and an EPCI.
   """
   use Ecto.Schema
   use TypedEctoSchema
+  import Ecto.Changeset
 
   typed_schema "epci" do
-    field(:code, :string)
+    field(:insee, :string)
     field(:nom, :string)
+    field(:type, :string)
+    field(:mode_financement, :string)
+    field(:geom, Geo.PostGIS.Geometry) :: Geo.MultiPolygon.t()
+    has_many(:communes, DB.Commune, foreign_key: :epci_insee)
+  end
 
-    # for the moment we don't need a link relational link to the Commune table,
-    # so we only store an array of insee code
-    field(:communes_insee, {:array, :string}, default: [])
+  def changeset(epci, attrs) do
+    epci
+    |> cast(attrs, [:insee, :nom, :geom, :type, :mode_financement])
+    |> validate_required([:insee, :nom, :geom, :type, :mode_financement])
+    |> validate_inclusion(:type, allowed_types())
+    |> validate_inclusion(:mode_financement, allowed_mode_financement())
   end
+
+  defp allowed_types,
+    do: ["Communauté d'agglomération", "Communauté urbaine", "Communauté de communes", "Métropole", "Métropole de Lyon"]
+
+  defp allowed_mode_financement, do: ["Fiscalité professionnelle unique", "Fiscalité additionnelle"]
 end

apps/transport/lib/jobs/dataset_now_on_nap_notification_job.ex

@@ -27,8 +27,7 @@ defmodule Transport.Jobs.DatasetNowOnNAPNotificationJob do
         Phoenix.View.render_to_string(TransportWeb.EmailView, "dataset_now_on_nap.html",
           dataset_url: TransportWeb.Router.Helpers.dataset_url(TransportWeb.Endpoint, :details, dataset.slug),
           dataset_custom_title: dataset.custom_title,
-          contact_email_address: Application.get_env(:transport, :contact_email),
-          espace_producteur_url: TransportWeb.Router.Helpers.page_url(TransportWeb.Endpoint, :espace_producteur)
+          contact_email_address: Application.get_env(:transport, :contact_email)
         )
       )
 

apps/transport/lib/jobs/update_contacts_job.ex

@@ -25,12 +25,23 @@ defmodule Transport.Jobs.UpdateContactsJob do
     DB.Contact.base_query()
     |> where([contact: c], c.datagouv_user_id in ^ids)
     |> DB.Repo.all()
-    |> Enum.each(fn %DB.Contact{datagouv_user_id: datagouv_user_id} = contact ->
-      {:ok, %{"organizations" => organizations}} = Datagouvfr.Client.User.get(datagouv_user_id)
+    |> Enum.each(&update_contact/1)
+  end
+
+  defp update_contact(%DB.Contact{datagouv_user_id: datagouv_user_id} = contact) do
+    # https://doc.data.gouv.fr/api/reference/#/users/get_user
+    # 404 status code: User not found
+    # 410 status code: User is not active or has been deleted
+    case Datagouvfr.Client.User.get(datagouv_user_id) do
+      {:ok, %{"organizations" => organizations}} ->
+        contact
+        |> DB.Contact.changeset(%{organizations: organizations})
+        |> DB.Repo.update!()
 
-      contact
-      |> DB.Contact.changeset(%{organizations: organizations})
-      |> DB.Repo.update!()
-    end)
+      {:error, reason} when reason in [:not_found, :gone] ->
+        contact
+        |> DB.Contact.changeset(%{organizations: [], datagouv_user_id: nil})
+        |> DB.Repo.update!()
+    end
   end
 end

apps/transport/lib/mix/tasks/transport/import_aoms.ex

@@ -1,4 +1,4 @@
-defmodule Mix.Tasks.Transport.ImportAoms do
+defmodule Mix.Tasks.Transport.ImportAOMs do
   @moduledoc """
   Import the AOM files and updates the database.
 
@@ -13,7 +13,7 @@ defmodule Mix.Tasks.Transport.ImportAoms do
 
   This is a one shot import task, run when the AOM have changed, at least every year.
 
-  The import can be launched through mix transport.import_aoms
+  The import can be launched through mix Transport.ImportAOMs
   """
 
   @shortdoc "Refreshes the database table `aom` with the latest data"

apps/transport/lib/mix/tasks/transport/import_communes.ex

@@ -1,6 +1,6 @@
 defmodule Mix.Tasks.Transport.ImportCommunes do
   @moduledoc """
-  Import or updates commune data (list, geometry) from official sources. Run with `mix transport.import_communes`.
+  Import or updates commune data (list, geometry) from official sources. Run with `mix Transport.ImportCommunes`.
 
   """
   @shortdoc "Refreshes the database table `commune` with the latest data"

apps/transport/lib/mix/tasks/transport/import_epci.ex

@@ -1,47 +1,45 @@
 defmodule Mix.Tasks.Transport.ImportEPCI do
   @moduledoc """
   Import the EPCI file to get the relation between the cities and the EPCI
+  Run: mix transport.importEPCI
   """
 
   use Mix.Task
   import Ecto.Query
   alias Ecto.Changeset
-  alias DB.{EPCI, Repo}
+  alias DB.{Commune, EPCI, Repo}
   require Logger
 
   @epci_file "https://unpkg.com/@etalab/decoupage-administratif@3.1.1/data/epci.json"
+  @epci_geojson_url "http://etalab-datasets.geo.data.gouv.fr/contours-administratifs/2023/geojson/epci-100m.geojson"
 
-  def run(params) do
-    Logger.info("importing epci")
+  def run(_params) do
+    Logger.info("Importing EPCIs")
 
-    if params[:no_start] do
-      HTTPoison.start()
-    else
-      Mix.Task.run("app.start", [])
-    end
+    Mix.Task.run("app.start")
 
-    with {:ok, %HTTPoison.Response{status_code: 200, body: body}} <- HTTPoison.get(@epci_file),
-         {:ok, json} <- Jason.decode(body) do
-      json |> Enum.each(&insert_epci/1)
-
-      # Remove EPCIs that have been removed
-      epci_codes = Enum.map(json, & &1["code"])
-      EPCI |> where([e], e.code not in ^epci_codes) |> Repo.delete_all()
-
-      nb_epci = Repo.aggregate(EPCI, :count, :id)
-      Logger.info("#{nb_epci} are now in database")
-      :ok
-    else
-      e ->
-        Logger.warning("impossible to fetch epci file, error #{inspect(e)}")
-        :error
-    end
+    %{status: 200, body: json} = Req.get!(@epci_file, connect_options: [timeout: 15_000], receive_timeout: 15_000)
+    check_communes_list(json)
+    geojsons = geojson_by_insee()
+
+    json |> Enum.each(&insert_epci(&1, geojsons))
+    json |> Enum.each(&update_communes_epci/1)
+
+    # Remove EPCIs that have been removed
+    epci_codes = json |> Enum.map(& &1["code"])
+    EPCI |> where([e], e.insee not in ^epci_codes) |> Repo.delete_all()
+
+    nb_epci = Repo.aggregate(EPCI, :count, :id)
+    Logger.info("#{nb_epci} are now in database")
+    Logger.info("Ensure valid geometries and rectify if needed.")
+    ensure_valid_geometries()
+    :ok
   end
 
   @spec get_or_create_epci(binary()) :: EPCI.t()
   defp get_or_create_epci(code) do
     EPCI
-    |> Repo.get_by(code: code)
+    |> Repo.get_by(insee: code)
     |> case do
       nil ->
         %EPCI{}
@@ -51,21 +49,82 @@ defmodule Mix.Tasks.Transport.ImportEPCI do
     end
   end
 
-  @spec insert_epci(map()) :: any()
-  defp insert_epci(%{"code" => code, "nom" => nom, "membres" => m}) do
+  @spec insert_epci(map(), map()) :: any()
+  defp insert_epci(%{"code" => code, "nom" => nom, "type" => type, "modeFinancement" => mode_financement}, geojsons) do
     code
     |> get_or_create_epci()
-    |> Changeset.change(%{
-      code: code,
+    |> EPCI.changeset(%{
+      insee: code,
       nom: nom,
-      communes_insee: get_insees(m)
+      type: normalize_type(type),
+      mode_financement: normalize_mode_financement(mode_financement),
+      geom: build_geometry(geojsons, code)
     })
     |> Repo.insert_or_update()
   end
 
+  defp check_communes_list(body) do
+    all_communes =
+      body
+      |> Enum.map(fn epci ->
+        epci["membres"] |> Enum.map(& &1["code"])
+      end)
+      |> List.flatten()
+
+    duplicate_communes = all_communes -- Enum.uniq(all_communes)
+
+    if duplicate_communes != [] do
+      raise "One or multiple communes belong to different EPCIs. List: #{duplicate_communes}"
+    end
+  end
+
+  defp update_communes_epci(%{"code" => code, "membres" => m}) do
+    communes_arr = get_insees(m)
+    communes = Repo.all(from(c in Commune, where: c.insee in ^communes_arr))
+
+    communes
+    |> Enum.each(fn commune ->
+      commune
+      |> Changeset.change(epci_insee: code)
+      |> Repo.update()
+    end)
+
+    :ok
+  end
+
   @spec get_insees([map()]) :: [binary()]
   defp get_insees(members) do
     members
     |> Enum.map(fn m -> m["code"] end)
   end
+
+  defp geojson_by_insee do
+    %{status: 200, body: body} =
+      Req.get!(@epci_geojson_url, connect_options: [timeout: 15_000], receive_timeout: 15_000)
+
+    body
+    # Req doesn’t decode GeoJSON body automatically as it does for JSON
+    |> Jason.decode!()
+    |> Map.fetch!("features")
+    |> Map.new(fn record -> {record["properties"]["code"], record["geometry"]} end)
+  end
+
+  defp build_geometry(geojsons, insee) do
+    {:ok, geom} = Geo.PostGIS.Geometry.cast(Map.fetch!(geojsons, insee))
+    %{geom | srid: 4326}
+  end
+
+  defp ensure_valid_geometries,
+    do: Repo.query!("UPDATE epci SET geom = ST_MakeValid(geom) WHERE NOT ST_IsValid(geom);")
+
+  @spec normalize_type(binary()) :: binary()
+  defp normalize_type("CA"), do: "Communauté d'agglomération"
+  defp normalize_type("CU"), do: "Communauté urbaine"
+  defp normalize_type("CC"), do: "Communauté de communes"
+  defp normalize_type("METRO"), do: "Métropole"
+  defp normalize_type("MET69"), do: "Métropole de Lyon"
+
+  @spec normalize_mode_financement(binary()) :: binary()
+  defp normalize_mode_financement("FPU"), do: "Fiscalité professionnelle unique"
+  defp normalize_mode_financement("FA"), do: "Fiscalité additionnelle"
 end

apps/transport/lib/transport/import_data.ex

@@ -6,7 +6,7 @@ defmodule Transport.ImportData do
   alias Datagouvfr.Client.CommunityResources
   alias Helpers
   alias Opendatasoft.UrlExtractor
-  alias DB.{Dataset, EPCI, LogsImport, Repo, Resource}
+  alias DB.{Commune, Dataset, LogsImport, Repo, Resource}
   alias Transport.Shared.ResourceSchema
   require Logger
   import Ecto.Query
@@ -261,16 +261,10 @@ defmodule Transport.ImportData do
         ]
       }) do
     # For the EPCI we get the list of cities contained by the EPCI
-    EPCI
-    |> Repo.get_by(code: code)
-    |> case do
-      nil ->
-        Logger.warning("impossible to find epci #{code}, no cities associated to the dataset")
-        []
-
-      epci ->
-        epci.communes_insee
-    end
+    Commune
+    |> where([c], c.epci_insee == ^code)
+    |> select([c], c.insee)
+    |> Repo.all()
   end
 
   def read_datagouv_zone(%{"features" => [%{"id" => id} | _]}) do