Merge branch 'master' into menu_link_to_producer_space

README.md

@@ -223,9 +223,11 @@ Some custom one shot tasks are available.
 
 To run a custom task: `mix <custom task>`
 
-* `Transport.ImportAom`: import the aom data from the cerema
-* `Transport.ImportEPCI`: import the french EPCI from data.gouv
-* `Transport.OpenApiSpec`: generate an OpenAPI specification file
+* `mix Transport.ImportAOMs`: import the aom data from the cerema
+* `mix Transport.ImportCommunes`: import the french communes from data.gouv
+* `mix Transport.ImportEPCI`: import the french EPCI from data.gouv
+* `mix Transport.ImportDepartements`: import the french Départements from data.gouv
+* `mix Transport.OpenApiSpec`: generate an OpenAPI specification file
 
 ## Testing emails
 

apps/transport/lib/db/commune.ex

@@ -4,7 +4,7 @@ defmodule DB.Commune do
   """
   use Ecto.Schema
   use TypedEctoSchema
-  alias DB.{AOM, Departement, Region}
+  alias DB.{AOM, Departement, EPCI, Region}
   alias Geo.MultiPolygon
 
   typed_schema "commune" do
@@ -20,5 +20,6 @@ defmodule DB.Commune do
     belongs_to(:aom_res, AOM, references: :composition_res_id)
     belongs_to(:region, Region)
     belongs_to(:departement, Departement, foreign_key: :departement_insee, references: :insee, type: :string)
+    belongs_to(:epci, EPCI, foreign_key: :epci_insee, references: :insee, type: :string)
   end
 end

apps/transport/lib/db/epci.ex

@@ -1,19 +1,34 @@
 defmodule DB.EPCI do
   @moduledoc """
-  EPCI schema
+  EPCI schema.
 
-  Link the EPCI to some Communes.
-  The EPCI are loaded once and for all by the task transport/lib/transport/import_epci.ex
+  The EPCI are loaded by the task transport/lib/transport/import_epci.ex.
+  The EPCI imported are only "à fiscalité propre". This excludes Etablissements Publics Territoriaux.
+  This allows to have a 1 to 1 relation between a commune and an EPCI.
   """
   use Ecto.Schema
   use TypedEctoSchema
+  import Ecto.Changeset
 
   typed_schema "epci" do
-    field(:code, :string)
+    field(:insee, :string)
     field(:nom, :string)
+    field(:type, :string)
+    field(:mode_financement, :string)
+    field(:geom, Geo.PostGIS.Geometry) :: Geo.MultiPolygon.t()
+    has_many(:communes, DB.Commune, foreign_key: :epci_insee)
+  end
 
-    # for the moment we don't need a link relational link to the Commune table,
-    # so we only store an array of insee code
-    field(:communes_insee, {:array, :string}, default: [])
+  def changeset(epci, attrs) do
+    epci
+    |> cast(attrs, [:insee, :nom, :geom, :type, :mode_financement])
+    |> validate_required([:insee, :nom, :geom, :type, :mode_financement])
+    |> validate_inclusion(:type, allowed_types())
+    |> validate_inclusion(:mode_financement, allowed_mode_financement())
   end
+
+  defp allowed_types,
+    do: ["Communauté d'agglomération", "Communauté urbaine", "Communauté de communes", "Métropole", "Métropole de Lyon"]
+
+  defp allowed_mode_financement, do: ["Fiscalité professionnelle unique", "Fiscalité additionnelle"]
 end

apps/transport/lib/mix/tasks/transport/import_aoms.ex

@@ -1,4 +1,4 @@
-defmodule Mix.Tasks.Transport.ImportAoms do
+defmodule Mix.Tasks.Transport.ImportAOMs do
   @moduledoc """
   Import the AOM files and updates the database.
 
@@ -13,7 +13,7 @@ defmodule Mix.Tasks.Transport.ImportAoms do
 
   This is a one shot import task, run when the AOM have changed, at least every year.
 
-  The import can be launched through mix transport.import_aoms
+  The import can be launched through mix Transport.ImportAOMs
   """
 
   @shortdoc "Refreshes the database table `aom` with the latest data"

apps/transport/lib/mix/tasks/transport/import_communes.ex

@@ -1,6 +1,6 @@
 defmodule Mix.Tasks.Transport.ImportCommunes do
   @moduledoc """
-  Import or updates commune data (list, geometry) from official sources. Run with `mix transport.import_communes`.
+  Import or updates commune data (list, geometry) from official sources. Run with `mix Transport.ImportCommunes`.
 
   """
   @shortdoc "Refreshes the database table `commune` with the latest data"

apps/transport/lib/mix/tasks/transport/import_epci.ex

@@ -1,47 +1,45 @@
 defmodule Mix.Tasks.Transport.ImportEPCI do
   @moduledoc """
   Import the EPCI file to get the relation between the cities and the EPCI
+  Run: mix transport.importEPCI
   """
 
   use Mix.Task
   import Ecto.Query
   alias Ecto.Changeset
-  alias DB.{EPCI, Repo}
+  alias DB.{Commune, EPCI, Repo}
   require Logger
 
   @epci_file "https://unpkg.com/@etalab/decoupage-administratif@3.1.1/data/epci.json"
+  @epci_geojson_url "http://etalab-datasets.geo.data.gouv.fr/contours-administratifs/2023/geojson/epci-100m.geojson"
 
-  def run(params) do
-    Logger.info("importing epci")
+  def run(_params) do
+    Logger.info("Importing EPCIs")
 
-    if params[:no_start] do
-      HTTPoison.start()
-    else
-      Mix.Task.run("app.start", [])
-    end
+    Mix.Task.run("app.start")
 
-    with {:ok, %HTTPoison.Response{status_code: 200, body: body}} <- HTTPoison.get(@epci_file),
-         {:ok, json} <- Jason.decode(body) do
-      json |> Enum.each(&insert_epci/1)
-
-      # Remove EPCIs that have been removed
-      epci_codes = Enum.map(json, & &1["code"])
-      EPCI |> where([e], e.code not in ^epci_codes) |> Repo.delete_all()
-
-      nb_epci = Repo.aggregate(EPCI, :count, :id)
-      Logger.info("#{nb_epci} are now in database")
-      :ok
-    else
-      e ->
-        Logger.warning("impossible to fetch epci file, error #{inspect(e)}")
-        :error
-    end
+    %{status: 200, body: json} = Req.get!(@epci_file, connect_options: [timeout: 15_000], receive_timeout: 15_000)
+    check_communes_list(json)
+    geojsons = geojson_by_insee()
+
+    json |> Enum.each(&insert_epci(&1, geojsons))
+    json |> Enum.each(&update_communes_epci/1)
+
+    # Remove EPCIs that have been removed
+    epci_codes = json |> Enum.map(& &1["code"])
+    EPCI |> where([e], e.insee not in ^epci_codes) |> Repo.delete_all()
+
+    nb_epci = Repo.aggregate(EPCI, :count, :id)
+    Logger.info("#{nb_epci} are now in database")
+    Logger.info("Ensure valid geometries and rectify if needed.")
+    ensure_valid_geometries()
+    :ok
   end
 
   @spec get_or_create_epci(binary()) :: EPCI.t()
   defp get_or_create_epci(code) do
     EPCI
-    |> Repo.get_by(code: code)
+    |> Repo.get_by(insee: code)
     |> case do
       nil ->
         %EPCI{}
@@ -51,21 +49,82 @@ defmodule Mix.Tasks.Transport.ImportEPCI do
     end
   end
 
-  @spec insert_epci(map()) :: any()
-  defp insert_epci(%{"code" => code, "nom" => nom, "membres" => m}) do
+  @spec insert_epci(map(), map()) :: any()
+  defp insert_epci(%{"code" => code, "nom" => nom, "type" => type, "modeFinancement" => mode_financement}, geojsons) do
     code
     |> get_or_create_epci()
-    |> Changeset.change(%{
-      code: code,
+    |> EPCI.changeset(%{
+      insee: code,
       nom: nom,
-      communes_insee: get_insees(m)
+      type: normalize_type(type),
+      mode_financement: normalize_mode_financement(mode_financement),
+      geom: build_geometry(geojsons, code)
     })
     |> Repo.insert_or_update()
   end
 
+  defp check_communes_list(body) do
+    all_communes =
+      body
+      |> Enum.map(fn epci ->
+        epci["membres"] |> Enum.map(& &1["code"])
+      end)
+      |> List.flatten()
+
+    duplicate_communes = all_communes -- Enum.uniq(all_communes)
+
+    if duplicate_communes != [] do
+      raise "One or multiple communes belong to different EPCIs. List: #{duplicate_communes}"
+    end
+  end
+
+  defp update_communes_epci(%{"code" => code, "membres" => m}) do
+    communes_arr = get_insees(m)
+    communes = Repo.all(from(c in Commune, where: c.insee in ^communes_arr))
+
+    communes
+    |> Enum.each(fn commune ->
+      commune
+      |> Changeset.change(epci_insee: code)
+      |> Repo.update()
+    end)
+
+    :ok
+  end
+
   @spec get_insees([map()]) :: [binary()]
   defp get_insees(members) do
     members
     |> Enum.map(fn m -> m["code"] end)
   end
+
+  defp geojson_by_insee do
+    %{status: 200, body: body} =
+      Req.get!(@epci_geojson_url, connect_options: [timeout: 15_000], receive_timeout: 15_000)
+
+    body
+    # Req doesn’t decode GeoJSON body automatically as it does for JSON
+    |> Jason.decode!()
+    |> Map.fetch!("features")
+    |> Map.new(fn record -> {record["properties"]["code"], record["geometry"]} end)
+  end
+
+  defp build_geometry(geojsons, insee) do
+    {:ok, geom} = Geo.PostGIS.Geometry.cast(Map.fetch!(geojsons, insee))
+    %{geom | srid: 4326}
+  end
+
+  defp ensure_valid_geometries,
+    do: Repo.query!("UPDATE epci SET geom = ST_MakeValid(geom) WHERE NOT ST_IsValid(geom);")
+
+  @spec normalize_type(binary()) :: binary()
+  defp normalize_type("CA"), do: "Communauté d'agglomération"
+  defp normalize_type("CU"), do: "Communauté urbaine"
+  defp normalize_type("CC"), do: "Communauté de communes"
+  defp normalize_type("METRO"), do: "Métropole"
+  defp normalize_type("MET69"), do: "Métropole de Lyon"
+
+  @spec normalize_mode_financement(binary()) :: binary()
+  defp normalize_mode_financement("FPU"), do: "Fiscalité professionnelle unique"
+  defp normalize_mode_financement("FA"), do: "Fiscalité additionnelle"
 end

apps/transport/lib/transport/import_data.ex

@@ -6,7 +6,7 @@ defmodule Transport.ImportData do
   alias Datagouvfr.Client.CommunityResources
   alias Helpers
   alias Opendatasoft.UrlExtractor
-  alias DB.{Dataset, EPCI, LogsImport, Repo, Resource}
+  alias DB.{Commune, Dataset, LogsImport, Repo, Resource}
   alias Transport.Shared.ResourceSchema
   require Logger
   import Ecto.Query
@@ -261,16 +261,10 @@ defmodule Transport.ImportData do
         ]
       }) do
     # For the EPCI we get the list of cities contained by the EPCI
-    EPCI
-    |> Repo.get_by(code: code)
-    |> case do
-      nil ->
-        Logger.warning("impossible to find epci #{code}, no cities associated to the dataset")
-        []
-
-      epci ->
-        epci.communes_insee
-    end
+    Commune
+    |> where([c], c.epci_insee == ^code)
+    |> select([c], c.insee)
+    |> Repo.all()
   end
 
   def read_datagouv_zone(%{"features" => [%{"id" => id} | _]}) do

apps/transport/priv/repo/migrations/20231222145809_improve_epci.exs

@@ -0,0 +1,39 @@
+defmodule DB.Repo.Migrations.ImproveEPCI do
+  use Ecto.Migration
+
+  def change do
+    rename(table(:epci), :code, to: :insee)
+    create(unique_index(:epci, [:insee]))
+
+    alter table(:commune) do
+      add(:epci_insee, references(:epci, column: :insee, type: :string, on_delete: :nilify_all), null: true)
+    end
+
+    create(index(:commune, [:epci_insee]))
+
+    # Migrate data from epci communes_insee array column to epci_insee column in commune table
+    execute(
+      """
+        UPDATE commune
+        SET epci_insee = epci.insee
+        FROM epci
+        WHERE commune.insee = ANY(epci.communes_insee)
+      """,
+      """
+        UPDATE epci
+        SET communes_insee = ARRAY(
+          SELECT insee
+          FROM commune
+          WHERE commune.epci_insee = epci.insee
+        )
+      """
+    )
+
+    alter table(:epci) do
+      add(:type, :string)
+      add(:mode_financement, :string)
+      add(:geom, :geometry)
+      remove(:communes_insee, {:array, :string}, default: [])
+    end
+  end
+end

apps/transport/test/transport/import_data_test.exs

@@ -358,16 +358,16 @@ defmodule Transport.ImportDataTest do
     end
 
     test "for an EPCI" do
-      commune = insert(:commune)
-      epci = insert(:epci, code: "242320109", nom: "Le Pays Dunois", communes_insee: [commune.insee])
+      epci = insert(:epci, insee: "242320109", nom: "Le Pays Dunois")
+      commune = insert(:commune, epci_insee: "242320109")
       # Example: https://www.data.gouv.fr/api/1/spatial/zones/fr:epci:242320109/
       assert [commune.insee] ==
                ImportData.read_datagouv_zone(%{
                  "features" => [
                    %{
-                     "id" => "fr:epci:#{epci.code}",
+                     "id" => "fr:epci:#{epci.insee}",
                      "properties" => %{
-                       "code" => epci.code,
+                       "code" => epci.insee,
                        "level" => "fr:epci",
                        "name" => epci.nom,
                        "slug" => "Le-Pays-Dunois",