Merge branch 'master' into remove-ods-comms

.github/workflows/ops_tests.yml

@@ -0,0 +1,14 @@
+name: CI ops tests
+on: push
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    name: Run ops tests
+    steps:
+      - uses: actions/checkout@v4
+      - uses: erlef/setup-beam@v1
+        with:
+          version-file: .tool-versions
+          version-type: strict
+      - run: elixir ops_tests/ops_tests.exs

.gitignore

@@ -23,9 +23,9 @@ apps/transport/priv/static/css/*
 apps/transport/priv/static/fonts/*
 apps/transport/priv/static/images/*
 apps/transport/priv/static/js/*
-# Environment varabiables files
+
+# Environment variables files
 /.env
-/.envrc
 
 # The config/prod.secret.exs file by default contains sensitive
 # data and you should not commit it into version control.

Dockerfile

@@ -17,6 +17,8 @@ RUN mix deps.compile
 RUN cd apps/transport/client && yarn install && npm run build
 # assets digest must happen after the npm build step
 RUN mix phx.digest
+# Package source code for Sentry https://hexdocs.pm/sentry/upgrade-10-x.html
+RUN mix sentry.package_source_code
 
 EXPOSE 8080
 

README.md

@@ -239,10 +239,9 @@ See the [learning track document](learning_track.md).
 
 ## Development
 
-If you don't plan to work a lot on this project, the docker installation is way easier.
+If you don't plan to work a lot on this project, the Docker installation is way easier.
 
-You need a .env file, and can use .env.example to see which variables need to be set.
-(No need to setup the variable `PG_URL`, it is defined in the docker-compose.yml)
+Some environment variables may be needed to configure the app, see the files in the `config` folder.
 
 Then you only need to run:
   `docker-compose up`

apps/datagouvfr/mix.exs

@@ -44,7 +44,7 @@ defmodule Datagouvfr.MixProject do
       {:vex, "~> 0.9"},
       {:exvcr, "~> 0.13", only: :test},
       {:mox, "~> 1.1", only: :test},
-      {:sentry, "~> 8.1"},
+      {:sentry, "~> 10.1"},
       {:dialyxir, "~> 1.2", only: [:dev, :test], runtime: false},
     ]
   end

apps/gbfs/mix.exs

@@ -42,7 +42,7 @@ defmodule GBFS.MixProject do
       {:sweet_xml, ">= 0.0.0"},
       {:jason, ">= 0.0.0"},
       {:cors_plug, "~> 3.0"},
-      {:sentry, "~> 8.1"},
+      {:sentry, "~> 10.1"},
       # Required for ConditionalJSONEncoder which is referenced by global config,
       # otherwise the tests won't run. Not sure if there's a better way for this,
       # since the GBFS app itself does not currently rely on conditional encoding.

apps/shared/mix.exs

@@ -52,7 +52,7 @@ defmodule Shared.MixProject do
       # otherwise running tests for an individual umbrella sub-app will raise error.
       # A better way to achieve this will be to configure it at runtime, like described
       # in https://github.com/getsentry/sentry-elixir/pull/472.
-      {:sentry, "~> 8.1"},
+      {:sentry, "~> 10.1"},
       # Similarly, Jason is configured as `json_library` by the main app, so it will
       # be required no matter what.
       {:jason, ">= 0.0.0"},

apps/transport/lib/jobs/import_dataset_monthly_metrics_job.ex

@@ -69,16 +69,18 @@ defmodule Transport.Jobs.ImportDatasetMonthlyMetricsJob do
          dataset_datagouv_id
        ) do
     Enum.each([{:views, monthly_visit}, {:downloads, monthly_download_resource}], fn {metric_name, count} ->
+      count = count || 0
+
       %DB.DatasetMonthlyMetric{}
       |> DB.DatasetMonthlyMetric.changeset(%{
         dataset_datagouv_id: dataset_datagouv_id,
         year_month: metric_month,
         metric_name: metric_name,
-        count: count || 0
+        count: count
       })
       |> DB.Repo.insert!(
         conflict_target: [:dataset_datagouv_id, :year_month, :metric_name],
-        on_conflict: {:replace, [:count]}
+        on_conflict: [set: [count: count, updated_at: DateTime.utc_now()]]
       )
     end)
   end

apps/transport/lib/jobs/resource_history_job.ex

@@ -288,7 +288,7 @@ defmodule Transport.Jobs.ResourceHistoryJob do
         {:error, "Got a non 200 status: #{status}"}
 
       {:error, %HTTPoison.Error{reason: reason}} ->
-        {:error, "Got an error: #{reason}"}
+        {:error, "Got an error: #{inspect(reason)}"}
     end
   end
 

apps/transport/lib/transport/gtfs_rt.ex

@@ -26,7 +26,7 @@ defmodule Transport.GTFSRT do
         {:error, "Got a non 200 HTTP status code: #{status_code}"}
 
       {:error, %HTTPoison.Error{reason: reason}} ->
-        {:error, "Got an HTTP error: #{reason}"}
+        {:error, "Got an HTTP error: #{inspect(reason)}"}
     end
   end
 

apps/transport/lib/transport/vault.ex

@@ -5,10 +5,6 @@ defmodule Transport.Vault do
   """
   use Cloak.Vault, otp_app: :transport
 
-  # This value should be base64 encrypted
-  # See https://github.com/danielberkompas/cloak#configuration
-  @prod_config_env_name "CLOAK_KEY"
-
   @impl GenServer
   def init(config) do
     # See config recommendation
@@ -22,8 +18,8 @@ defmodule Transport.Vault do
   defp key do
     case Application.fetch_env!(:transport, :app_env) do
       env when env in [:production, :staging] ->
-        @prod_config_env_name
-        |> System.fetch_env!()
+        :transport
+        |> Application.fetch_env!(:cloak_key)
         |> Base.decode64!()
 
       _ ->

apps/transport/lib/transport_web/controllers/page_controller.ex

@@ -95,26 +95,29 @@ defmodule TransportWeb.PageController do
     |> render("infos_producteurs.html")
   end
 
-  def robots_txt(conn, _params) do
-    # See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
-    content =
-      if Application.fetch_env!(:transport, :app_env) == :staging do
-        """
-        User-agent: *
-        Disallow: /
-        """
-      else
-        """
-        User-agent: *
-        Allow: /
-        Disallow: /backoffice/
-        Disallow: /validation/*
-        Disallow: /login/*
-        Disallow: /resources/conversions/*
-        """
-      end
+  def robots_txt(%Plug.Conn{} = conn, _params) do
+    # See http://www.robotstxt.org/robotstxt.html
+    # for documentation on how to use the robots.txt file
+    app_env = Application.fetch_env!(:transport, :app_env)
+    text(conn, robots_txt_content(app_env))
+  end
 
-    conn |> text(content)
+  def robots_txt_content(:staging = _app_env) do
+    """
+    User-agent: *
+    Disallow: /
+    """
+  end
+
+  def robots_txt_content(_app_env) do
+    """
+    User-agent: *
+    Allow: /
+    Disallow: /backoffice/
+    Disallow: /validation/*
+    Disallow: /login/*
+    Disallow: /resources/conversions/*
+    """
   end
 
   def security_txt(conn, _params) do

apps/transport/lib/transport_web/live/siri_querier_live.ex

@@ -106,7 +106,7 @@ defmodule TransportWeb.Live.SIRIQuerierLive do
           })
 
         {:error, %HTTPoison.Error{reason: reason}} ->
-          socket |> assign(siri_response_error: reason, siri_response_status_code: nil)
+          socket |> assign(siri_response_error: inspect(reason), siri_response_status_code: nil)
       end
 
     {:noreply, socket}

apps/transport/lib/transport_web/views/dataset_view.ex

@@ -15,13 +15,14 @@ defmodule TransportWeb.DatasetView do
   @gtfs_rt_validator_name Transport.Validators.GTFSRT.validator_name()
 
   @doc """
-  Count the number of resources (official + community), excluding resources with a `documentation` type.
+  Count the number of resources, excluding:
+  - community resources
+  - resources with a `documentation` type.
   """
   @spec count_resources(Dataset.t()) :: non_neg_integer
   def count_resources(dataset) do
-    nb_resources = Enum.count(official_available_resources(dataset))
-    nb_community_resources = Enum.count(community_resources(dataset))
-    nb_resources + nb_community_resources - count_documentation_resources(dataset)
+    nb_official_resources = dataset |> official_available_resources() |> Enum.count()
+    nb_official_resources - count_documentation_resources(dataset)
   end
 
   @spec count_documentation_resources(Dataset.t()) :: non_neg_integer

apps/transport/mix.exs

@@ -72,7 +72,7 @@ defmodule Transport.Mixfile do
       {:recon, "~> 2.4"},
       {:quantum, "~> 3.4"},
       {:timex, "~> 3.7"},
-      {:sentry, "~> 8.1"},
+      {:sentry, "~> 10.1"},
       {:scrivener, "~> 2.5"},
       # Compilation issue for this dependency, see https://github.com/etalab/transport-site/issues/3499
       {:scrivener_html, git: "https://github.com/etalab/scrivener_html.git", ref: "f0245703abf7d0ce2b48a0f7e96997def7649e5f"},

apps/transport/priv/repo/migrations/20240102144643_dataset_add_not_null.exs

@@ -0,0 +1,31 @@
+defmodule DB.Repo.Migrations.DatasetAddNotNull do
+  use Ecto.Migration
+
+  @attributes [
+    :datagouv_id,
+    :custom_title,
+    :licence,
+    :logo,
+    :full_logo,
+    :slug,
+    :tags,
+    :datagouv_title,
+    :type,
+    :frequency,
+    :has_realtime,
+    :is_active,
+    :nb_reuses
+  ]
+
+  def up do
+    Enum.each(@attributes, fn attribute ->
+      execute("ALTER TABLE dataset ALTER COLUMN #{attribute} SET NOT NULL")
+    end)
+  end
+
+  def down do
+    Enum.each(@attributes, fn attribute ->
+      execute("ALTER TABLE dataset ALTER COLUMN #{attribute} DROP NOT NULL")
+    end)
+  end
+end

apps/transport/test/support/factory.ex

@@ -59,7 +59,13 @@ defmodule DB.Factory do
       # NOTE: need to figure out how to pass aom/region together with changeset checks here
       aom: build(:aom),
       tags: [],
-      type: "public-transit"
+      type: "public-transit",
+      logo: "https://example.com/#{Ecto.UUID.generate()}_small.png",
+      full_logo: "https://example.com/#{Ecto.UUID.generate()}.png",
+      frequency: "daily",
+      has_realtime: false,
+      is_active: true,
+      nb_reuses: Enum.random(0..10)
     }
   end
 
@@ -211,15 +217,17 @@ defmodule DB.Factory do
       region_id: Keyword.get(opts, :region_id),
       has_realtime: Keyword.get(opts, :has_realtime),
       type: Keyword.get(opts, :type),
-      aom: Keyword.get(opts, :aom),
+      aom: aom = Keyword.get(opts, :aom),
       custom_title: Keyword.get(opts, :custom_title)
     ]
 
     dataset_opts =
-      case Keyword.get(opts, :aom) do
+      aom
+      |> case do
         nil -> dataset_opts
         aom -> dataset_opts |> Keyword.merge(aom: aom)
       end
+      |> Enum.reject(fn {_, v} -> is_nil(v) end)
 
     dataset = Keyword.get(opts, :dataset, insert(:dataset, dataset_opts))
 
@@ -319,4 +327,28 @@ defmodule DB.Factory do
     |> Map.merge(args)
     |> DB.Contact.insert!()
   end
+
+  def datagouv_dataset_response(%{} = attributes \\ %{}) do
+    Map.merge(
+      %{
+        "id" => Ecto.UUID.generate(),
+        "title" => "dataset",
+        "created_at" => DateTime.utc_now() |> to_string(),
+        "last_update" => DateTime.utc_now() |> to_string(),
+        "slug" => "dataset-slug",
+        "license" => "lov2",
+        "frequency" => "daily",
+        "tags" => [],
+        "organization" => %{
+          "id" => Ecto.UUID.generate(),
+          "name" => "Org " <> Ecto.UUID.generate(),
+          "badges" => [],
+          "logo" => "https://example.com/img.jpg",
+          "logo_thumbnail" => "https://example.com/img.small.jpg",
+          "slug" => Ecto.UUID.generate()
+        }
+      },
+      attributes
+    )
+  end
 end

apps/transport/test/transport/community_resource_cleaner_test.exs

@@ -13,7 +13,11 @@ defmodule Transport.CommunityResourcesCleanerTest do
     :ok = Ecto.Adapters.SQL.Sandbox.checkout(DB.Repo)
   end
 
-  defp insert_dataset_associated_with_ressources(resources, datagouv_id \\ nil) do
+  defp insert_dataset_associated_with_ressources(resources) do
+    insert_dataset_associated_with_ressources(resources, Ecto.UUID.generate())
+  end
+
+  defp insert_dataset_associated_with_ressources(resources, datagouv_id) do
     :dataset
     |> insert(%{datagouv_id: datagouv_id})
     |> Repo.preload(:resources)

apps/transport/test/transport/gtfs_rt_test.exs

@@ -39,9 +39,15 @@ defmodule Transport.GTFSRTTest do
     end
 
     test "HTTPoison error" do
-      setup_http_response(@url, {:error, %HTTPoison.Error{reason: "SSL problemz"}})
+      reason =
+        {:tls_alert,
+         {:certificate_expired,
+          ~c"TLS client: In state certify at ssl_handshake.erl:2065 generated CLIENT ALERT: Fatal - Certificate Expired\n"}}
 
-      assert {:error, "Got an HTTP error: SSL problemz"} == GTFSRT.decode_remote_feed(@url)
+      setup_http_response(@url, {:error, %HTTPoison.Error{reason: reason}})
+
+      {:error, reason} = GTFSRT.decode_remote_feed(@url)
+      assert reason =~ ~r"^Got an HTTP error:"
     end
   end