Merge branch 'master' into delete_old_features_ods

.github/workflows/ops_tests.yml

@@ -0,0 +1,14 @@
+name: CI ops tests
+on: push
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    name: Run ops tests
+    steps:
+      - uses: actions/checkout@v4
+      - uses: erlef/setup-beam@v1
+        with:
+          version-file: .tool-versions
+          version-type: strict
+      - run: elixir ops_tests/ops_tests.exs

.gitignore

@@ -23,9 +23,9 @@ apps/transport/priv/static/css/*
 apps/transport/priv/static/fonts/*
 apps/transport/priv/static/images/*
 apps/transport/priv/static/js/*
-# Environment varabiables files
+
+# Environment variables files
 /.env
-/.envrc
 
 # The config/prod.secret.exs file by default contains sensitive
 # data and you should not commit it into version control.

Dockerfile

@@ -17,6 +17,8 @@ RUN mix deps.compile
 RUN cd apps/transport/client && yarn install && npm run build
 # assets digest must happen after the npm build step
 RUN mix phx.digest
+# Package source code for Sentry https://hexdocs.pm/sentry/upgrade-10-x.html
+RUN mix sentry.package_source_code
 
 EXPOSE 8080
 

README.md

@@ -20,51 +20,64 @@ You can install this 2 different ways:
 
 ## Manual installation <a name="manual_install"></a>
 
-  * Make sure you have **Elixir**, **Node**, **Yarn** and **Docker** installed and up-to-date
-    * **Elixir** is often installed with [asdf](https://asdf-vm.com/) since it makes it easy to handle different **Elixir** versions accross projects. The project needs at least **Elixir** 1.8 and **Erlang** 21.0
-  * Install Elixir dependencies with `mix deps.get`
-  * Install Node.js dependencies with `mix yarn install`
+### 1. Install Elixir, Node and Yarn
+
+Make sure you have **Elixir**, **Node.js** and **Yarn** installed and up-to-date. **Docker** is optionnally needed if you want to run on your machine the [extra transport tools](https://github.com/etalab/transport-tools).
+
+**Elixir** is often installed with [asdf](https://asdf-vm.com/) since it makes it easy to handle different **Elixir** versions accross projects.
 
 If you wish to use `asdf` (recommended), make sure to install the correct plugins:
 
 * `asdf plugin-add erlang` (https://github.com/asdf-vm/asdf-erlang)
 * `asdf plugin-add elixir` (https://github.com/asdf-vm/asdf-elixir)
 * `asdf plugin-add nodejs` (https://github.com/asdf-vm/asdf-nodejs)
 
-Installation can then be done with:
+Installation for Erlang, Elixir and Node.js can then be done with:
 * `asdf install`
 
-### Postgresql
+For Yarn, bring your version or also use asdf:
+* `asdf plugin-add yarn` (https://github.com/twuni/asdf-yarn)
+* `asdf install yarn 1.22.19` (or any other version)
+* `asdf global yarn 1.22.19`
 
-You also need an up to date postgresql with postgis installed. Version 14+ is recommended.
+### 2. Install PostgreSQL
+
+You also need an up-to-date PostgreSQL with Postgis installed. Version 14+ is recommended.
 
 For Mac users, you can use https://postgresapp.com/.
 
-#### Dependencies
+### 3. Build project
+
+  * Install Elixir dependencies with `mix deps.get`
+  * Install Node.js dependencies with `mix yarn install`
+
+## Docker installation
 
-Download depencies using `mix deps.get`.
+Alternatively, you can use Docker, see the [Docker section](#docker-installation-).
 
-Reply "Yes" to the question "Shall I install Hex? (if running non-interactively, use "mix local.hex --force")".
+## Prepare the PostgreSQL database
 
-#### Creating a database
+### Creating a database
+
+Make sure you have a postgres user with postgres password, and that the identification methode is set to md5 in your `pg_hba.conf` file.
 
 Create the database with the command `mix ecto.create`.
 
 Alternatively, you can create it manually. With the permission to create a database (on Debian based system, you need to be logged as `postgres`), type
 `createdb transport_repo`.
 
-#### Applying the migrations
+### Applying the migrations
 
 To have an up to date database schema run `mix ecto.migrate`.
 
-#### Restoring the production database
+### Restoring the production database
 
 The production database does not contains any sensitive data, you can retreive it for dev purpose.
 * You can retrieve the [latest clever-cloud backup](https://console.clever-cloud.com/organisations/orga_f33ebcbc-4403-4e4c-82f5-12305e0ecb1b/addons/addon_beebaa5e-c3a4-4c57-b124-cf9d1473450a) (you need some permissions to access it, if you don't have them, you can ask someone on the team to give you the database)
 * On the clever-cloud website, under transport-site-postgresql, there is a Backups section with download links.
 * restore the downloaded backup on you database: `./restore_db.sh <path_to_the_backup>`
 
-#### Binary CLI dependencies
+## Binary CLI dependencies (optional)
 
 The app uses a number of tools via [transport-tools](https://github.com/etalab/transport-tools).
 
@@ -86,7 +99,7 @@ For Rust binaries, you will have to compile them locally and copy them to the sa
 
 Once this is done, make sure to configure your configuration via `:transport_tools_folder`.
 
-## Usage
+# Usage
 
 Run the server with `mix phx.server` and you can visit [`127.0.0.1:5000`](http://127.0.0.1:5000) on your browser.
 
@@ -134,11 +147,11 @@ config :oauth2, Datagouvfr.Authentication,
   redirect_uri: "http://localhost:5000/login/callback"
 ```
 
-## Development
+# Development
 
-### Testing
+## Testing
 
-#### Running the tests
+### Running the tests
 
 Run the tests with `mix test`
 
@@ -159,7 +172,7 @@ mix cmd --app transport mix test --color test/transport_web/integrations/backoff
 
 The filenames must be relative to the app folder. This [will be improved](https://dockyard.com/blog/2019/06/17/testing-in-umbrella-apps-improves-in-elixir-1-9) when we upgrade to a more modern Elixir version.
 
-#### Measuring test coverage
+### Measuring test coverage
 
 We use [excoveralls](https://github.com/parroty/excoveralls) to measure which parts of the code are covered by testing (or not). This is useful to determine where we can improve the testing quality.
 
@@ -179,22 +192,22 @@ The coverage is written on screen by default, or in the `cover` subfolders for H
 
 Running in `--umbrella` mode will generate coverage report at the top-level `cover` folder, while running without it will generate reports under each umbrella sub-app (e.g. `apps/transport/cover`).
 
-### Linting
+## Linting
 
   * Run the elixir linter with `mix credo --strict`
   * Run the javascript linter with `mix npm "run linter:ecma"`
   * Run the sass linter with `mix npm "run linter:sass"`
 
-### Misc Elixir command
+## Misc Elixir command
 
-#### Translations
+### Translations
 
 To extract all translations from the source, you can run `mix gettext.extract --merge` (and then edit the modified .po files).
 
-#### Check all
+### Check all
 To perform all the checks done on the CI with a single command, you can run `mix check_all`. It will run the different linters, credo, check the translations are up-to-date, and launch the tests.
 
-#### DB migrations
+### DB migrations
 
 To generate a new migration file:
 `cd apps/transport && mix ecto.gen.migration <name of the migration> && cd ../..`
@@ -204,7 +217,7 @@ The generated [ecto](https://hexdocs.pm/ecto/Ecto.html) migration file will be `
 To apply all migrations on you database:
 `mix ecto.migrate`
 
-#### One shot tasks
+### One shot tasks
 
 Some custom one shot tasks are available.
 
@@ -214,18 +227,21 @@ To run a custom task: `mix <custom task>`
 * `Transport.ImportEPCI`: import the french EPCI from data.gouv
 * `Transport.OpenApiSpec`: generate an OpenAPI specification file
 
-### Testing emails
+## Testing emails
 
 To locally test emails in a dev environment, a Swoosh preview inbox available in your browser at `/dev/mailbox`. Your server needs to be run through iex : `iex -S mix phx.server`.
 
-## Docker installation <a name="docker_install"></a>
+## Learn more and debug
 
-### Development
+See the [learning track document](learning_track.md).
+
+# Docker installation <a name="docker_install"></a>
+
+## Development
 
-If you don't plan to work a lot on this project, the docker installation is way easier.
+If you don't plan to work a lot on this project, the Docker installation is way easier.
 
-You need a .env file, and can use .env.example to see which variables need to be set.
-(No need to setup the variable `PG_URL`, it is defined in the docker-compose.yml)
+Some environment variables may be needed to configure the app, see the files in the `config` folder.
 
 Then you only need to run:
   `docker-compose up`
@@ -242,14 +258,14 @@ For the tests you also need to add an environment variable:
 
 `docker-compose run -e web mix test`
 
-### Production
+## Production
 
   The Dockerfile needed to run the continuous integration is in the project:
   https://github.com/etalab/transport-ops
 
   Update it if needed (e.g. updating Elixir’s version) and then update `.circleci/config.yml`.
 
-### Domain names
+# Domain names
 
 The following domain names are currently in use by the deployed Elixir app:
 

apps/datagouvfr/mix.exs

@@ -43,8 +43,8 @@ defmodule Datagouvfr.MixProject do
       {:shared, in_umbrella: true},
       {:vex, "~> 0.9"},
       {:exvcr, "~> 0.13", only: :test},
-      {:mox, "~> 1.0.0", only: :test},
-      {:sentry, "~> 8.1"},
+      {:mox, "~> 1.1", only: :test},
+      {:sentry, "~> 10.1"},
       {:dialyxir, "~> 1.2", only: [:dev, :test], runtime: false},
     ]
   end

apps/gbfs/mix.exs

@@ -42,7 +42,7 @@ defmodule GBFS.MixProject do
       {:sweet_xml, ">= 0.0.0"},
       {:jason, ">= 0.0.0"},
       {:cors_plug, "~> 3.0"},
-      {:sentry, "~> 8.1"},
+      {:sentry, "~> 10.1"},
       # Required for ConditionalJSONEncoder which is referenced by global config,
       # otherwise the tests won't run. Not sure if there's a better way for this,
       # since the GBFS app itself does not currently rely on conditional encoding.

apps/shared/mix.exs

@@ -39,7 +39,7 @@ defmodule Shared.MixProject do
       {:httpoison, ">= 0.0.0"},
       {:req, "~> 0.4.0"},
       {:bypass, "~> 2.1", only: :test},
-      {:mox, "~> 1.0.0", only: :test},
+      {:mox, "~> 1.1", only: :test},
       # Mint is used by our HttpStream shared component, so we add an explicity dependency
       {:mint, "~> 1.2"},
       # Finch is used for built-in streaming
@@ -52,7 +52,7 @@ defmodule Shared.MixProject do
       # otherwise running tests for an individual umbrella sub-app will raise error.
       # A better way to achieve this will be to configure it at runtime, like described
       # in https://github.com/getsentry/sentry-elixir/pull/472.
-      {:sentry, "~> 8.1"},
+      {:sentry, "~> 10.1"},
       # Similarly, Jason is configured as `json_library` by the main app, so it will
       # be required no matter what.
       {:jason, ">= 0.0.0"},

apps/transport/lib/S3/s3.ex

@@ -48,15 +48,4 @@ defmodule Transport.S3 do
     |> ExAws.S3.upload(Transport.S3.bucket_name(feature), upload_path, options)
     |> Transport.Wrapper.ExAWS.impl().request!()
   end
-
-  @spec upload_to_s3!(bucket_feature(), binary(), binary(), acl: atom()) :: any()
-  def upload_to_s3!(feature, body, path, options \\ []) do
-    Logger.debug("Uploading file to #{path}")
-    options = Keyword.validate!(options, acl: :private)
-
-    feature
-    |> Transport.S3.bucket_name()
-    |> ExAws.S3.put_object(path, body, options)
-    |> Transport.Wrapper.ExAWS.impl().request!()
-  end
 end

apps/transport/lib/db/dataset_monthly_metric.ex

@@ -0,0 +1,28 @@
+defmodule DB.DatasetMonthlyMetric do
+  @moduledoc """
+  Monthly metrics related to datasets as given by the data.gouv.fr
+  API.
+  Example: https://metric-api.data.gouv.fr/api/datasets/data/?metric_month__sort=asc&dataset_id__exact=5b3cc551c751df4822526c1c
+  """
+  use Ecto.Schema
+  use TypedEctoSchema
+  import Ecto.Changeset
+
+  typed_schema "dataset_monthly_metrics" do
+    # Foreign key constraint is not enforced on the dataset table
+    # See: https://github.com/etalab/transport-site/pull/3663/files#r1429890393
+    belongs_to(:dataset, DB.Dataset, foreign_key: :dataset_datagouv_id, references: :datagouv_id, type: :string)
+    field(:year_month, :string)
+    field(:metric_name, Ecto.Enum, values: [:views, :downloads])
+    field(:count, :integer)
+    timestamps(type: :utc_datetime_usec)
+  end
+
+  def changeset(struct, attrs \\ %{}) do
+    struct
+    |> cast(attrs, [:dataset_datagouv_id, :year_month, :metric_name, :count])
+    |> validate_required([:dataset_datagouv_id, :year_month, :metric_name, :count])
+    |> validate_format(:year_month, ~r/\A2\d{3}-(0[1-9]|1[012])\z/)
+    |> validate_number(:count, greater_than_or_equal_to: 0)
+  end
+end

apps/transport/lib/jobs/consolidate_bnlc_job.ex

@@ -132,10 +132,9 @@ defmodule Transport.Jobs.ConsolidateBNLCJob do
   end
 
   defp upload_temporary_file do
-    content = File.read!(@bnlc_path)
     now = DateTime.utc_now() |> DateTime.truncate(:microsecond) |> DateTime.to_string() |> String.replace(" ", "_")
     filename = "bnlc-#{now}.csv"
-    Transport.S3.upload_to_s3!(@s3_bucket, content, filename, acl: :public_read)
+    Transport.S3.stream_to_s3!(@s3_bucket, @bnlc_path, filename, acl: :public_read)
     filename
   end
 
@@ -208,7 +207,7 @@ defmodule Transport.Jobs.ConsolidateBNLCJob do
 
     """
     <h2>Erreurs liées aux jeux de données</h2>
-    #{Enum.map_join(dataset_errors, "\n", fn el -> format.(el) end)}
+    #{Enum.map_join(dataset_errors, "<br/>", fn el -> format.(el) end)}
     """
   end
 
@@ -217,7 +216,7 @@ defmodule Transport.Jobs.ConsolidateBNLCJob do
   def format_validation_errors(%{validation_errors: validation_errors}) do
     """
     <h2>Ressources non valides par rapport au schéma #{@schema_name}</h2>
-    #{Enum.map_join(validation_errors, "\n", &link_to_resource/1)}
+    #{Enum.map_join(validation_errors, "<br/>", &link_to_resource/1)}
     """
   end
 
@@ -232,7 +231,7 @@ defmodule Transport.Jobs.ConsolidateBNLCJob do
 
     """
     <h2>Impossible de télécharger les ressources suivantes</h2>
-    #{Enum.map_join(errors, "\n", &link_to_resource/1)}
+    #{Enum.map_join(errors, "<br/>", &link_to_resource/1)}
     """
   end
 
@@ -247,7 +246,7 @@ defmodule Transport.Jobs.ConsolidateBNLCJob do
 
     """
     <h2>Impossible de décoder les fichiers CSV suivants</h2>
-    #{Enum.map_join(errors, "\n", &link_to_resource/1)}
+    #{Enum.map_join(errors, "<br/>", &link_to_resource/1)}
     """
   end
 

apps/transport/lib/jobs/conversions/gtfs_generic_converter.ex

@@ -140,13 +140,12 @@ defmodule Transport.Jobs.GTFSGenericConverter do
               conversion_output_path
             end
 
-          file = File.read!(path)
           %File.Stat{size: filesize} = File.stat!(path)
 
           conversion_file_name =
             resource_filename |> conversion_file_name(format_lower) |> add_zip_extension(zip_conversion?)
 
-          Transport.S3.upload_to_s3!(:history, file, conversion_file_name, acl: :public_read)
+          Transport.S3.stream_to_s3!(:history, path, conversion_file_name, acl: :public_read)
 
           %DataConversion{
             convert_from: :GTFS,