Pt pax volumes into pt dashboard (#3552)

* cleanup, remove features without tests

* add pt passenger volumes to PublicTransitDashboard by running PtStop2StopAnalysis

* clean up

* add support for sampleScaleFactor, fix bug in link list introduced with last commit

* small improvements, add some todos

* simplified tests, use unfiltered avro network for pt

* clean input string for mode filter

---------

Co-authored-by: rakow <rakow@vsp.tu-berlin.de>
Co-authored-by: rakow <rakow@users.noreply.github.com>

contribs/application/pom.xml

@@ -87,7 +87,7 @@
 		<dependency>
 			<groupId>com.github.matsim-org</groupId>
 			<artifactId>gtfs2matsim</artifactId>
-			<version>0bd5850fd6</version>
+			<version>47b0802a29</version>
 			<exclusions>
 				<!-- Exclude unneeded dependencies and these with known CVE -->
 				<exclusion>

contribs/application/src/main/java/org/matsim/application/analysis/pt/PublicTransitAnalysis.java

@@ -0,0 +1,105 @@
+/* *********************************************************************** *
+ * project: org.matsim.*
+ *                                                                         *
+ * *********************************************************************** *
+ *                                                                         *
+ * copyright       : (C) 2024 by the members listed in the COPYING,        *
+ *                   LICENSE and WARRANTY file.                            *
+ * email           : info at matsim dot org                                *
+ *                                                                         *
+ * *********************************************************************** *
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *   See also COPYING, LICENSE and WARRANTY file                           *
+ *                                                                         *
+ * *********************************************************************** */
+
+package org.matsim.application.analysis.pt;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.matsim.analysis.pt.stop2stop.PtStop2StopAnalysis;
+import org.matsim.api.core.v01.Scenario;
+import org.matsim.application.ApplicationUtils;
+import org.matsim.application.CommandSpec;
+import org.matsim.application.MATSimAppCommand;
+import org.matsim.application.analysis.emissions.AirPollutionAnalysis;
+import org.matsim.application.options.InputOptions;
+import org.matsim.application.options.OutputOptions;
+import org.matsim.application.options.SampleOptions;
+import org.matsim.core.api.experimental.events.EventsManager;
+import org.matsim.core.config.Config;
+import org.matsim.core.config.ConfigUtils;
+import org.matsim.core.events.EventsUtils;
+import org.matsim.core.events.MatsimEventsReader;
+import org.matsim.core.scenario.ScenarioUtils;
+import picocli.CommandLine;
+
+@CommandLine.Command(
+	name = "transit", description = "General public transit analysis.",
+	mixinStandardHelpOptions = true, showDefaultValues = true
+)
+@CommandSpec(requireRunDirectory = true,
+	produces = {
+		"pt_pax_volumes.csv.gz",
+	}
+)
+public class PublicTransitAnalysis implements MATSimAppCommand {
+
+	private static final Logger log = LogManager.getLogger(PublicTransitAnalysis.class);
+
+	@CommandLine.Mixin
+	private final InputOptions input = InputOptions.ofCommand(PublicTransitAnalysis.class);
+	@CommandLine.Mixin
+	private final OutputOptions output = OutputOptions.ofCommand(PublicTransitAnalysis.class);
+	@CommandLine.Mixin
+	private SampleOptions sample;
+
+	public static void main(String[] args) {
+		new PublicTransitAnalysis().execute(args);
+	}
+
+	@Override
+	public Integer call() throws Exception {
+
+		Config config = prepareConfig();
+		Scenario scenario = ScenarioUtils.loadScenario(config);
+		EventsManager eventsManager = EventsUtils.createEventsManager();
+
+		String eventsFile = ApplicationUtils.matchInput("events", input.getRunDirectory()).toString();
+
+		PtStop2StopAnalysis ptStop2StopEventHandler = new PtStop2StopAnalysis(scenario.getTransitVehicles(), sample.getUpscaleFactor());
+		eventsManager.addHandler(ptStop2StopEventHandler);
+		eventsManager.initProcessing();
+		MatsimEventsReader matsimEventsReader = new MatsimEventsReader(eventsManager);
+		matsimEventsReader.readFile(eventsFile);
+
+		log.info("Done reading the events file.");
+		log.info("Finish processing...");
+		eventsManager.finishProcessing();
+
+		ptStop2StopEventHandler.writeStop2StopEntriesByDepartureCsv(output.getPath("pt_pax_volumes.csv.gz"),
+			",", ";");
+
+		return 0;
+	}
+
+	private Config prepareConfig() {
+		Config config = ConfigUtils.loadConfig(ApplicationUtils.matchInput("config.xml", input.getRunDirectory()).toAbsolutePath().toString());
+
+		config.vehicles().setVehiclesFile(ApplicationUtils.matchInput("vehicles", input.getRunDirectory()).toAbsolutePath().toString());
+		config.network().setInputFile(ApplicationUtils.matchInput("network", input.getRunDirectory()).toAbsolutePath().toString());
+		config.transit().setTransitScheduleFile(ApplicationUtils.matchInput("transitSchedule", input.getRunDirectory()).toAbsolutePath().toString());
+		config.transit().setVehiclesFile(ApplicationUtils.matchInput("transitVehicles", input.getRunDirectory()).toAbsolutePath().toString());
+		config.plans().setInputFile(null);
+		config.facilities().setInputFile(null);
+		config.eventsManager().setNumberOfThreads(null);
+		config.eventsManager().setEstimatedNumberOfEvents(null);
+		config.global().setNumberOfThreads(1);
+
+		return config;
+	}
+}

contribs/application/src/main/java/org/matsim/application/prepare/network/CreateAvroNetwork.java

@@ -108,8 +108,11 @@ public Integer call() throws Exception {
 			filter = filter.and(link -> p.matcher(link.getId().toString()).matches());
 		}
 
+		// Strings that could have been added in the list, due to command line parsing
+		modes.removeIf(m -> m.isBlank() || m.equals("none") || m.equals("\"") || m.equals("\"\""));
+
 		// At least one of the specified modes needs to be contained
-		if (!modes.isEmpty() && !modes.equals(Set.of("none"))) {
+		if (!modes.isEmpty()) {
 			filter = filter.and(link -> modes.stream().anyMatch(m -> link.getAllowedModes().contains(m)));
 		}
 

contribs/simwrapper/src/main/java/org/matsim/simwrapper/dashboard/PublicTransitDashboard.java

@@ -1,5 +1,7 @@
 package org.matsim.simwrapper.dashboard;
 
+import org.matsim.application.analysis.pt.PublicTransitAnalysis;
+import org.matsim.application.prepare.network.CreateAvroNetwork;
 import org.matsim.simwrapper.Dashboard;
 import org.matsim.simwrapper.Header;
 import org.matsim.simwrapper.Layout;
@@ -34,8 +36,13 @@ public void configure(Header header, Layout layout) {
 			viz.title = "Transit Viewer";
 			viz.height = 12d;
 			viz.description = "Visualize the transit schedule.";
-			viz.network = "(*.)?output_network.xml.gz";
+
+			// Include a network that has not been filtered
+			viz.network = data.withContext("all").compute(CreateAvroNetwork.class, "network.avro",
+				"--mode-filter", "", "--shp", "none");
+
 			viz.transitSchedule = data.output("(*.)?output_transitSchedule.xml.gz");
+			viz.ptStop2stopFile = data.compute(PublicTransitAnalysis.class, "pt_pax_volumes.csv.gz");
 
 			if (!customRouteTypes.isEmpty())
 				viz.customRouteTypes = customRouteTypes;

contribs/simwrapper/src/main/java/org/matsim/simwrapper/viz/TransitViewer.java

@@ -15,19 +15,17 @@ public class TransitViewer extends Viz {
 	@JsonProperty(required = true)
 	public String transitSchedule;
 
+	@JsonProperty
+	public String ptStop2stopFile;
+
 	public List<CustomRouteType> customRouteTypes;
 
 	public TransitViewer() {
 		super("transit");
 	}
 
-	public CustomRouteType addCustomRouteType(String label, String color, boolean hide) {
-		CustomRouteType crt = new CustomRouteType();
-		crt.label = label;
-		crt.color = color;
-		crt.hide = hide;
-		customRouteTypes.add(crt);
-		return crt;
+	public static CustomRouteType customRouteType(String label, String color) {
+		return new CustomRouteType(label, color);
 	}
 
 	public static class CustomRouteType {
@@ -36,6 +34,11 @@ public static class CustomRouteType {
 		public Boolean hide;
 		Match match;
 
+		private CustomRouteType(String label, String color) {
+			this.label = label;
+			this.color = color;
+		}
+
 		public CustomRouteType addMatchTransportMode(String... transportMode) {
 			if (match == null)
 				match = new Match();

contribs/simwrapper/src/test/java/org/matsim/simwrapper/dashboard/DashboardTests.java

@@ -29,7 +29,7 @@ public class DashboardTests {
 	private void run(Dashboard... dashboards) {
 
 		Config config = TestScenario.loadConfig(utils);
-		config.controller().setLastIteration(2);
+		config.controller().setLastIteration(1);
 
 		SimWrapperConfigGroup group = ConfigUtils.addOrGetModule(config, SimWrapperConfigGroup.class);
 		group.sampleSize = 0.001;
@@ -43,41 +43,29 @@ private void run(Dashboard... dashboards) {
 		controler.run();
 	}
 
+
 	@Test
 	void defaults() {
 
-		Path out = Path.of(utils.getOutputDirectory(), "analysis", "population");
+		Path out = Path.of(utils.getOutputDirectory(), "analysis");
 
 		run();
 
 		// Ensure default dashboards have been added
 		Assertions.assertThat(out)
-			.isDirectoryContaining("glob:**stuck_agents.csv");
-	}
-
-	@Test
-	void stuckAgents() {
-
-		Path out = Path.of(utils.getOutputDirectory(), "analysis", "population");
-
-		run(new StuckAgentDashboard());
-
-		Assertions.assertThat(out)
-			.isDirectoryContaining("glob:**stuck_agents.csv");
-
-	}
-
-	@Test
-	void trip() {
-
-		Path out = Path.of(utils.getOutputDirectory(), "analysis", "population");
-
-		run(new TripDashboard());
-		Assertions.assertThat(out)
-			.isDirectoryContaining("glob:**trip_stats.csv")
-			.isDirectoryContaining("glob:**mode_share.csv")
-			.isDirectoryContaining("glob:**mode_share_per_purpose.csv")
-			.isDirectoryContaining("glob:**mode_shift.csv");
+			// Stuck agents
+			.isDirectoryRecursivelyContaining("glob:**stuck_agents.csv")
+			// Trip stats
+			.isDirectoryRecursivelyContaining("glob:**trip_stats.csv")
+			.isDirectoryRecursivelyContaining("glob:**mode_share.csv")
+			.isDirectoryRecursivelyContaining("glob:**mode_share_per_purpose.csv")
+			.isDirectoryRecursivelyContaining("glob:**mode_shift.csv")
+			// Traffic stats
+			.isDirectoryRecursivelyContaining("glob:**traffic_stats_by_link_daily.csv")
+			.isDirectoryRecursivelyContaining("glob:**traffic_stats_by_road_type_and_hour.csv")
+			.isDirectoryRecursivelyContaining("glob:**traffic_stats_by_road_type_daily.csv")
+			// PT
+			.isDirectoryRecursivelyContaining("glob:**pt_pax_volumes.csv.gz");
 	}
 
 	@Test
@@ -129,21 +117,6 @@ void populationAttribute() {
 			.isDirectoryContaining("glob:**total_agents.csv");
 
 
-	}
-
-	@Test
-	void traffic() {
-
-		Path out = Path.of(utils.getOutputDirectory(), "analysis", "traffic");
-
-		run(new TrafficDashboard());
-
-		Assertions.assertThat(out)
-			.isDirectoryContaining("glob:**traffic_stats_by_link_daily.csv")
-			.isDirectoryContaining("glob:**traffic_stats_by_road_type_and_hour.csv")
-			.isDirectoryContaining("glob:**traffic_stats_by_road_type_daily.csv");
-
-
 	}
 
 	@Test
@@ -164,19 +137,20 @@ void ptCustom() {
 		PublicTransitDashboard pt = new PublicTransitDashboard();
 
 		// bus
-		TransitViewer.CustomRouteType crt = new TransitViewer.CustomRouteType();
-		crt.label = "Bus";
-		crt.color = "#109192";
+		TransitViewer.CustomRouteType crt = TransitViewer.customRouteType("Bus", "#109192");
 		crt.addMatchGtfsRouteType(3);
 
 		// rail
-		TransitViewer.CustomRouteType crtRail = new TransitViewer.CustomRouteType();
-		crtRail.label = "Rail";
-		crtRail.color = "#EC0016";
+		TransitViewer.CustomRouteType crtRail = TransitViewer.customRouteType("Rail", "#EC0016");
 		crtRail.addMatchGtfsRouteType(2);
 
 		pt.withCustomRouteTypes(crt, crtRail);
 
 		run(pt);
+
+		Path out = Path.of(utils.getOutputDirectory(), "analysis", "pt");
+
+		Assertions.assertThat(out)
+			.isDirectoryContaining("glob:**pt_pax_volumes.csv.gz");
 	}
 }