From 1975de4dac6a5d32ae50adcfb54e4d136b316a38 Mon Sep 17 00:00:00 2001 From: patoche Date: Mon, 17 Jan 2022 08:50:41 +0100 Subject: [PATCH 01/10] [tech] make some items public --- Cargo.toml | 1 + src/calendars.rs | 3 ++- src/gtfs/mod.rs | 42 +++++++++++++++++++++++++++++-- src/gtfs/read.rs | 64 +++++++++++++---------------------------------- src/read_utils.rs | 24 ++++++++++-------- 5 files changed, 75 insertions(+), 59 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 40ef2c897..804ccc29d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ members = [ [features] xmllint = ["proj"] +gtfs_parser = [] # Experimental feature, use at your own risks mutable-model = [] diff --git a/src/calendars.rs b/src/calendars.rs index 7905506b7..5246e959e 100644 --- a/src/calendars.rs +++ b/src/calendars.rs @@ -166,7 +166,8 @@ where Ok(()) } -pub(crate) fn manage_calendars(file_handler: &mut H, collections: &mut Collections) -> Result<()> +/// read calendar_dates.txt and / or calendar.txt files from a file handler +pub fn manage_calendars(file_handler: &mut H, collections: &mut Collections) -> Result<()> where for<'a> &'a mut H: FileHandler, { diff --git a/src/gtfs/mod.rs b/src/gtfs/mod.rs index 9470a8042..21a36593e 100644 --- a/src/gtfs/mod.rs +++ b/src/gtfs/mod.rs @@ -19,7 +19,6 @@ mod write; use crate::{ calendars::{manage_calendars, write_calendar_dates}, - gtfs::read::EquipmentList, model::{Collections, Model}, objects::{self, Availability, Contributor, Dataset, StopType, Time}, read_utils, @@ -31,10 +30,20 @@ use anyhow::{anyhow, Context}; use chrono_tz::Tz; use derivative::Derivative; use serde::{Deserialize, Serialize}; -use std::{collections::BTreeMap, fmt, path::Path}; +use std::{ + collections::{BTreeMap, HashMap}, + fmt, + path::Path, +}; use tracing::info; use typed_index_collection::CollectionWithId; +#[cfg(feature = "gtfs_parser")] +pub use read::{ + manage_frequencies, manage_pathways, manage_shapes, manage_stop_times, read_agency, + read_routes, read_stops, read_transfers, +}; + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] struct Agency { #[serde(rename = "agency_id")] @@ -552,6 +561,35 @@ struct Route { sort_order: Option, } +#[derive(Default)] +///to associate a list of equipment with a stop +pub struct EquipmentList { + equipments: HashMap, +} + +impl EquipmentList { + ///convert EquipmentList to a list of transit model equipment + pub fn into_equipments(self) -> Vec { + let mut eqs: Vec<_> = self + .equipments + .into_iter() + .map(|(mut eq, id)| { + eq.id = id; + eq + }) + .collect(); + + eqs.sort_by(|l, r| l.id.cmp(&r.id)); + eqs + } + ///insert transit model equipment into EquipmentList + pub fn push(&mut self, equipment: objects::Equipment) -> String { + let equipment_id = self.equipments.len().to_string(); + let id = self.equipments.entry(equipment).or_insert(equipment_id); + id.clone() + } +} + /// Exports a `Model` to [GTFS](https://gtfs.org/reference/static) files /// in the given directory. /// see [NTFS to GTFS conversion](https://github.com/CanalTP/transit_model/blob/master/src/documentation/ntfs2gtfs.md) diff --git a/src/gtfs/read.rs b/src/gtfs/read.rs index ead66dc2a..cd38260f2 100644 --- a/src/gtfs/read.rs +++ b/src/gtfs/read.rs @@ -13,8 +13,8 @@ // along with this program. If not, see use super::{ - Agency, DirectionType, Route, RouteType, Shape, Stop, StopLocationType, StopTime, Transfer, - TransferType, Trip, + Agency, DirectionType, EquipmentList, Route, RouteType, Shape, Stop, StopLocationType, + StopTime, Transfer, TransferType, Trip, }; use crate::{ model::Collections, @@ -331,10 +331,8 @@ impl Trip { } } -pub(in crate::gtfs) fn manage_shapes( - collections: &mut Collections, - file_handler: &mut H, -) -> Result<()> +/// Reading rules for mapping vehicle travel paths, sometimes referred to as route alignments. +pub fn manage_shapes(collections: &mut Collections, file_handler: &mut H) -> Result<()> where for<'a> &'a mut H: FileHandler, { @@ -364,7 +362,8 @@ where Ok(()) } -pub(in crate::gtfs) fn manage_stop_times( +/// Reading times that a vehicle arrives at and departs from stops for each trip +pub fn manage_stop_times( collections: &mut Collections, file_handler: &mut H, on_demand_transport: bool, @@ -564,7 +563,8 @@ fn interpolate_undefined_stop_times( } } -pub(in crate::gtfs) fn read_agency( +///Reading transit agencies with service represented in this dataset. +pub fn read_agency( file_handler: &mut H, ) -> Result<( CollectionWithId, @@ -683,33 +683,6 @@ fn manage_odt_comment_from_stop_time( ); } -#[derive(Default)] -pub struct EquipmentList { - equipments: HashMap, -} - -impl EquipmentList { - pub fn into_equipments(self) -> Vec { - let mut eqs: Vec<_> = self - .equipments - .into_iter() - .map(|(mut eq, id)| { - eq.id = id; - eq - }) - .collect(); - - eqs.sort_by(|l, r| l.id.cmp(&r.id)); - eqs - } - - pub fn push(&mut self, equipment: objects::Equipment) -> String { - let equipment_id = self.equipments.len().to_string(); - let id = self.equipments.entry(equipment).or_insert(equipment_id); - id.clone() - } -} - fn get_equipment_id_and_populate_equipments( equipments: &mut EquipmentList, stop: &Stop, @@ -734,7 +707,8 @@ fn get_equipment_id_and_populate_equipments( } } -pub(in crate::gtfs) fn read_stops( +/// Reading stops where vehicles pick up or drop off riders. Also defines stations and station entrances. +pub fn read_stops( file_handler: &mut H, comments: &mut CollectionWithId, equipments: &mut EquipmentList, @@ -794,10 +768,8 @@ where Ok((stopareas, stoppoints, stoplocations)) } -pub(in crate::gtfs) fn manage_pathways( - collections: &mut Collections, - file_handler: &mut H, -) -> Result<()> +/// Reading pathways linking together locations within stations. +pub fn manage_pathways(collections: &mut Collections, file_handler: &mut H) -> Result<()> where for<'a> &'a mut H: FileHandler, { @@ -843,7 +815,8 @@ where Ok(()) } -pub(in crate::gtfs) fn read_transfers( +/// Reading rules for making connections at transfer points between routes. +pub fn read_transfers( file_handler: &mut H, stop_points: &CollectionWithId, stop_areas: &CollectionWithId, @@ -1122,7 +1095,8 @@ fn make_ntfs_vehicle_journeys( (vehicle_journeys, trip_properties) } -pub(in crate::gtfs) fn read_routes( +/// Reading transit routes. A route is a group of trips that are displayed to riders as a single service. +pub fn read_routes( file_handler: &mut H, collections: &mut Collections, read_as_line: bool, @@ -1194,10 +1168,8 @@ struct Frequency { exact_times: FrequencyPrecision, } -pub(in crate::gtfs) fn manage_frequencies( - collections: &mut Collections, - file_handler: &mut H, -) -> Result<()> +///Reading headway (time between trips) for headway-based service or a compressed representation of fixed-schedule service. +pub fn manage_frequencies(collections: &mut Collections, file_handler: &mut H) -> Result<()> where for<'a> &'a mut H: FileHandler, { diff --git a/src/read_utils.rs b/src/read_utils.rs index 5b9937803..53a363d50 100644 --- a/src/read_utils.rs +++ b/src/read_utils.rs @@ -94,14 +94,18 @@ pub fn read_config>( Ok((contributor, dataset, feed_infos)) } -pub(crate) trait FileHandler +/// Allows files in a directory or ZipArchive to be read either +pub trait FileHandler where Self: std::marker::Sized, { + /// Reader type Reader: Read; + /// Return a file if exist fn get_file_if_exists(self, name: &str) -> Result<(Option, PathBuf)>; + /// Return a file or an error if not exist fn get_file(self, name: &str) -> Result<(Self::Reader, PathBuf)> { let (reader, path) = self.get_file_if_exists(name)?; Ok(( @@ -110,16 +114,18 @@ where )) } + /// Allows to have nicer error messages fn source_name(&self) -> &str; } /// PathFileHandler is used to read files for a directory -pub(crate) struct PathFileHandler> { +pub struct PathFileHandler> { base_path: P, } impl> PathFileHandler

{ - pub(crate) fn new(path: P) -> Self { + /// Constructs a new PathFileHandler + pub fn new(path: P) -> Self { PathFileHandler { base_path: path } } } @@ -205,7 +211,7 @@ where } /// Read a vector of objects from a zip in a file_handler -pub(crate) fn read_objects( +pub fn read_objects( file_handler: &mut H, file_name: &str, required_file: bool, @@ -278,11 +284,8 @@ where } } -/// Read a CollectionId from a zip in a file_handler -pub(crate) fn read_collection( - file_handler: &mut H, - file_name: &str, -) -> Result> +/// Read a CollectionId from a required file in a file_handler +pub fn read_collection(file_handler: &mut H, file_name: &str) -> Result> where for<'a> &'a mut H: FileHandler, O: for<'de> serde::Deserialize<'de> + Id, @@ -291,7 +294,8 @@ where CollectionWithId::new(vec).map_err(|e| anyhow!("{}", e)) } -pub(crate) fn read_opt_collection( +/// Read a CollectionId from a optional file in a file_handler +pub fn read_opt_collection( file_handler: &mut H, file_name: &str, ) -> Result> From a56e0ac1c61e9962766ea5141964297c6b838f69 Mon Sep 17 00:00:00 2001 From: patoche Date: Mon, 17 Jan 2022 16:13:39 +0100 Subject: [PATCH 02/10] typo --- src/gtfs/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/gtfs/mod.rs b/src/gtfs/mod.rs index 21a36593e..6717d0854 100644 --- a/src/gtfs/mod.rs +++ b/src/gtfs/mod.rs @@ -562,13 +562,13 @@ struct Route { } #[derive(Default)] -///to associate a list of equipment with a stop +/// To associate a list of equipment with a stop pub struct EquipmentList { equipments: HashMap, } impl EquipmentList { - ///convert EquipmentList to a list of transit model equipment + /// Convert EquipmentList to a list of transit model equipments pub fn into_equipments(self) -> Vec { let mut eqs: Vec<_> = self .equipments @@ -582,7 +582,7 @@ impl EquipmentList { eqs.sort_by(|l, r| l.id.cmp(&r.id)); eqs } - ///insert transit model equipment into EquipmentList + /// Insert transit model equipment into EquipmentList pub fn push(&mut self, equipment: objects::Equipment) -> String { let equipment_id = self.equipments.len().to_string(); let id = self.equipments.entry(equipment).or_insert(equipment_id); From 9f62effede75edfb4a712d35288fb08d2f5b71de Mon Sep 17 00:00:00 2001 From: patoche Date: Tue, 18 Jan 2022 09:43:59 +0100 Subject: [PATCH 03/10] add feature parser --- Cargo.toml | 3 ++- src/gtfs/mod.rs | 25 ++++++++++++++++--------- src/read_utils.rs | 2 +- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 804ccc29d..db51230a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,8 @@ members = [ [features] xmllint = ["proj"] -gtfs_parser = [] +gtfs = [] +parser = [] # Experimental feature, use at your own risks mutable-model = [] diff --git a/src/gtfs/mod.rs b/src/gtfs/mod.rs index 6717d0854..16e08c681 100644 --- a/src/gtfs/mod.rs +++ b/src/gtfs/mod.rs @@ -18,10 +18,10 @@ mod read; mod write; use crate::{ - calendars::{manage_calendars, write_calendar_dates}, + calendars::write_calendar_dates, model::{Collections, Model}, objects::{self, Availability, Contributor, Dataset, StopType, Time}, - read_utils, + read_utils::read_opt_collection, serde_utils::*, utils::*, validity_period, AddPrefix, PrefixConfiguration, Result, @@ -35,15 +35,22 @@ use std::{ fmt, path::Path, }; + use tracing::info; use typed_index_collection::CollectionWithId; -#[cfg(feature = "gtfs_parser")] +#[cfg(feature = "gtfs")] pub use read::{ manage_frequencies, manage_pathways, manage_shapes, manage_stop_times, read_agency, read_routes, read_stops, read_transfers, }; +#[cfg(feature = "parser")] +pub use crate::{ + calendars::manage_calendars, + read_utils::{FileHandler, PathFileHandler, ZipHandler}, +}; + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] struct Agency { #[serde(rename = "agency_id")] @@ -286,7 +293,7 @@ pub struct Configuration { fn read_file_handler(file_handler: &mut H, configuration: Configuration) -> Result where - for<'a> &'a mut H: read_utils::FileHandler, + for<'a> &'a mut H: FileHandler, { let collections = read_file_handler_to_collections(file_handler, configuration)?; Model::new(collections) @@ -297,7 +304,7 @@ fn read_file_handler_to_collections( configuration: Configuration, ) -> Result where - for<'a> &'a mut H: read_utils::FileHandler, + for<'a> &'a mut H: FileHandler, { let mut collections = Collections::default(); let mut equipments = EquipmentList::default(); @@ -341,7 +348,7 @@ where )?; read::manage_frequencies(&mut collections, file_handler)?; read::manage_pathways(&mut collections, file_handler)?; - collections.levels = read_utils::read_opt_collection(file_handler, "levels.txt")?; + collections.levels = read_opt_collection(file_handler, "levels.txt")?; //add prefixes if let Some(prefix_conf) = prefix_conf { @@ -474,7 +481,7 @@ impl Reader { /// Imports `Collections` from the [GTFS](https://gtfs.org/reference/static) /// files in the `path` directory. fn parse_dir_collections(self, path: impl AsRef) -> Result { - let mut file_handler = read_utils::PathFileHandler::new(path.as_ref().to_path_buf()); + let mut file_handler = PathFileHandler::new(path.as_ref().to_path_buf()); read_file_handler_to_collections(&mut file_handler, self.configuration) } @@ -482,7 +489,7 @@ impl Reader { /// [GTFS](https://gtfs.org/reference/static). fn parse_zip_collections(self, path: impl AsRef) -> Result { let reader = std::fs::File::open(path.as_ref())?; - let mut file_handler = read_utils::ZipHandler::new(reader, path)?; + let mut file_handler = ZipHandler::new(reader, path)?; read_file_handler_to_collections(&mut file_handler, self.configuration) } @@ -505,7 +512,7 @@ impl Reader { where R: std::io::Seek + std::io::Read, { - let mut file_handler = read_utils::ZipHandler::new(reader, source_name)?; + let mut file_handler = ZipHandler::new(reader, source_name)?; read_file_handler(&mut file_handler, self.configuration) } } diff --git a/src/read_utils.rs b/src/read_utils.rs index 53a363d50..d4bc7649b 100644 --- a/src/read_utils.rs +++ b/src/read_utils.rs @@ -159,7 +159,7 @@ impl<'a, P: AsRef> FileHandler for &'a mut PathFileHandler

{ /// Unlike ZipArchive, it gives access to a file by its name not regarding its path in the ZipArchive /// It thus cannot be correct if there are 2 files with the same name in the archive, /// but for transport data if will make it possible to handle a zip with a sub directory -pub(crate) struct ZipHandler { +pub struct ZipHandler { archive: zip::ZipArchive, archive_path: PathBuf, index_by_name: BTreeMap, From bbd9ee8a45badd6865ea8844667022182696b023 Mon Sep 17 00:00:00 2001 From: patoche Date: Tue, 18 Jan 2022 12:29:37 +0100 Subject: [PATCH 04/10] refactoring --- src/calendars.rs | 25 +++++- src/file_handler/mod.rs | 168 ++++++++++++++++++++++++++++++++++++++++ src/gtfs/mod.rs | 9 +-- src/gtfs/read.rs | 11 ++- src/lib.rs | 5 +- src/ntfs/mod.rs | 18 ++--- src/ntfs/read.rs | 7 +- src/read_utils.rs | 165 +-------------------------------------- src/utils.rs | 2 +- src/validity_period.rs | 8 +- 10 files changed, 221 insertions(+), 197 deletions(-) create mode 100644 src/file_handler/mod.rs diff --git a/src/calendars.rs b/src/calendars.rs index 5246e959e..ba6c16744 100644 --- a/src/calendars.rs +++ b/src/calendars.rs @@ -17,9 +17,10 @@ //! - calendar.txt and calendar_dates.txt format are identical between the GTFS //! and NTFS +use crate::file_handler::FileHandler; use crate::model::Collections; use crate::objects::{self, Date, ExceptionType}; -use crate::read_utils::{read_objects, FileHandler}; +use crate::read_utils::read_objects; use crate::serde_utils::*; use crate::vptranslator::translate; use crate::Result; @@ -166,8 +167,10 @@ where Ok(()) } -/// read calendar_dates.txt and / or calendar.txt files from a file handler -pub fn manage_calendars(file_handler: &mut H, collections: &mut Collections) -> Result<()> +pub(crate) fn _manage_calendars( + file_handler: &mut H, + collections: &mut Collections, +) -> Result<()> where for<'a> &'a mut H: FileHandler, { @@ -189,6 +192,22 @@ where Ok(()) } +#[cfg(not(feature = "parser"))] +pub(crate) fn manage_calendars(file_handler: &mut H, collections: &mut Collections) -> Result<()> +where + for<'a> &'a mut H: FileHandler, +{ + _manage_calendars(file_handler, collections) +} +#[cfg(feature = "parser")] +/// Read calendar_dates.txt and calendar.txt files +pub fn manage_calendars(file_handler: &mut H, collections: &mut Collections) -> Result<()> +where + for<'a> &'a mut H: FileHandler, +{ + _manage_calendars(file_handler, collections) +} + /// Write the calendar_dates.txt file into a Path from a list of Calendar pub fn write_calendar_dates( path: &path::Path, diff --git a/src/file_handler/mod.rs b/src/file_handler/mod.rs new file mode 100644 index 000000000..b33fb307f --- /dev/null +++ b/src/file_handler/mod.rs @@ -0,0 +1,168 @@ +//! Provides an easy way to access directory or flat zip archive +use crate::Result; +use anyhow::{anyhow, Context}; +use std::{ + collections::BTreeMap, + fs::File, + io::{Read, Seek}, + path::{Path, PathBuf}, +}; + +/// Allows files in a directory or ZipArchive to be read either +pub trait FileHandler +where + Self: std::marker::Sized, +{ + /// Reader + type Reader: Read; + + /// Return a file if exist + fn get_file_if_exists(self, name: &str) -> Result<(Option, PathBuf)>; + + /// Return a file or an error if not exist + fn get_file(self, name: &str) -> Result<(Self::Reader, PathBuf)> { + let (reader, path) = self.get_file_if_exists(name)?; + Ok(( + reader.ok_or_else(|| anyhow!("file {:?} not found", path))?, + path, + )) + } + + /// Allows to have nicer error messages + fn source_name(&self) -> &str; +} + +/// PathFileHandler is used to read files for a directory +pub struct PathFileHandler> { + base_path: P, +} + +impl> PathFileHandler

{ + /// Constructs a new PathFileHandler + pub fn new(path: P) -> Self { + PathFileHandler { base_path: path } + } +} + +impl<'a, P: AsRef> FileHandler for &'a mut PathFileHandler

{ + type Reader = File; + fn get_file_if_exists(self, name: &str) -> Result<(Option, PathBuf)> { + let f = self.base_path.as_ref().join(name); + if f.exists() { + Ok(( + Some(File::open(&f).with_context(|| format!("Error reading {:?}", &f))?), + f, + )) + } else { + Ok((None, f)) + } + } + fn source_name(&self) -> &str { + self.base_path.as_ref().to_str().unwrap_or_else(|| { + panic!( + "the path '{:?}' should be valid UTF-8", + self.base_path.as_ref() + ) + }) + } +} + +/// ZipHandler is a wrapper around a ZipArchive +/// It provides a way to access the archive's file by their names +/// +/// Unlike ZipArchive, it gives access to a file by its name not regarding its path in the ZipArchive +/// It thus cannot be correct if there are 2 files with the same name in the archive, +/// but for transport data if will make it possible to handle a zip with a sub directory +pub struct ZipHandler { + archive: zip::ZipArchive, + archive_path: PathBuf, + index_by_name: BTreeMap, +} + +impl ZipHandler +where + R: Seek + Read, +{ + pub(crate) fn new>(r: R, path: P) -> Result { + let mut archive = zip::ZipArchive::new(r)?; + Ok(ZipHandler { + index_by_name: Self::files_by_name(&mut archive), + archive, + archive_path: path.as_ref().to_path_buf(), + }) + } + + fn files_by_name(archive: &mut zip::ZipArchive) -> BTreeMap { + (0..archive.len()) + .filter_map(|i| { + let file = archive.by_index(i).ok()?; + // we get the name of the file, not regarding its path in the ZipArchive + let real_name = Path::new(file.name()).file_name()?; + let real_name: String = real_name.to_str()?.into(); + Some((real_name, i)) + }) + .collect() + } +} + +impl<'a, R> FileHandler for &'a mut ZipHandler +where + R: Seek + Read, +{ + type Reader = zip::read::ZipFile<'a>; + fn get_file_if_exists(self, name: &str) -> Result<(Option, PathBuf)> { + let p = self.archive_path.join(name); + match self.index_by_name.get(name) { + None => Ok((None, p)), + Some(i) => Ok((Some(self.archive.by_index(*i)?), p)), + } + } + fn source_name(&self) -> &str { + self.archive_path + .to_str() + .unwrap_or_else(|| panic!("the path '{:?}' should be valid UTF-8", self.archive_path)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use std::io::Read; + + #[test] + fn path_file_handler() { + let mut file_handler = PathFileHandler::new(PathBuf::from("tests/fixtures/file-handler")); + + let (mut hello, _) = file_handler.get_file("hello.txt").unwrap(); + let mut hello_str = String::new(); + hello.read_to_string(&mut hello_str).unwrap(); + assert_eq!("hello\n", hello_str); + + let (mut world, _) = file_handler.get_file("folder/world.txt").unwrap(); + let mut world_str = String::new(); + world.read_to_string(&mut world_str).unwrap(); + assert_eq!("world\n", world_str); + } + + #[test] + fn zip_file_handler() { + let p = "tests/fixtures/file-handler.zip"; + let reader = File::open(p).unwrap(); + let mut file_handler = ZipHandler::new(reader, p).unwrap(); + + { + let (mut hello, _) = file_handler.get_file("hello.txt").unwrap(); + let mut hello_str = String::new(); + hello.read_to_string(&mut hello_str).unwrap(); + assert_eq!("hello\n", hello_str); + } + + { + let (mut world, _) = file_handler.get_file("world.txt").unwrap(); + let mut world_str = String::new(); + world.read_to_string(&mut world_str).unwrap(); + assert_eq!("world\n", world_str); + } + } +} diff --git a/src/gtfs/mod.rs b/src/gtfs/mod.rs index 16e08c681..f6401489f 100644 --- a/src/gtfs/mod.rs +++ b/src/gtfs/mod.rs @@ -18,7 +18,8 @@ mod read; mod write; use crate::{ - calendars::write_calendar_dates, + calendars::{manage_calendars, write_calendar_dates}, + file_handler::{FileHandler, PathFileHandler, ZipHandler}, model::{Collections, Model}, objects::{self, Availability, Contributor, Dataset, StopType, Time}, read_utils::read_opt_collection, @@ -45,12 +46,6 @@ pub use read::{ read_routes, read_stops, read_transfers, }; -#[cfg(feature = "parser")] -pub use crate::{ - calendars::manage_calendars, - read_utils::{FileHandler, PathFileHandler, ZipHandler}, -}; - #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] struct Agency { #[serde(rename = "agency_id")] diff --git a/src/gtfs/read.rs b/src/gtfs/read.rs index cd38260f2..ba5b2272d 100644 --- a/src/gtfs/read.rs +++ b/src/gtfs/read.rs @@ -17,13 +17,14 @@ use super::{ StopTime, Transfer, TransferType, Trip, }; use crate::{ + file_handler::FileHandler, model::Collections, objects::{ self, Availability, CommentLinksT, Coord, KeysValues, Pathway, PropertiesMap, StopLocation, StopPoint, StopTime as NtfsStopTime, StopTimePrecision, StopType, Time, TransportType, VehicleJourney, }, - read_utils::{read_collection, read_objects, read_objects_loose, FileHandler}, + read_utils::{read_collection, read_objects, read_objects_loose}, serde_utils::de_with_empty_default, Result, }; @@ -1314,11 +1315,12 @@ mod tests { use super::*; use crate::{ calendars, + file_handler::PathFileHandler, gtfs::read::EquipmentList, model::Collections, objects::*, objects::{Calendar, Comment, CommentType, Equipment, Geometry, Rgb, StopTime, Transfer}, - read_utils::{self, read_opt_collection, PathFileHandler}, + read_utils::{self, read_opt_collection}, test_utils::*, AddPrefix, PrefixConfiguration, }; @@ -3313,10 +3315,7 @@ mod tests { mod read_gtfs_routes { use super::*; - use crate::{ - model::Collections, - read_utils::{self, PathFileHandler}, - }; + use crate::{file_handler::PathFileHandler, model::Collections}; use pretty_assertions::assert_eq; use std::path; diff --git a/src/lib.rs b/src/lib.rs index 6bc8432a0..cc525501c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -51,6 +51,10 @@ pub mod calendars; #[macro_use] pub mod objects; mod enhancers; +#[cfg(not(feature = "parser"))] +pub(crate) mod file_handler; // to keep backward compatibility, not exposing FileHandler +#[cfg(feature = "parser")] +pub mod file_handler; pub mod gtfs; pub mod model; #[cfg(feature = "proj")] @@ -64,7 +68,6 @@ pub mod transfers; pub mod validity_period; mod version_utils; pub mod vptranslator; - /// Current version of the NTFS format pub const NTFS_VERSION: &str = "0.12.1"; diff --git a/src/ntfs/mod.rs b/src/ntfs/mod.rs index 775b88f8f..b4cf3986e 100644 --- a/src/ntfs/mod.rs +++ b/src/ntfs/mod.rs @@ -20,9 +20,9 @@ mod write; use crate::{ calendars::{manage_calendars, write_calendar_dates}, + file_handler::{FileHandler, PathFileHandler, ZipHandler}, model::{Collections, Model}, objects::*, - read_utils::{self, FileHandler}, serde_utils::*, utils::*, Result, @@ -180,14 +180,14 @@ fn has_fares_v1(collections: &Collections) -> bool { /// [NTFS](https://github.com/CanalTP/ntfs-specification/blob/master/ntfs_fr.md) /// files in the given directory. pub fn from_dir>(p: P) -> Result { - let mut file_handle = read_utils::PathFileHandler::new(p.as_ref().to_path_buf()); + let mut file_handle = PathFileHandler::new(p.as_ref().to_path_buf()); read_file_handler(&mut file_handle) } /// Imports a `Model` from a zip file containing the /// [NTFS](https://github.com/CanalTP/ntfs-specification/blob/master/ntfs_fr.md). pub fn from_zip>(p: P) -> Result { let reader = std::fs::File::open(p.as_ref())?; - let mut file_handler = read_utils::ZipHandler::new(reader, p)?; + let mut file_handler = ZipHandler::new(reader, p)?; read_file_handler(&mut file_handler) } @@ -195,7 +195,7 @@ pub fn from_zip>(p: P) -> Result { /// [NTFS](https://github.com/CanalTP/ntfs-specification/blob/master/ntfs_fr.md). pub fn collections_from_zip>(p: P) -> Result { let reader = std::fs::File::open(p.as_ref())?; - let mut file_handler = read_utils::ZipHandler::new(reader, p)?; + let mut file_handler = ZipHandler::new(reader, p)?; read_collections_file_handler(&mut file_handler) } @@ -203,7 +203,7 @@ pub fn collections_from_zip>(p: P) -> Result { /// [NTFS](https://github.com/CanalTP/ntfs-specification/blob/master/ntfs_fr.md) /// files in the given directory. pub fn collections_from_dir>(p: P) -> Result { - let mut file_handle = read_utils::PathFileHandler::new(p.as_ref().to_path_buf()); + let mut file_handle = PathFileHandler::new(p.as_ref().to_path_buf()); read_collections_file_handler(&mut file_handle) } @@ -226,7 +226,7 @@ pub fn from_zip_reader(reader: R, source_name: &str) -> Result where R: std::io::Seek + std::io::Read, { - let mut file_handler = read_utils::ZipHandler::new(reader, &source_name)?; + let mut file_handler = ZipHandler::new(reader, &source_name)?; read_file_handler(&mut file_handler) } @@ -277,7 +277,7 @@ pub fn read_collections>(path: P) -> Result { fn read_file_handler(file_handler: &mut H) -> Result where - for<'a> &'a mut H: read_utils::FileHandler, + for<'a> &'a mut H: FileHandler, { let collections = read_collections_file_handler(file_handler)?; info!("Indexing"); @@ -288,7 +288,7 @@ where fn read_collections_file_handler(file_handler: &mut H) -> Result where - for<'a> &'a mut H: read_utils::FileHandler, + for<'a> &'a mut H: FileHandler, { info!("Loading NTFS from {:?}", file_handler.source_name()); let mut collections = Collections { @@ -434,7 +434,7 @@ mod tests { use super::{read, write}; use crate::calendars::{manage_calendars, write_calendar_dates}; use crate::objects; - use crate::{read_utils::PathFileHandler, test_utils::*}; + use crate::{file_handler::PathFileHandler, test_utils::*}; use geo::line_string; use pretty_assertions::assert_eq; use std::{ diff --git a/src/ntfs/read.rs b/src/ntfs/read.rs index 286131638..4c05a844c 100644 --- a/src/ntfs/read.rs +++ b/src/ntfs/read.rs @@ -13,10 +13,11 @@ // along with this program. If not, see use super::{Code, CommentLink, ObjectProperty, Stop, StopLocationType, StopTime}; +use crate::file_handler::FileHandler; use crate::model::Collections; use crate::ntfs::has_fares_v2; use crate::objects::*; -use crate::read_utils::{read_objects, read_objects_loose, FileHandler}; +use crate::read_utils::{read_objects, read_objects_loose}; use crate::utils::make_opt_collection_with_id; use crate::Result; use anyhow::{anyhow, bail, ensure, Context}; @@ -671,8 +672,8 @@ where mod tests { use super::*; use crate::calendars; + use crate::file_handler::PathFileHandler; use crate::objects; - use crate::read_utils::{self, PathFileHandler}; use crate::test_utils::*; use crate::utils::make_collection_with_id; use pretty_assertions::assert_eq; @@ -740,7 +741,7 @@ mod tests { fn make_collection(path: &path::Path) -> Collections { let mut collections = Collections::default(); - let mut file_handler = read_utils::PathFileHandler::new(path.to_path_buf()); + let mut file_handler = PathFileHandler::new(path.to_path_buf()); collections.contributors = make_collection_with_id(&mut file_handler, "contributors.txt").unwrap(); collections.datasets = make_collection_with_id(&mut file_handler, "datasets.txt").unwrap(); diff --git a/src/read_utils.rs b/src/read_utils.rs index d4bc7649b..8abc2dd1a 100644 --- a/src/read_utils.rs +++ b/src/read_utils.rs @@ -14,16 +14,16 @@ //! Some utilities for input dataset to the library. use crate::{ + file_handler::FileHandler, objects::{self, Contributor}, Result, }; use anyhow::{anyhow, bail, Context}; use serde::Deserialize; use skip_error::SkipError; +use std::collections::BTreeMap; +use std::fs::File; use std::path; -use std::path::{Path, PathBuf}; -use std::{collections::BTreeMap, io::Read}; -use std::{fs::File, io::Seek}; use tracing::info; use typed_index_collection::{CollectionWithId, Id}; @@ -94,122 +94,6 @@ pub fn read_config>( Ok((contributor, dataset, feed_infos)) } -/// Allows files in a directory or ZipArchive to be read either -pub trait FileHandler -where - Self: std::marker::Sized, -{ - /// Reader - type Reader: Read; - - /// Return a file if exist - fn get_file_if_exists(self, name: &str) -> Result<(Option, PathBuf)>; - - /// Return a file or an error if not exist - fn get_file(self, name: &str) -> Result<(Self::Reader, PathBuf)> { - let (reader, path) = self.get_file_if_exists(name)?; - Ok(( - reader.ok_or_else(|| anyhow!("file {:?} not found", path))?, - path, - )) - } - - /// Allows to have nicer error messages - fn source_name(&self) -> &str; -} - -/// PathFileHandler is used to read files for a directory -pub struct PathFileHandler> { - base_path: P, -} - -impl> PathFileHandler

{ - /// Constructs a new PathFileHandler - pub fn new(path: P) -> Self { - PathFileHandler { base_path: path } - } -} - -impl<'a, P: AsRef> FileHandler for &'a mut PathFileHandler

{ - type Reader = File; - fn get_file_if_exists(self, name: &str) -> Result<(Option, PathBuf)> { - let f = self.base_path.as_ref().join(name); - if f.exists() { - Ok(( - Some(File::open(&f).with_context(|| format!("Error reading {:?}", &f))?), - f, - )) - } else { - Ok((None, f)) - } - } - fn source_name(&self) -> &str { - self.base_path.as_ref().to_str().unwrap_or_else(|| { - panic!( - "the path '{:?}' should be valid UTF-8", - self.base_path.as_ref() - ) - }) - } -} - -/// ZipHandler is a wrapper around a ZipArchive -/// It provides a way to access the archive's file by their names -/// -/// Unlike ZipArchive, it gives access to a file by its name not regarding its path in the ZipArchive -/// It thus cannot be correct if there are 2 files with the same name in the archive, -/// but for transport data if will make it possible to handle a zip with a sub directory -pub struct ZipHandler { - archive: zip::ZipArchive, - archive_path: PathBuf, - index_by_name: BTreeMap, -} - -impl ZipHandler -where - R: Seek + Read, -{ - pub(crate) fn new>(r: R, path: P) -> Result { - let mut archive = zip::ZipArchive::new(r)?; - Ok(ZipHandler { - index_by_name: Self::files_by_name(&mut archive), - archive, - archive_path: path.as_ref().to_path_buf(), - }) - } - - fn files_by_name(archive: &mut zip::ZipArchive) -> BTreeMap { - (0..archive.len()) - .filter_map(|i| { - let file = archive.by_index(i).ok()?; - // we get the name of the file, not regarding its path in the ZipArchive - let real_name = Path::new(file.name()).file_name()?; - let real_name: String = real_name.to_str()?.into(); - Some((real_name, i)) - }) - .collect() - } -} - -impl<'a, R> FileHandler for &'a mut ZipHandler -where - R: Seek + Read, -{ - type Reader = zip::read::ZipFile<'a>; - fn get_file_if_exists(self, name: &str) -> Result<(Option, PathBuf)> { - let p = self.archive_path.join(name); - match self.index_by_name.get(name) { - None => Ok((None, p)), - Some(i) => Ok((Some(self.archive.by_index(*i)?), p)), - } - } - fn source_name(&self) -> &str { - self.archive_path - .to_str() - .unwrap_or_else(|| panic!("the path '{:?}' should be valid UTF-8", self.archive_path)) - } -} - /// Read a vector of objects from a zip in a file_handler pub fn read_objects( file_handler: &mut H, @@ -306,46 +190,3 @@ where let vec = read_objects(file_handler, file_name, false)?; CollectionWithId::new(vec).map_err(|e| anyhow!("{}", e)) } - -#[cfg(test)] -mod tests { - use super::*; - use pretty_assertions::assert_eq; - use std::io::Read; - - #[test] - fn path_file_handler() { - let mut file_handler = PathFileHandler::new(PathBuf::from("tests/fixtures/file-handler")); - - let (mut hello, _) = file_handler.get_file("hello.txt").unwrap(); - let mut hello_str = String::new(); - hello.read_to_string(&mut hello_str).unwrap(); - assert_eq!("hello\n", hello_str); - - let (mut world, _) = file_handler.get_file("folder/world.txt").unwrap(); - let mut world_str = String::new(); - world.read_to_string(&mut world_str).unwrap(); - assert_eq!("world\n", world_str); - } - - #[test] - fn zip_file_handler() { - let p = "tests/fixtures/file-handler.zip"; - let reader = File::open(p).unwrap(); - let mut file_handler = ZipHandler::new(reader, p).unwrap(); - - { - let (mut hello, _) = file_handler.get_file("hello.txt").unwrap(); - let mut hello_str = String::new(); - hello.read_to_string(&mut hello_str).unwrap(); - assert_eq!("hello\n", hello_str); - } - - { - let (mut world, _) = file_handler.get_file("world.txt").unwrap(); - let mut world_str = String::new(); - world.read_to_string(&mut world_str).unwrap(); - assert_eq!("world\n", world_str); - } - } -} diff --git a/src/utils.rs b/src/utils.rs index a38df92d7..c01e78e5e 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -12,7 +12,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see -use crate::read_utils::{read_objects, FileHandler}; +use crate::{file_handler::FileHandler, read_utils::read_objects}; use anyhow::Context; use skip_error::skip_error_and_warn; use std::{ diff --git a/src/validity_period.rs b/src/validity_period.rs index 130b47f1a..62c9051b9 100644 --- a/src/validity_period.rs +++ b/src/validity_period.rs @@ -143,9 +143,7 @@ mod tests { mod compute_dataset_validity_period { use super::super::*; use crate::{ - calendars, - model::Collections, - read_utils::{self, PathFileHandler}, + calendars, file_handler::PathFileHandler, model::Collections, read_utils::*, test_utils::*, }; @@ -164,7 +162,7 @@ mod tests { create_file_with_content(path, "calendar_dates.txt", calendar_dates_content); let mut collections = Collections::default(); - let (_, mut dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (_, mut dataset, _) = read_config(None::<&str>).unwrap(); calendars::manage_calendars(&mut handler, &mut collections).unwrap(); compute_dataset_validity_period(&mut dataset, &collections.calendars).unwrap(); @@ -195,7 +193,7 @@ mod tests { create_file_with_content(path, "calendar.txt", calendars_content); let mut collections = Collections::default(); - let (_, mut dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (_, mut dataset, _) = read_config(None::<&str>).unwrap(); calendars::manage_calendars(&mut handler, &mut collections).unwrap(); compute_dataset_validity_period(&mut dataset, &collections.calendars).unwrap(); From fe6325c6c70f044e1d5c24e2c14eaa8efac139b3 Mon Sep 17 00:00:00 2001 From: patoche Date: Tue, 18 Jan 2022 16:18:01 +0100 Subject: [PATCH 05/10] consideration of comments --- src/gtfs/mod.rs | 41 ++--------------- src/gtfs/read.rs | 33 +++++++++++++- src/lib.rs | 2 +- src/read_utils.rs | 112 ++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 144 insertions(+), 44 deletions(-) diff --git a/src/gtfs/mod.rs b/src/gtfs/mod.rs index f6401489f..6ffcf1282 100644 --- a/src/gtfs/mod.rs +++ b/src/gtfs/mod.rs @@ -31,19 +31,15 @@ use anyhow::{anyhow, Context}; use chrono_tz::Tz; use derivative::Derivative; use serde::{Deserialize, Serialize}; -use std::{ - collections::{BTreeMap, HashMap}, - fmt, - path::Path, -}; +use std::{collections::BTreeMap, fmt, path::Path}; use tracing::info; use typed_index_collection::CollectionWithId; -#[cfg(feature = "gtfs")] +#[cfg(all(feature = "gtfs", feature = "parser"))] pub use read::{ manage_frequencies, manage_pathways, manage_shapes, manage_stop_times, read_agency, - read_routes, read_stops, read_transfers, + read_routes, read_stops, read_transfers, EquipmentList, }; #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] @@ -302,7 +298,7 @@ where for<'a> &'a mut H: FileHandler, { let mut collections = Collections::default(); - let mut equipments = EquipmentList::default(); + let mut equipments = read::EquipmentList::default(); let Configuration { contributor, @@ -563,35 +559,6 @@ struct Route { sort_order: Option, } -#[derive(Default)] -/// To associate a list of equipment with a stop -pub struct EquipmentList { - equipments: HashMap, -} - -impl EquipmentList { - /// Convert EquipmentList to a list of transit model equipments - pub fn into_equipments(self) -> Vec { - let mut eqs: Vec<_> = self - .equipments - .into_iter() - .map(|(mut eq, id)| { - eq.id = id; - eq - }) - .collect(); - - eqs.sort_by(|l, r| l.id.cmp(&r.id)); - eqs - } - /// Insert transit model equipment into EquipmentList - pub fn push(&mut self, equipment: objects::Equipment) -> String { - let equipment_id = self.equipments.len().to_string(); - let id = self.equipments.entry(equipment).or_insert(equipment_id); - id.clone() - } -} - /// Exports a `Model` to [GTFS](https://gtfs.org/reference/static) files /// in the given directory. /// see [NTFS to GTFS conversion](https://github.com/CanalTP/transit_model/blob/master/src/documentation/ntfs2gtfs.md) diff --git a/src/gtfs/read.rs b/src/gtfs/read.rs index ba5b2272d..179a37262 100644 --- a/src/gtfs/read.rs +++ b/src/gtfs/read.rs @@ -13,8 +13,8 @@ // along with this program. If not, see use super::{ - Agency, DirectionType, EquipmentList, Route, RouteType, Shape, Stop, StopLocationType, - StopTime, Transfer, TransferType, Trip, + Agency, DirectionType, Route, RouteType, Shape, Stop, StopLocationType, StopTime, Transfer, + TransferType, Trip, }; use crate::{ file_handler::FileHandler, @@ -684,6 +684,35 @@ fn manage_odt_comment_from_stop_time( ); } +/// To associate a list of equipment with a stop +#[derive(Default)] +pub struct EquipmentList { + equipments: HashMap, +} + +impl EquipmentList { + /// Convert EquipmentList to a list of transit model equipments + pub fn into_equipments(self) -> Vec { + let mut eqs: Vec<_> = self + .equipments + .into_iter() + .map(|(mut eq, id)| { + eq.id = id; + eq + }) + .collect(); + + eqs.sort_by(|l, r| l.id.cmp(&r.id)); + eqs + } + /// Insert transit model equipment into EquipmentList + pub fn push(&mut self, equipment: objects::Equipment) -> String { + let equipment_id = self.equipments.len().to_string(); + let id = self.equipments.entry(equipment).or_insert(equipment_id); + id.clone() + } +} + fn get_equipment_id_and_populate_equipments( equipments: &mut EquipmentList, stop: &Stop, diff --git a/src/lib.rs b/src/lib.rs index cc525501c..2b8178e98 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -52,7 +52,7 @@ pub mod calendars; pub mod objects; mod enhancers; #[cfg(not(feature = "parser"))] -pub(crate) mod file_handler; // to keep backward compatibility, not exposing FileHandler +pub(crate) mod file_handler; #[cfg(feature = "parser")] pub mod file_handler; pub mod gtfs; diff --git a/src/read_utils.rs b/src/read_utils.rs index 8abc2dd1a..e10134424 100644 --- a/src/read_utils.rs +++ b/src/read_utils.rs @@ -95,7 +95,7 @@ pub fn read_config>( } /// Read a vector of objects from a zip in a file_handler -pub fn read_objects( +pub(crate) fn _read_objects( file_handler: &mut H, file_name: &str, required_file: bool, @@ -130,8 +130,35 @@ where } } +#[cfg(not(feature = "parser"))] +pub(crate) fn read_objects( + file_handler: &mut H, + file_name: &str, + required_file: bool, +) -> Result> +where + for<'a> &'a mut H: FileHandler, + O: for<'de> serde::Deserialize<'de>, +{ + _read_objects(file_handler, file_name, required_file) +} + +#[cfg(feature = "parser")] +/// See function _read_objects +pub fn read_objects( + file_handler: &mut H, + file_name: &str, + required_file: bool, +) -> Result> +where + for<'a> &'a mut H: FileHandler, + O: for<'de> serde::Deserialize<'de>, +{ + _read_objects(file_handler, file_name, required_file) +} + /// Read a vector of objects from a zip in a file_handler ignoring error -pub(crate) fn read_objects_loose( +pub(crate) fn _read_objects_loose( file_handler: &mut H, file_name: &str, required_file: bool, @@ -168,8 +195,38 @@ where } } +#[cfg(not(feature = "parser"))] +pub(crate) fn read_objects_loose( + file_handler: &mut H, + file_name: &str, + required_file: bool, +) -> Result> +where + for<'a> &'a mut H: FileHandler, + O: for<'de> serde::Deserialize<'de>, +{ + _read_objects_loose(file_handler, file_name, required_file) +} + +#[cfg(feature = "parser")] +/// See function _read_objects_loose +pub fn read_objects_loose( + file_handler: &mut H, + file_name: &str, + required_file: bool, +) -> Result> +where + for<'a> &'a mut H: FileHandler, + O: for<'de> serde::Deserialize<'de>, +{ + _read_objects_loose(file_handler, file_name, required_file) +} + /// Read a CollectionId from a required file in a file_handler -pub fn read_collection(file_handler: &mut H, file_name: &str) -> Result> +pub(crate) fn _read_collection( + file_handler: &mut H, + file_name: &str, +) -> Result> where for<'a> &'a mut H: FileHandler, O: for<'de> serde::Deserialize<'de> + Id, @@ -178,8 +235,30 @@ where CollectionWithId::new(vec).map_err(|e| anyhow!("{}", e)) } +#[cfg(not(feature = "parser"))] +pub(crate) fn read_collection( + file_handler: &mut H, + file_name: &str, +) -> Result> +where + for<'a> &'a mut H: FileHandler, + O: for<'de> serde::Deserialize<'de> + Id, +{ + _read_collection(file_handler, file_name) +} + +#[cfg(feature = "parser")] +/// See function _read_collection +pub fn read_collection(file_handler: &mut H, file_name: &str) -> Result> +where + for<'a> &'a mut H: FileHandler, + O: for<'de> serde::Deserialize<'de> + Id, +{ + _read_collection(file_handler, file_name) +} + /// Read a CollectionId from a optional file in a file_handler -pub fn read_opt_collection( +pub(crate) fn _read_opt_collection( file_handler: &mut H, file_name: &str, ) -> Result> @@ -190,3 +269,28 @@ where let vec = read_objects(file_handler, file_name, false)?; CollectionWithId::new(vec).map_err(|e| anyhow!("{}", e)) } + +#[cfg(not(feature = "parser"))] +pub(crate) fn read_opt_collection( + file_handler: &mut H, + file_name: &str, +) -> Result> +where + for<'a> &'a mut H: FileHandler, + O: for<'de> serde::Deserialize<'de> + Id, +{ + _read_opt_collection(file_handler, file_name) +} + +#[cfg(feature = "parser")] +/// See function _read_opt_collection +pub fn read_opt_collection( + file_handler: &mut H, + file_name: &str, +) -> Result> +where + for<'a> &'a mut H: FileHandler, + O: for<'de> serde::Deserialize<'de> + Id, +{ + _read_opt_collection(file_handler, file_name) +} From fb2f8dd9663d93e24137aee5523db704b21cf6be Mon Sep 17 00:00:00 2001 From: patoche Date: Tue, 18 Jan 2022 17:33:58 +0100 Subject: [PATCH 06/10] moving parsing functions in their own module --- src/calendars.rs | 2 +- src/gtfs/mod.rs | 2 +- src/gtfs/read.rs | 49 +++++------ src/lib.rs | 4 + src/ntfs/read.rs | 2 +- src/parser/mod.rs | 116 ++++++++++++++++++++++++++ src/read_utils.rs | 205 ---------------------------------------------- src/utils.rs | 2 +- 8 files changed, 149 insertions(+), 233 deletions(-) create mode 100644 src/parser/mod.rs diff --git a/src/calendars.rs b/src/calendars.rs index ba6c16744..2ca8a5e0f 100644 --- a/src/calendars.rs +++ b/src/calendars.rs @@ -20,7 +20,7 @@ use crate::file_handler::FileHandler; use crate::model::Collections; use crate::objects::{self, Date, ExceptionType}; -use crate::read_utils::read_objects; +use crate::parser::read_objects; use crate::serde_utils::*; use crate::vptranslator::translate; use crate::Result; diff --git a/src/gtfs/mod.rs b/src/gtfs/mod.rs index 6ffcf1282..ef2812637 100644 --- a/src/gtfs/mod.rs +++ b/src/gtfs/mod.rs @@ -22,7 +22,7 @@ use crate::{ file_handler::{FileHandler, PathFileHandler, ZipHandler}, model::{Collections, Model}, objects::{self, Availability, Contributor, Dataset, StopType, Time}, - read_utils::read_opt_collection, + parser::read_opt_collection, serde_utils::*, utils::*, validity_period, AddPrefix, PrefixConfiguration, Result, diff --git a/src/gtfs/read.rs b/src/gtfs/read.rs index 179a37262..a23975957 100644 --- a/src/gtfs/read.rs +++ b/src/gtfs/read.rs @@ -24,7 +24,7 @@ use crate::{ StopPoint, StopTime as NtfsStopTime, StopTimePrecision, StopType, Time, TransportType, VehicleJourney, }, - read_utils::{read_collection, read_objects, read_objects_loose}, + parser::{read_collection, read_objects, read_objects_loose}, serde_utils::de_with_empty_default, Result, }; @@ -1349,7 +1349,8 @@ mod tests { model::Collections, objects::*, objects::{Calendar, Comment, CommentType, Equipment, Geometry, Rgb, StopTime, Transfer}, - read_utils::{self, read_opt_collection}, + parser::read_opt_collection, + read_utils::read_config, test_utils::*, AddPrefix, PrefixConfiguration, }; @@ -1602,7 +1603,7 @@ mod tests { create_file_with_content(path, "routes.txt", routes_content); create_file_with_content(path, "trips.txt", trips_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); super::read_routes(&mut handler, &mut collections, false).unwrap(); @@ -1671,7 +1672,7 @@ mod tests { let mut collections = Collections::default(); let (networks, _) = super::read_agency(&mut handler).unwrap(); collections.networks = networks; - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); super::read_routes(&mut handler, &mut collections, false).unwrap(); @@ -1714,7 +1715,7 @@ mod tests { let mut collections = Collections::default(); let (networks, _) = super::read_agency(&mut handler).unwrap(); collections.networks = networks; - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); super::read_routes(&mut handler, &mut collections, false).unwrap(); @@ -1786,7 +1787,7 @@ mod tests { let mut collections = Collections::default(); let (networks, _) = super::read_agency(&mut handler).unwrap(); collections.networks = networks; - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); super::read_routes(&mut handler, &mut collections, false).unwrap(); @@ -1817,7 +1818,7 @@ mod tests { create_file_with_content(path, "trips.txt", trips_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); super::read_routes(&mut handler, &mut collections, false).unwrap(); @@ -1852,7 +1853,7 @@ mod tests { let mut collections = Collections::default(); let (networks, _) = super::read_agency(&mut handler).unwrap(); collections.networks = networks; - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); super::read_routes(&mut handler, &mut collections, false).unwrap(); @@ -1895,7 +1896,7 @@ mod tests { create_file_with_content(path, "routes.txt", routes_content); create_file_with_content(path, "trips.txt", trips_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); super::read_routes(&mut handler, &mut collections, false).unwrap(); @@ -1928,7 +1929,7 @@ mod tests { create_file_with_content(path, "routes.txt", routes_content); create_file_with_content(path, "trips.txt", trips_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); super::read_routes(&mut handler, &mut collections, false).unwrap(); @@ -1962,7 +1963,7 @@ mod tests { create_file_with_content(path, "trips.txt", trips_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); super::read_routes(&mut handler, &mut collections, false).unwrap(); @@ -2019,7 +2020,7 @@ mod tests { let mut comments: CollectionWithId = CollectionWithId::default(); let mut equipments = EquipmentList::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); let (stop_areas, stop_points, stop_locations) = @@ -2198,7 +2199,7 @@ mod tests { create_file_with_content(path, "trips.txt", trips_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -2239,7 +2240,7 @@ mod tests { let mut collections = Collections::default(); let (networks, _) = super::read_agency(&mut handler).unwrap(); collections.networks = networks; - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -2272,7 +2273,7 @@ mod tests { create_file_with_content(path, "trips.txt", trips_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -2306,7 +2307,7 @@ mod tests { create_file_with_content(path, "trips.txt", trips_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -2487,7 +2488,7 @@ mod tests { create_file_with_content(path, "stops.txt", stops_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -2579,7 +2580,7 @@ mod tests { create_file_with_content(path, "stops.txt", stops_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -2662,7 +2663,7 @@ mod tests { create_file_with_content(path, "stops.txt", stops_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -2979,7 +2980,7 @@ mod tests { create_file_with_content(path, "trips.txt", trips_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -3084,7 +3085,7 @@ mod tests { create_file_with_content(path, "stops.txt", stops_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -3155,7 +3156,7 @@ mod tests { create_file_with_content(path, "stops.txt", stops_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -3285,7 +3286,7 @@ mod tests { create_file_with_content(path, "stops.txt", stops_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -3374,7 +3375,7 @@ mod tests { let mut collections = Collections::default(); let (networks, _) = super::read_agency(&mut handler).unwrap(); collections.networks = networks; - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); super::read_routes(&mut handler, &mut collections, read_as_line).unwrap(); diff --git a/src/lib.rs b/src/lib.rs index 2b8178e98..d851b0cfa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -61,6 +61,10 @@ pub mod model; pub mod netex_france; pub mod netex_utils; pub mod ntfs; +#[cfg(not(feature = "parser"))] +pub(crate) mod parser; +#[cfg(feature = "parser")] +pub mod parser; pub mod read_utils; #[doc(hidden)] pub mod test_utils; diff --git a/src/ntfs/read.rs b/src/ntfs/read.rs index 4c05a844c..3695fd8aa 100644 --- a/src/ntfs/read.rs +++ b/src/ntfs/read.rs @@ -17,7 +17,7 @@ use crate::file_handler::FileHandler; use crate::model::Collections; use crate::ntfs::has_fares_v2; use crate::objects::*; -use crate::read_utils::{read_objects, read_objects_loose}; +use crate::parser::{read_objects, read_objects_loose}; use crate::utils::make_opt_collection_with_id; use crate::Result; use anyhow::{anyhow, bail, ensure, Context}; diff --git a/src/parser/mod.rs b/src/parser/mod.rs new file mode 100644 index 000000000..6490fc517 --- /dev/null +++ b/src/parser/mod.rs @@ -0,0 +1,116 @@ +// Copyright (C) 2017 Kisio Digital and/or its affiliates. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Affero General Public License as published by the +// Free Software Foundation, version 3. + +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +// FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +// details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see +//! Some utilities for input dataset to the library. + +use crate::{file_handler::FileHandler, Result}; +use anyhow::{anyhow, bail, Context}; +use skip_error::SkipError; +use tracing::info; +use typed_index_collection::{CollectionWithId, Id}; + +/// Read a vector of objects from a zip in a file_handler +pub fn read_objects( + file_handler: &mut H, + file_name: &str, + required_file: bool, +) -> Result> +where + for<'a> &'a mut H: FileHandler, + O: for<'de> serde::Deserialize<'de>, +{ + let (reader, path) = file_handler.get_file_if_exists(file_name)?; + let file_name = path.file_name(); + let basename = file_name.map_or(path.to_string_lossy(), |b| b.to_string_lossy()); + + match (reader, required_file) { + (None, false) => { + info!("Skipping {}", basename); + Ok(vec![]) + } + (None, true) => { + bail!("file {:?} not found", path) + } + (Some(reader), _) => { + info!("Reading {}", basename); + let mut rdr = csv::ReaderBuilder::new() + .flexible(true) + .trim(csv::Trim::All) + .from_reader(reader); + Ok(rdr + .deserialize() + .collect::>() + .with_context(|| format!("Error reading {:?}", path))?) + } + } +} + +/// Read a vector of objects from a zip in a file_handler ignoring error +pub fn read_objects_loose( + file_handler: &mut H, + file_name: &str, + required_file: bool, +) -> Result> +where + for<'a> &'a mut H: FileHandler, + O: for<'de> serde::Deserialize<'de>, +{ + let (reader, path) = file_handler.get_file_if_exists(file_name)?; + let file_name = path.file_name(); + let basename = file_name.map_or(path.to_string_lossy(), |b| b.to_string_lossy()); + + match (reader, required_file) { + (None, false) => { + info!("Skipping {}", basename); + Ok(vec![]) + } + (None, true) => { + bail!("file {:?} not found", path) + } + (Some(reader), _) => { + info!("Reading {}", basename); + let mut rdr = csv::ReaderBuilder::new() + .flexible(true) + .trim(csv::Trim::All) + .from_reader(reader); + let objects = rdr + .deserialize() + .map(|object| object.with_context(|| format!("Error reading {:?}", path))) + .skip_error_and_warn() + .collect(); + Ok(objects) + } + } +} +/// Read a CollectionId from a zip in a file_handler +pub fn read_collection(file_handler: &mut H, file_name: &str) -> Result> +where + for<'a> &'a mut H: FileHandler, + O: for<'de> serde::Deserialize<'de> + Id, +{ + let vec = read_objects(file_handler, file_name, true)?; + CollectionWithId::new(vec).map_err(|e| anyhow!("{}", e)) +} + +/// Read a CollectionId from a optional file in a file_handler +pub fn read_opt_collection( + file_handler: &mut H, + file_name: &str, +) -> Result> +where + for<'a> &'a mut H: FileHandler, + O: for<'de> serde::Deserialize<'de> + Id, +{ + let vec = read_objects(file_handler, file_name, false)?; + CollectionWithId::new(vec).map_err(|e| anyhow!("{}", e)) +} diff --git a/src/read_utils.rs b/src/read_utils.rs index e10134424..2a763bcfc 100644 --- a/src/read_utils.rs +++ b/src/read_utils.rs @@ -14,18 +14,14 @@ //! Some utilities for input dataset to the library. use crate::{ - file_handler::FileHandler, objects::{self, Contributor}, Result, }; -use anyhow::{anyhow, bail, Context}; use serde::Deserialize; -use skip_error::SkipError; use std::collections::BTreeMap; use std::fs::File; use std::path; use tracing::info; -use typed_index_collection::{CollectionWithId, Id}; #[derive(Deserialize, Debug)] struct ConfigDataset { @@ -93,204 +89,3 @@ pub fn read_config>( Ok((contributor, dataset, feed_infos)) } - -/// Read a vector of objects from a zip in a file_handler -pub(crate) fn _read_objects( - file_handler: &mut H, - file_name: &str, - required_file: bool, -) -> Result> -where - for<'a> &'a mut H: FileHandler, - O: for<'de> serde::Deserialize<'de>, -{ - let (reader, path) = file_handler.get_file_if_exists(file_name)?; - let file_name = path.file_name(); - let basename = file_name.map_or(path.to_string_lossy(), |b| b.to_string_lossy()); - - match (reader, required_file) { - (None, false) => { - info!("Skipping {}", basename); - Ok(vec![]) - } - (None, true) => { - bail!("file {:?} not found", path) - } - (Some(reader), _) => { - info!("Reading {}", basename); - let mut rdr = csv::ReaderBuilder::new() - .flexible(true) - .trim(csv::Trim::All) - .from_reader(reader); - Ok(rdr - .deserialize() - .collect::>() - .with_context(|| format!("Error reading {:?}", path))?) - } - } -} - -#[cfg(not(feature = "parser"))] -pub(crate) fn read_objects( - file_handler: &mut H, - file_name: &str, - required_file: bool, -) -> Result> -where - for<'a> &'a mut H: FileHandler, - O: for<'de> serde::Deserialize<'de>, -{ - _read_objects(file_handler, file_name, required_file) -} - -#[cfg(feature = "parser")] -/// See function _read_objects -pub fn read_objects( - file_handler: &mut H, - file_name: &str, - required_file: bool, -) -> Result> -where - for<'a> &'a mut H: FileHandler, - O: for<'de> serde::Deserialize<'de>, -{ - _read_objects(file_handler, file_name, required_file) -} - -/// Read a vector of objects from a zip in a file_handler ignoring error -pub(crate) fn _read_objects_loose( - file_handler: &mut H, - file_name: &str, - required_file: bool, -) -> Result> -where - for<'a> &'a mut H: FileHandler, - O: for<'de> serde::Deserialize<'de>, -{ - let (reader, path) = file_handler.get_file_if_exists(file_name)?; - let file_name = path.file_name(); - let basename = file_name.map_or(path.to_string_lossy(), |b| b.to_string_lossy()); - - match (reader, required_file) { - (None, false) => { - info!("Skipping {}", basename); - Ok(vec![]) - } - (None, true) => { - bail!("file {:?} not found", path) - } - (Some(reader), _) => { - info!("Reading {}", basename); - let mut rdr = csv::ReaderBuilder::new() - .flexible(true) - .trim(csv::Trim::All) - .from_reader(reader); - let objects = rdr - .deserialize() - .map(|object| object.with_context(|| format!("Error reading {:?}", path))) - .skip_error_and_warn() - .collect(); - Ok(objects) - } - } -} - -#[cfg(not(feature = "parser"))] -pub(crate) fn read_objects_loose( - file_handler: &mut H, - file_name: &str, - required_file: bool, -) -> Result> -where - for<'a> &'a mut H: FileHandler, - O: for<'de> serde::Deserialize<'de>, -{ - _read_objects_loose(file_handler, file_name, required_file) -} - -#[cfg(feature = "parser")] -/// See function _read_objects_loose -pub fn read_objects_loose( - file_handler: &mut H, - file_name: &str, - required_file: bool, -) -> Result> -where - for<'a> &'a mut H: FileHandler, - O: for<'de> serde::Deserialize<'de>, -{ - _read_objects_loose(file_handler, file_name, required_file) -} - -/// Read a CollectionId from a required file in a file_handler -pub(crate) fn _read_collection( - file_handler: &mut H, - file_name: &str, -) -> Result> -where - for<'a> &'a mut H: FileHandler, - O: for<'de> serde::Deserialize<'de> + Id, -{ - let vec = read_objects(file_handler, file_name, true)?; - CollectionWithId::new(vec).map_err(|e| anyhow!("{}", e)) -} - -#[cfg(not(feature = "parser"))] -pub(crate) fn read_collection( - file_handler: &mut H, - file_name: &str, -) -> Result> -where - for<'a> &'a mut H: FileHandler, - O: for<'de> serde::Deserialize<'de> + Id, -{ - _read_collection(file_handler, file_name) -} - -#[cfg(feature = "parser")] -/// See function _read_collection -pub fn read_collection(file_handler: &mut H, file_name: &str) -> Result> -where - for<'a> &'a mut H: FileHandler, - O: for<'de> serde::Deserialize<'de> + Id, -{ - _read_collection(file_handler, file_name) -} - -/// Read a CollectionId from a optional file in a file_handler -pub(crate) fn _read_opt_collection( - file_handler: &mut H, - file_name: &str, -) -> Result> -where - for<'a> &'a mut H: FileHandler, - O: for<'de> serde::Deserialize<'de> + Id, -{ - let vec = read_objects(file_handler, file_name, false)?; - CollectionWithId::new(vec).map_err(|e| anyhow!("{}", e)) -} - -#[cfg(not(feature = "parser"))] -pub(crate) fn read_opt_collection( - file_handler: &mut H, - file_name: &str, -) -> Result> -where - for<'a> &'a mut H: FileHandler, - O: for<'de> serde::Deserialize<'de> + Id, -{ - _read_opt_collection(file_handler, file_name) -} - -#[cfg(feature = "parser")] -/// See function _read_opt_collection -pub fn read_opt_collection( - file_handler: &mut H, - file_name: &str, -) -> Result> -where - for<'a> &'a mut H: FileHandler, - O: for<'de> serde::Deserialize<'de> + Id, -{ - _read_opt_collection(file_handler, file_name) -} diff --git a/src/utils.rs b/src/utils.rs index c01e78e5e..9ee949fb6 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -12,7 +12,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see -use crate::{file_handler::FileHandler, read_utils::read_objects}; +use crate::{file_handler::FileHandler, parser::read_objects}; use anyhow::Context; use skip_error::skip_error_and_warn; use std::{ From c5048e45707533ac1393e81434e0391826771973 Mon Sep 17 00:00:00 2001 From: patoche Date: Wed, 19 Jan 2022 11:19:25 +0100 Subject: [PATCH 07/10] renaming --- gtfs2netexfr/src/main.rs | 4 ++-- gtfs2ntfs/src/main.rs | 4 ++-- src/{read_utils.rs => configuration.rs} | 0 src/gtfs/read.rs | 2 +- src/lib.rs | 2 +- src/{parser/mod.rs => parser.rs} | 0 src/validity_period.rs | 2 +- tests/gtfs2ntfs.rs | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) rename src/{read_utils.rs => configuration.rs} (100%) rename src/{parser/mod.rs => parser.rs} (100%) diff --git a/gtfs2netexfr/src/main.rs b/gtfs2netexfr/src/main.rs index cf9ac4746..a54c54e27 100644 --- a/gtfs2netexfr/src/main.rs +++ b/gtfs2netexfr/src/main.rs @@ -23,7 +23,7 @@ use tracing_subscriber::{ layer::SubscriberExt as _, util::SubscriberInitExt as _, }; -use transit_model::{read_utils, Result}; +use transit_model::{configuration, Result}; lazy_static::lazy_static! { pub static ref GIT_VERSION: String = transit_model::binary_full_version(env!("CARGO_PKG_VERSION")); @@ -106,7 +106,7 @@ fn init_logger() { fn run(opt: Opt) -> Result<()> { info!("Launching gtfs2netexfr..."); - let (contributor, dataset, feed_infos) = read_utils::read_config(opt.config)?; + let (contributor, dataset, feed_infos) = configuration::read_config(opt.config)?; let configuration = transit_model::gtfs::Configuration { contributor, dataset, diff --git a/gtfs2ntfs/src/main.rs b/gtfs2ntfs/src/main.rs index 2624e1d7d..6b4c2820f 100644 --- a/gtfs2ntfs/src/main.rs +++ b/gtfs2ntfs/src/main.rs @@ -23,7 +23,7 @@ use tracing_subscriber::{ layer::SubscriberExt as _, util::SubscriberInitExt as _, }; -use transit_model::{read_utils, transfers::generates_transfers, PrefixConfiguration, Result}; +use transit_model::{configuration, transfers::generates_transfers, PrefixConfiguration, Result}; lazy_static::lazy_static! { pub static ref GIT_VERSION: String = transit_model::binary_full_version(env!("CARGO_PKG_VERSION")); @@ -100,7 +100,7 @@ struct Opt { fn run(opt: Opt) -> Result<()> { info!("Launching gtfs2ntfs..."); - let (contributor, dataset, feed_infos) = read_utils::read_config(opt.config)?; + let (contributor, dataset, feed_infos) = configuration::read_config(opt.config)?; let mut prefix_conf = PrefixConfiguration::default(); if let Some(data_prefix) = opt.prefix { prefix_conf.set_data_prefix(data_prefix); diff --git a/src/read_utils.rs b/src/configuration.rs similarity index 100% rename from src/read_utils.rs rename to src/configuration.rs diff --git a/src/gtfs/read.rs b/src/gtfs/read.rs index a23975957..2ba6d31df 100644 --- a/src/gtfs/read.rs +++ b/src/gtfs/read.rs @@ -1344,13 +1344,13 @@ mod tests { use super::*; use crate::{ calendars, + configuration::read_config, file_handler::PathFileHandler, gtfs::read::EquipmentList, model::Collections, objects::*, objects::{Calendar, Comment, CommentType, Equipment, Geometry, Rgb, StopTime, Transfer}, parser::read_opt_collection, - read_utils::read_config, test_utils::*, AddPrefix, PrefixConfiguration, }; diff --git a/src/lib.rs b/src/lib.rs index d851b0cfa..96e7ea1c2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -50,6 +50,7 @@ pub use add_prefix::{AddPrefix, PrefixConfiguration}; pub mod calendars; #[macro_use] pub mod objects; +pub mod configuration; mod enhancers; #[cfg(not(feature = "parser"))] pub(crate) mod file_handler; @@ -65,7 +66,6 @@ pub mod ntfs; pub(crate) mod parser; #[cfg(feature = "parser")] pub mod parser; -pub mod read_utils; #[doc(hidden)] pub mod test_utils; pub mod transfers; diff --git a/src/parser/mod.rs b/src/parser.rs similarity index 100% rename from src/parser/mod.rs rename to src/parser.rs diff --git a/src/validity_period.rs b/src/validity_period.rs index 62c9051b9..40f400e55 100644 --- a/src/validity_period.rs +++ b/src/validity_period.rs @@ -143,7 +143,7 @@ mod tests { mod compute_dataset_validity_period { use super::super::*; use crate::{ - calendars, file_handler::PathFileHandler, model::Collections, read_utils::*, + calendars, configuration::*, file_handler::PathFileHandler, model::Collections, test_utils::*, }; diff --git a/tests/gtfs2ntfs.rs b/tests/gtfs2ntfs.rs index b420a376e..ea1ba1715 100644 --- a/tests/gtfs2ntfs.rs +++ b/tests/gtfs2ntfs.rs @@ -14,9 +14,9 @@ use std::collections::BTreeMap; use transit_model::{ + configuration::read_config, gtfs, ntfs, objects::{Contributor, Dataset}, - read_utils::read_config, test_utils::*, PrefixConfiguration, }; From 49920c2dbec2f3f938487a88e555904e92547199 Mon Sep 17 00:00:00 2001 From: patoche Date: Fri, 21 Jan 2022 16:08:20 +0100 Subject: [PATCH 08/10] add documentation --- src/lib.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 96e7ea1c2..040feb781 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,6 +38,13 @@ //! mutate a `Model`. It might not be completely stable at the moment so use //! with care (or not at all!). //! +//! ## `gtfs` +//! This feature is only used to expose some gtfs functions for use in external projects +//! +//! ## `parser` +//! Some utilities to turn csv files into vector of objects or CollectionWithId (See +//! https://github.com/CanalTP/typed_index_collection/) +//! //! [`CONTRIBUTING.md`]: https://github.com/CanalTP/transit_model/blob/master/CONTRIBUTING.md #![deny(missing_docs)] From 6f11bef5115732b36972ed4fe3f9e8477c629b99 Mon Sep 17 00:00:00 2001 From: patoche Date: Fri, 21 Jan 2022 16:29:06 +0100 Subject: [PATCH 09/10] update documentation --- src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 040feb781..121bf3707 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,7 +39,8 @@ //! with care (or not at all!). //! //! ## `gtfs` -//! This feature is only used to expose some gtfs functions for use in external projects +//! This is an experimental feature that exposes some gtfs functions for use +//! in external projects //! //! ## `parser` //! Some utilities to turn csv files into vector of objects or CollectionWithId (See From d18eb0d61a7a8a1e766ddf03fa564072959356f5 Mon Sep 17 00:00:00 2001 From: patoche Date: Mon, 24 Jan 2022 10:16:50 +0100 Subject: [PATCH 10/10] make ZipHandler constructor public --- src/file_handler/mod.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/file_handler/mod.rs b/src/file_handler/mod.rs index b33fb307f..b63521825 100644 --- a/src/file_handler/mod.rs +++ b/src/file_handler/mod.rs @@ -79,11 +79,13 @@ pub struct ZipHandler { index_by_name: BTreeMap, } +/// ZipHandler is used to read files from an archive impl ZipHandler where R: Seek + Read, { - pub(crate) fn new>(r: R, path: P) -> Result { + /// Constructs a new ZipHandler + pub fn new>(r: R, path: P) -> Result { let mut archive = zip::ZipArchive::new(r)?; Ok(ZipHandler { index_by_name: Self::files_by_name(&mut archive), @@ -91,7 +93,6 @@ where archive_path: path.as_ref().to_path_buf(), }) } - fn files_by_name(archive: &mut zip::ZipArchive) -> BTreeMap { (0..archive.len()) .filter_map(|i| {