diff --git a/Cargo.toml b/Cargo.toml index 40ef2c897..db51230a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,8 @@ members = [ [features] xmllint = ["proj"] +gtfs = [] +parser = [] # Experimental feature, use at your own risks mutable-model = [] diff --git a/gtfs2netexfr/src/main.rs b/gtfs2netexfr/src/main.rs index cf9ac4746..a54c54e27 100644 --- a/gtfs2netexfr/src/main.rs +++ b/gtfs2netexfr/src/main.rs @@ -23,7 +23,7 @@ use tracing_subscriber::{ layer::SubscriberExt as _, util::SubscriberInitExt as _, }; -use transit_model::{read_utils, Result}; +use transit_model::{configuration, Result}; lazy_static::lazy_static! { pub static ref GIT_VERSION: String = transit_model::binary_full_version(env!("CARGO_PKG_VERSION")); @@ -106,7 +106,7 @@ fn init_logger() { fn run(opt: Opt) -> Result<()> { info!("Launching gtfs2netexfr..."); - let (contributor, dataset, feed_infos) = read_utils::read_config(opt.config)?; + let (contributor, dataset, feed_infos) = configuration::read_config(opt.config)?; let configuration = transit_model::gtfs::Configuration { contributor, dataset, diff --git a/gtfs2ntfs/src/main.rs b/gtfs2ntfs/src/main.rs index 2624e1d7d..6b4c2820f 100644 --- a/gtfs2ntfs/src/main.rs +++ b/gtfs2ntfs/src/main.rs @@ -23,7 +23,7 @@ use tracing_subscriber::{ layer::SubscriberExt as _, util::SubscriberInitExt as _, }; -use transit_model::{read_utils, transfers::generates_transfers, PrefixConfiguration, Result}; +use transit_model::{configuration, transfers::generates_transfers, PrefixConfiguration, Result}; lazy_static::lazy_static! { pub static ref GIT_VERSION: String = transit_model::binary_full_version(env!("CARGO_PKG_VERSION")); @@ -100,7 +100,7 @@ struct Opt { fn run(opt: Opt) -> Result<()> { info!("Launching gtfs2ntfs..."); - let (contributor, dataset, feed_infos) = read_utils::read_config(opt.config)?; + let (contributor, dataset, feed_infos) = configuration::read_config(opt.config)?; let mut prefix_conf = PrefixConfiguration::default(); if let Some(data_prefix) = opt.prefix { prefix_conf.set_data_prefix(data_prefix); diff --git a/src/calendars.rs b/src/calendars.rs index 7905506b7..2ca8a5e0f 100644 --- a/src/calendars.rs +++ b/src/calendars.rs @@ -17,9 +17,10 @@ //! - calendar.txt and calendar_dates.txt format are identical between the GTFS //! and NTFS +use crate::file_handler::FileHandler; use crate::model::Collections; use crate::objects::{self, Date, ExceptionType}; -use crate::read_utils::{read_objects, FileHandler}; +use crate::parser::read_objects; use crate::serde_utils::*; use crate::vptranslator::translate; use crate::Result; @@ -166,7 +167,10 @@ where Ok(()) } -pub(crate) fn manage_calendars(file_handler: &mut H, collections: &mut Collections) -> Result<()> +pub(crate) fn _manage_calendars( + file_handler: &mut H, + collections: &mut Collections, +) -> Result<()> where for<'a> &'a mut H: FileHandler, { @@ -188,6 +192,22 @@ where Ok(()) } +#[cfg(not(feature = "parser"))] +pub(crate) fn manage_calendars(file_handler: &mut H, collections: &mut Collections) -> Result<()> +where + for<'a> &'a mut H: FileHandler, +{ + _manage_calendars(file_handler, collections) +} +#[cfg(feature = "parser")] +/// Read calendar_dates.txt and calendar.txt files +pub fn manage_calendars(file_handler: &mut H, collections: &mut Collections) -> Result<()> +where + for<'a> &'a mut H: FileHandler, +{ + _manage_calendars(file_handler, collections) +} + /// Write the calendar_dates.txt file into a Path from a list of Calendar pub fn write_calendar_dates( path: &path::Path, diff --git a/src/configuration.rs b/src/configuration.rs new file mode 100644 index 000000000..2a763bcfc --- /dev/null +++ b/src/configuration.rs @@ -0,0 +1,91 @@ +// Copyright (C) 2017 Kisio Digital and/or its affiliates. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Affero General Public License as published by the +// Free Software Foundation, version 3. + +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +// FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +// details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see +//! Some utilities for input dataset to the library. + +use crate::{ + objects::{self, Contributor}, + Result, +}; +use serde::Deserialize; +use std::collections::BTreeMap; +use std::fs::File; +use std::path; +use tracing::info; + +#[derive(Deserialize, Debug)] +struct ConfigDataset { + dataset_id: String, +} + +#[derive(Deserialize, Debug)] +struct Config { + contributor: objects::Contributor, + dataset: ConfigDataset, + feed_infos: Option>, +} + +/// Read a JSON configuration file to facilitate the creation of: +/// - a Contributor +/// - a Dataset +/// - a list of key/value which will be used in 'feed_infos.txt' +/// Below is an example of this file +/// ```text +/// { +/// "contributor": { +/// "contributor_id": "contributor_id", +/// "contributor_name": "Contributor Name", +/// "contributor_license": "AGPIT", +/// "contributor_website": "http://www.datasource-website.com" +/// }, +/// "dataset": { +/// "dataset_id": "dataset-id" +/// }, +/// "feed_infos": { +/// "feed_publisher_name": "The Great Data Publisher", +/// "feed_license": "AGPIT", +/// "feed_license_url": "http://www.datasource-website.com", +/// "tartare_platform": "dev", +/// "tartare_contributor_id": "contributor_id" +/// } +/// } +/// ``` +pub fn read_config>( + config_path: Option

, +) -> Result<( + objects::Contributor, + objects::Dataset, + BTreeMap, +)> { + let contributor; + let dataset; + let mut feed_infos = BTreeMap::default(); + + if let Some(config_path) = config_path { + let config_path = config_path.as_ref(); + info!("Reading dataset and contributor from {:?}", config_path); + let json_config_file = File::open(config_path)?; + let config: Config = serde_json::from_reader(json_config_file)?; + + contributor = config.contributor; + dataset = objects::Dataset::new(config.dataset.dataset_id, contributor.id.clone()); + if let Some(config_feed_infos) = config.feed_infos { + feed_infos = config_feed_infos; + } + } else { + contributor = Contributor::default(); + dataset = objects::Dataset::default(); + } + + Ok((contributor, dataset, feed_infos)) +} diff --git a/src/file_handler/mod.rs b/src/file_handler/mod.rs new file mode 100644 index 000000000..b63521825 --- /dev/null +++ b/src/file_handler/mod.rs @@ -0,0 +1,169 @@ +//! Provides an easy way to access directory or flat zip archive +use crate::Result; +use anyhow::{anyhow, Context}; +use std::{ + collections::BTreeMap, + fs::File, + io::{Read, Seek}, + path::{Path, PathBuf}, +}; + +/// Allows files in a directory or ZipArchive to be read either +pub trait FileHandler +where + Self: std::marker::Sized, +{ + /// Reader + type Reader: Read; + + /// Return a file if exist + fn get_file_if_exists(self, name: &str) -> Result<(Option, PathBuf)>; + + /// Return a file or an error if not exist + fn get_file(self, name: &str) -> Result<(Self::Reader, PathBuf)> { + let (reader, path) = self.get_file_if_exists(name)?; + Ok(( + reader.ok_or_else(|| anyhow!("file {:?} not found", path))?, + path, + )) + } + + /// Allows to have nicer error messages + fn source_name(&self) -> &str; +} + +/// PathFileHandler is used to read files for a directory +pub struct PathFileHandler> { + base_path: P, +} + +impl> PathFileHandler

{ + /// Constructs a new PathFileHandler + pub fn new(path: P) -> Self { + PathFileHandler { base_path: path } + } +} + +impl<'a, P: AsRef> FileHandler for &'a mut PathFileHandler

{ + type Reader = File; + fn get_file_if_exists(self, name: &str) -> Result<(Option, PathBuf)> { + let f = self.base_path.as_ref().join(name); + if f.exists() { + Ok(( + Some(File::open(&f).with_context(|| format!("Error reading {:?}", &f))?), + f, + )) + } else { + Ok((None, f)) + } + } + fn source_name(&self) -> &str { + self.base_path.as_ref().to_str().unwrap_or_else(|| { + panic!( + "the path '{:?}' should be valid UTF-8", + self.base_path.as_ref() + ) + }) + } +} + +/// ZipHandler is a wrapper around a ZipArchive +/// It provides a way to access the archive's file by their names +/// +/// Unlike ZipArchive, it gives access to a file by its name not regarding its path in the ZipArchive +/// It thus cannot be correct if there are 2 files with the same name in the archive, +/// but for transport data if will make it possible to handle a zip with a sub directory +pub struct ZipHandler { + archive: zip::ZipArchive, + archive_path: PathBuf, + index_by_name: BTreeMap, +} + +/// ZipHandler is used to read files from an archive +impl ZipHandler +where + R: Seek + Read, +{ + /// Constructs a new ZipHandler + pub fn new>(r: R, path: P) -> Result { + let mut archive = zip::ZipArchive::new(r)?; + Ok(ZipHandler { + index_by_name: Self::files_by_name(&mut archive), + archive, + archive_path: path.as_ref().to_path_buf(), + }) + } + fn files_by_name(archive: &mut zip::ZipArchive) -> BTreeMap { + (0..archive.len()) + .filter_map(|i| { + let file = archive.by_index(i).ok()?; + // we get the name of the file, not regarding its path in the ZipArchive + let real_name = Path::new(file.name()).file_name()?; + let real_name: String = real_name.to_str()?.into(); + Some((real_name, i)) + }) + .collect() + } +} + +impl<'a, R> FileHandler for &'a mut ZipHandler +where + R: Seek + Read, +{ + type Reader = zip::read::ZipFile<'a>; + fn get_file_if_exists(self, name: &str) -> Result<(Option, PathBuf)> { + let p = self.archive_path.join(name); + match self.index_by_name.get(name) { + None => Ok((None, p)), + Some(i) => Ok((Some(self.archive.by_index(*i)?), p)), + } + } + fn source_name(&self) -> &str { + self.archive_path + .to_str() + .unwrap_or_else(|| panic!("the path '{:?}' should be valid UTF-8", self.archive_path)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + use std::io::Read; + + #[test] + fn path_file_handler() { + let mut file_handler = PathFileHandler::new(PathBuf::from("tests/fixtures/file-handler")); + + let (mut hello, _) = file_handler.get_file("hello.txt").unwrap(); + let mut hello_str = String::new(); + hello.read_to_string(&mut hello_str).unwrap(); + assert_eq!("hello\n", hello_str); + + let (mut world, _) = file_handler.get_file("folder/world.txt").unwrap(); + let mut world_str = String::new(); + world.read_to_string(&mut world_str).unwrap(); + assert_eq!("world\n", world_str); + } + + #[test] + fn zip_file_handler() { + let p = "tests/fixtures/file-handler.zip"; + let reader = File::open(p).unwrap(); + let mut file_handler = ZipHandler::new(reader, p).unwrap(); + + { + let (mut hello, _) = file_handler.get_file("hello.txt").unwrap(); + let mut hello_str = String::new(); + hello.read_to_string(&mut hello_str).unwrap(); + assert_eq!("hello\n", hello_str); + } + + { + let (mut world, _) = file_handler.get_file("world.txt").unwrap(); + let mut world_str = String::new(); + world.read_to_string(&mut world_str).unwrap(); + assert_eq!("world\n", world_str); + } + } +} diff --git a/src/gtfs/mod.rs b/src/gtfs/mod.rs index 9470a8042..ef2812637 100644 --- a/src/gtfs/mod.rs +++ b/src/gtfs/mod.rs @@ -19,10 +19,10 @@ mod write; use crate::{ calendars::{manage_calendars, write_calendar_dates}, - gtfs::read::EquipmentList, + file_handler::{FileHandler, PathFileHandler, ZipHandler}, model::{Collections, Model}, objects::{self, Availability, Contributor, Dataset, StopType, Time}, - read_utils, + parser::read_opt_collection, serde_utils::*, utils::*, validity_period, AddPrefix, PrefixConfiguration, Result, @@ -32,9 +32,16 @@ use chrono_tz::Tz; use derivative::Derivative; use serde::{Deserialize, Serialize}; use std::{collections::BTreeMap, fmt, path::Path}; + use tracing::info; use typed_index_collection::CollectionWithId; +#[cfg(all(feature = "gtfs", feature = "parser"))] +pub use read::{ + manage_frequencies, manage_pathways, manage_shapes, manage_stop_times, read_agency, + read_routes, read_stops, read_transfers, EquipmentList, +}; + #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] struct Agency { #[serde(rename = "agency_id")] @@ -277,7 +284,7 @@ pub struct Configuration { fn read_file_handler(file_handler: &mut H, configuration: Configuration) -> Result where - for<'a> &'a mut H: read_utils::FileHandler, + for<'a> &'a mut H: FileHandler, { let collections = read_file_handler_to_collections(file_handler, configuration)?; Model::new(collections) @@ -288,10 +295,10 @@ fn read_file_handler_to_collections( configuration: Configuration, ) -> Result where - for<'a> &'a mut H: read_utils::FileHandler, + for<'a> &'a mut H: FileHandler, { let mut collections = Collections::default(); - let mut equipments = EquipmentList::default(); + let mut equipments = read::EquipmentList::default(); let Configuration { contributor, @@ -332,7 +339,7 @@ where )?; read::manage_frequencies(&mut collections, file_handler)?; read::manage_pathways(&mut collections, file_handler)?; - collections.levels = read_utils::read_opt_collection(file_handler, "levels.txt")?; + collections.levels = read_opt_collection(file_handler, "levels.txt")?; //add prefixes if let Some(prefix_conf) = prefix_conf { @@ -465,7 +472,7 @@ impl Reader { /// Imports `Collections` from the [GTFS](https://gtfs.org/reference/static) /// files in the `path` directory. fn parse_dir_collections(self, path: impl AsRef) -> Result { - let mut file_handler = read_utils::PathFileHandler::new(path.as_ref().to_path_buf()); + let mut file_handler = PathFileHandler::new(path.as_ref().to_path_buf()); read_file_handler_to_collections(&mut file_handler, self.configuration) } @@ -473,7 +480,7 @@ impl Reader { /// [GTFS](https://gtfs.org/reference/static). fn parse_zip_collections(self, path: impl AsRef) -> Result { let reader = std::fs::File::open(path.as_ref())?; - let mut file_handler = read_utils::ZipHandler::new(reader, path)?; + let mut file_handler = ZipHandler::new(reader, path)?; read_file_handler_to_collections(&mut file_handler, self.configuration) } @@ -496,7 +503,7 @@ impl Reader { where R: std::io::Seek + std::io::Read, { - let mut file_handler = read_utils::ZipHandler::new(reader, source_name)?; + let mut file_handler = ZipHandler::new(reader, source_name)?; read_file_handler(&mut file_handler, self.configuration) } } diff --git a/src/gtfs/read.rs b/src/gtfs/read.rs index ead66dc2a..2ba6d31df 100644 --- a/src/gtfs/read.rs +++ b/src/gtfs/read.rs @@ -17,13 +17,14 @@ use super::{ TransferType, Trip, }; use crate::{ + file_handler::FileHandler, model::Collections, objects::{ self, Availability, CommentLinksT, Coord, KeysValues, Pathway, PropertiesMap, StopLocation, StopPoint, StopTime as NtfsStopTime, StopTimePrecision, StopType, Time, TransportType, VehicleJourney, }, - read_utils::{read_collection, read_objects, read_objects_loose, FileHandler}, + parser::{read_collection, read_objects, read_objects_loose}, serde_utils::de_with_empty_default, Result, }; @@ -331,10 +332,8 @@ impl Trip { } } -pub(in crate::gtfs) fn manage_shapes( - collections: &mut Collections, - file_handler: &mut H, -) -> Result<()> +/// Reading rules for mapping vehicle travel paths, sometimes referred to as route alignments. +pub fn manage_shapes(collections: &mut Collections, file_handler: &mut H) -> Result<()> where for<'a> &'a mut H: FileHandler, { @@ -364,7 +363,8 @@ where Ok(()) } -pub(in crate::gtfs) fn manage_stop_times( +/// Reading times that a vehicle arrives at and departs from stops for each trip +pub fn manage_stop_times( collections: &mut Collections, file_handler: &mut H, on_demand_transport: bool, @@ -564,7 +564,8 @@ fn interpolate_undefined_stop_times( } } -pub(in crate::gtfs) fn read_agency( +///Reading transit agencies with service represented in this dataset. +pub fn read_agency( file_handler: &mut H, ) -> Result<( CollectionWithId, @@ -683,12 +684,14 @@ fn manage_odt_comment_from_stop_time( ); } +/// To associate a list of equipment with a stop #[derive(Default)] pub struct EquipmentList { equipments: HashMap, } impl EquipmentList { + /// Convert EquipmentList to a list of transit model equipments pub fn into_equipments(self) -> Vec { let mut eqs: Vec<_> = self .equipments @@ -702,7 +705,7 @@ impl EquipmentList { eqs.sort_by(|l, r| l.id.cmp(&r.id)); eqs } - + /// Insert transit model equipment into EquipmentList pub fn push(&mut self, equipment: objects::Equipment) -> String { let equipment_id = self.equipments.len().to_string(); let id = self.equipments.entry(equipment).or_insert(equipment_id); @@ -734,7 +737,8 @@ fn get_equipment_id_and_populate_equipments( } } -pub(in crate::gtfs) fn read_stops( +/// Reading stops where vehicles pick up or drop off riders. Also defines stations and station entrances. +pub fn read_stops( file_handler: &mut H, comments: &mut CollectionWithId, equipments: &mut EquipmentList, @@ -794,10 +798,8 @@ where Ok((stopareas, stoppoints, stoplocations)) } -pub(in crate::gtfs) fn manage_pathways( - collections: &mut Collections, - file_handler: &mut H, -) -> Result<()> +/// Reading pathways linking together locations within stations. +pub fn manage_pathways(collections: &mut Collections, file_handler: &mut H) -> Result<()> where for<'a> &'a mut H: FileHandler, { @@ -843,7 +845,8 @@ where Ok(()) } -pub(in crate::gtfs) fn read_transfers( +/// Reading rules for making connections at transfer points between routes. +pub fn read_transfers( file_handler: &mut H, stop_points: &CollectionWithId, stop_areas: &CollectionWithId, @@ -1122,7 +1125,8 @@ fn make_ntfs_vehicle_journeys( (vehicle_journeys, trip_properties) } -pub(in crate::gtfs) fn read_routes( +/// Reading transit routes. A route is a group of trips that are displayed to riders as a single service. +pub fn read_routes( file_handler: &mut H, collections: &mut Collections, read_as_line: bool, @@ -1194,10 +1198,8 @@ struct Frequency { exact_times: FrequencyPrecision, } -pub(in crate::gtfs) fn manage_frequencies( - collections: &mut Collections, - file_handler: &mut H, -) -> Result<()> +///Reading headway (time between trips) for headway-based service or a compressed representation of fixed-schedule service. +pub fn manage_frequencies(collections: &mut Collections, file_handler: &mut H) -> Result<()> where for<'a> &'a mut H: FileHandler, { @@ -1342,11 +1344,13 @@ mod tests { use super::*; use crate::{ calendars, + configuration::read_config, + file_handler::PathFileHandler, gtfs::read::EquipmentList, model::Collections, objects::*, objects::{Calendar, Comment, CommentType, Equipment, Geometry, Rgb, StopTime, Transfer}, - read_utils::{self, read_opt_collection, PathFileHandler}, + parser::read_opt_collection, test_utils::*, AddPrefix, PrefixConfiguration, }; @@ -1599,7 +1603,7 @@ mod tests { create_file_with_content(path, "routes.txt", routes_content); create_file_with_content(path, "trips.txt", trips_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); super::read_routes(&mut handler, &mut collections, false).unwrap(); @@ -1668,7 +1672,7 @@ mod tests { let mut collections = Collections::default(); let (networks, _) = super::read_agency(&mut handler).unwrap(); collections.networks = networks; - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); super::read_routes(&mut handler, &mut collections, false).unwrap(); @@ -1711,7 +1715,7 @@ mod tests { let mut collections = Collections::default(); let (networks, _) = super::read_agency(&mut handler).unwrap(); collections.networks = networks; - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); super::read_routes(&mut handler, &mut collections, false).unwrap(); @@ -1783,7 +1787,7 @@ mod tests { let mut collections = Collections::default(); let (networks, _) = super::read_agency(&mut handler).unwrap(); collections.networks = networks; - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); super::read_routes(&mut handler, &mut collections, false).unwrap(); @@ -1814,7 +1818,7 @@ mod tests { create_file_with_content(path, "trips.txt", trips_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); super::read_routes(&mut handler, &mut collections, false).unwrap(); @@ -1849,7 +1853,7 @@ mod tests { let mut collections = Collections::default(); let (networks, _) = super::read_agency(&mut handler).unwrap(); collections.networks = networks; - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); super::read_routes(&mut handler, &mut collections, false).unwrap(); @@ -1892,7 +1896,7 @@ mod tests { create_file_with_content(path, "routes.txt", routes_content); create_file_with_content(path, "trips.txt", trips_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); super::read_routes(&mut handler, &mut collections, false).unwrap(); @@ -1925,7 +1929,7 @@ mod tests { create_file_with_content(path, "routes.txt", routes_content); create_file_with_content(path, "trips.txt", trips_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); super::read_routes(&mut handler, &mut collections, false).unwrap(); @@ -1959,7 +1963,7 @@ mod tests { create_file_with_content(path, "trips.txt", trips_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); super::read_routes(&mut handler, &mut collections, false).unwrap(); @@ -2016,7 +2020,7 @@ mod tests { let mut comments: CollectionWithId = CollectionWithId::default(); let mut equipments = EquipmentList::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); let (stop_areas, stop_points, stop_locations) = @@ -2195,7 +2199,7 @@ mod tests { create_file_with_content(path, "trips.txt", trips_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -2236,7 +2240,7 @@ mod tests { let mut collections = Collections::default(); let (networks, _) = super::read_agency(&mut handler).unwrap(); collections.networks = networks; - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -2269,7 +2273,7 @@ mod tests { create_file_with_content(path, "trips.txt", trips_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -2303,7 +2307,7 @@ mod tests { create_file_with_content(path, "trips.txt", trips_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -2484,7 +2488,7 @@ mod tests { create_file_with_content(path, "stops.txt", stops_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -2576,7 +2580,7 @@ mod tests { create_file_with_content(path, "stops.txt", stops_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -2659,7 +2663,7 @@ mod tests { create_file_with_content(path, "stops.txt", stops_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -2976,7 +2980,7 @@ mod tests { create_file_with_content(path, "trips.txt", trips_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -3081,7 +3085,7 @@ mod tests { create_file_with_content(path, "stops.txt", stops_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -3152,7 +3156,7 @@ mod tests { create_file_with_content(path, "stops.txt", stops_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -3282,7 +3286,7 @@ mod tests { create_file_with_content(path, "stops.txt", stops_content); let mut collections = Collections::default(); - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); @@ -3341,10 +3345,7 @@ mod tests { mod read_gtfs_routes { use super::*; - use crate::{ - model::Collections, - read_utils::{self, PathFileHandler}, - }; + use crate::{file_handler::PathFileHandler, model::Collections}; use pretty_assertions::assert_eq; use std::path; @@ -3374,7 +3375,7 @@ mod tests { let mut collections = Collections::default(); let (networks, _) = super::read_agency(&mut handler).unwrap(); collections.networks = networks; - let (contributor, dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (contributor, dataset, _) = read_config(None::<&str>).unwrap(); collections.contributors = CollectionWithId::new(vec![contributor]).unwrap(); collections.datasets = CollectionWithId::new(vec![dataset]).unwrap(); super::read_routes(&mut handler, &mut collections, read_as_line).unwrap(); diff --git a/src/lib.rs b/src/lib.rs index 6bc8432a0..121bf3707 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,6 +38,14 @@ //! mutate a `Model`. It might not be completely stable at the moment so use //! with care (or not at all!). //! +//! ## `gtfs` +//! This is an experimental feature that exposes some gtfs functions for use +//! in external projects +//! +//! ## `parser` +//! Some utilities to turn csv files into vector of objects or CollectionWithId (See +//! https://github.com/CanalTP/typed_index_collection/) +//! //! [`CONTRIBUTING.md`]: https://github.com/CanalTP/transit_model/blob/master/CONTRIBUTING.md #![deny(missing_docs)] @@ -50,21 +58,28 @@ pub use add_prefix::{AddPrefix, PrefixConfiguration}; pub mod calendars; #[macro_use] pub mod objects; +pub mod configuration; mod enhancers; +#[cfg(not(feature = "parser"))] +pub(crate) mod file_handler; +#[cfg(feature = "parser")] +pub mod file_handler; pub mod gtfs; pub mod model; #[cfg(feature = "proj")] pub mod netex_france; pub mod netex_utils; pub mod ntfs; -pub mod read_utils; +#[cfg(not(feature = "parser"))] +pub(crate) mod parser; +#[cfg(feature = "parser")] +pub mod parser; #[doc(hidden)] pub mod test_utils; pub mod transfers; pub mod validity_period; mod version_utils; pub mod vptranslator; - /// Current version of the NTFS format pub const NTFS_VERSION: &str = "0.12.1"; diff --git a/src/ntfs/mod.rs b/src/ntfs/mod.rs index 775b88f8f..b4cf3986e 100644 --- a/src/ntfs/mod.rs +++ b/src/ntfs/mod.rs @@ -20,9 +20,9 @@ mod write; use crate::{ calendars::{manage_calendars, write_calendar_dates}, + file_handler::{FileHandler, PathFileHandler, ZipHandler}, model::{Collections, Model}, objects::*, - read_utils::{self, FileHandler}, serde_utils::*, utils::*, Result, @@ -180,14 +180,14 @@ fn has_fares_v1(collections: &Collections) -> bool { /// [NTFS](https://github.com/CanalTP/ntfs-specification/blob/master/ntfs_fr.md) /// files in the given directory. pub fn from_dir>(p: P) -> Result { - let mut file_handle = read_utils::PathFileHandler::new(p.as_ref().to_path_buf()); + let mut file_handle = PathFileHandler::new(p.as_ref().to_path_buf()); read_file_handler(&mut file_handle) } /// Imports a `Model` from a zip file containing the /// [NTFS](https://github.com/CanalTP/ntfs-specification/blob/master/ntfs_fr.md). pub fn from_zip>(p: P) -> Result { let reader = std::fs::File::open(p.as_ref())?; - let mut file_handler = read_utils::ZipHandler::new(reader, p)?; + let mut file_handler = ZipHandler::new(reader, p)?; read_file_handler(&mut file_handler) } @@ -195,7 +195,7 @@ pub fn from_zip>(p: P) -> Result { /// [NTFS](https://github.com/CanalTP/ntfs-specification/blob/master/ntfs_fr.md). pub fn collections_from_zip>(p: P) -> Result { let reader = std::fs::File::open(p.as_ref())?; - let mut file_handler = read_utils::ZipHandler::new(reader, p)?; + let mut file_handler = ZipHandler::new(reader, p)?; read_collections_file_handler(&mut file_handler) } @@ -203,7 +203,7 @@ pub fn collections_from_zip>(p: P) -> Result { /// [NTFS](https://github.com/CanalTP/ntfs-specification/blob/master/ntfs_fr.md) /// files in the given directory. pub fn collections_from_dir>(p: P) -> Result { - let mut file_handle = read_utils::PathFileHandler::new(p.as_ref().to_path_buf()); + let mut file_handle = PathFileHandler::new(p.as_ref().to_path_buf()); read_collections_file_handler(&mut file_handle) } @@ -226,7 +226,7 @@ pub fn from_zip_reader(reader: R, source_name: &str) -> Result where R: std::io::Seek + std::io::Read, { - let mut file_handler = read_utils::ZipHandler::new(reader, &source_name)?; + let mut file_handler = ZipHandler::new(reader, &source_name)?; read_file_handler(&mut file_handler) } @@ -277,7 +277,7 @@ pub fn read_collections>(path: P) -> Result { fn read_file_handler(file_handler: &mut H) -> Result where - for<'a> &'a mut H: read_utils::FileHandler, + for<'a> &'a mut H: FileHandler, { let collections = read_collections_file_handler(file_handler)?; info!("Indexing"); @@ -288,7 +288,7 @@ where fn read_collections_file_handler(file_handler: &mut H) -> Result where - for<'a> &'a mut H: read_utils::FileHandler, + for<'a> &'a mut H: FileHandler, { info!("Loading NTFS from {:?}", file_handler.source_name()); let mut collections = Collections { @@ -434,7 +434,7 @@ mod tests { use super::{read, write}; use crate::calendars::{manage_calendars, write_calendar_dates}; use crate::objects; - use crate::{read_utils::PathFileHandler, test_utils::*}; + use crate::{file_handler::PathFileHandler, test_utils::*}; use geo::line_string; use pretty_assertions::assert_eq; use std::{ diff --git a/src/ntfs/read.rs b/src/ntfs/read.rs index 286131638..3695fd8aa 100644 --- a/src/ntfs/read.rs +++ b/src/ntfs/read.rs @@ -13,10 +13,11 @@ // along with this program. If not, see use super::{Code, CommentLink, ObjectProperty, Stop, StopLocationType, StopTime}; +use crate::file_handler::FileHandler; use crate::model::Collections; use crate::ntfs::has_fares_v2; use crate::objects::*; -use crate::read_utils::{read_objects, read_objects_loose, FileHandler}; +use crate::parser::{read_objects, read_objects_loose}; use crate::utils::make_opt_collection_with_id; use crate::Result; use anyhow::{anyhow, bail, ensure, Context}; @@ -671,8 +672,8 @@ where mod tests { use super::*; use crate::calendars; + use crate::file_handler::PathFileHandler; use crate::objects; - use crate::read_utils::{self, PathFileHandler}; use crate::test_utils::*; use crate::utils::make_collection_with_id; use pretty_assertions::assert_eq; @@ -740,7 +741,7 @@ mod tests { fn make_collection(path: &path::Path) -> Collections { let mut collections = Collections::default(); - let mut file_handler = read_utils::PathFileHandler::new(path.to_path_buf()); + let mut file_handler = PathFileHandler::new(path.to_path_buf()); collections.contributors = make_collection_with_id(&mut file_handler, "contributors.txt").unwrap(); collections.datasets = make_collection_with_id(&mut file_handler, "datasets.txt").unwrap(); diff --git a/src/parser.rs b/src/parser.rs new file mode 100644 index 000000000..6490fc517 --- /dev/null +++ b/src/parser.rs @@ -0,0 +1,116 @@ +// Copyright (C) 2017 Kisio Digital and/or its affiliates. +// +// This program is free software: you can redistribute it and/or modify it +// under the terms of the GNU Affero General Public License as published by the +// Free Software Foundation, version 3. + +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +// FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +// details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see +//! Some utilities for input dataset to the library. + +use crate::{file_handler::FileHandler, Result}; +use anyhow::{anyhow, bail, Context}; +use skip_error::SkipError; +use tracing::info; +use typed_index_collection::{CollectionWithId, Id}; + +/// Read a vector of objects from a zip in a file_handler +pub fn read_objects( + file_handler: &mut H, + file_name: &str, + required_file: bool, +) -> Result> +where + for<'a> &'a mut H: FileHandler, + O: for<'de> serde::Deserialize<'de>, +{ + let (reader, path) = file_handler.get_file_if_exists(file_name)?; + let file_name = path.file_name(); + let basename = file_name.map_or(path.to_string_lossy(), |b| b.to_string_lossy()); + + match (reader, required_file) { + (None, false) => { + info!("Skipping {}", basename); + Ok(vec![]) + } + (None, true) => { + bail!("file {:?} not found", path) + } + (Some(reader), _) => { + info!("Reading {}", basename); + let mut rdr = csv::ReaderBuilder::new() + .flexible(true) + .trim(csv::Trim::All) + .from_reader(reader); + Ok(rdr + .deserialize() + .collect::>() + .with_context(|| format!("Error reading {:?}", path))?) + } + } +} + +/// Read a vector of objects from a zip in a file_handler ignoring error +pub fn read_objects_loose( + file_handler: &mut H, + file_name: &str, + required_file: bool, +) -> Result> +where + for<'a> &'a mut H: FileHandler, + O: for<'de> serde::Deserialize<'de>, +{ + let (reader, path) = file_handler.get_file_if_exists(file_name)?; + let file_name = path.file_name(); + let basename = file_name.map_or(path.to_string_lossy(), |b| b.to_string_lossy()); + + match (reader, required_file) { + (None, false) => { + info!("Skipping {}", basename); + Ok(vec![]) + } + (None, true) => { + bail!("file {:?} not found", path) + } + (Some(reader), _) => { + info!("Reading {}", basename); + let mut rdr = csv::ReaderBuilder::new() + .flexible(true) + .trim(csv::Trim::All) + .from_reader(reader); + let objects = rdr + .deserialize() + .map(|object| object.with_context(|| format!("Error reading {:?}", path))) + .skip_error_and_warn() + .collect(); + Ok(objects) + } + } +} +/// Read a CollectionId from a zip in a file_handler +pub fn read_collection(file_handler: &mut H, file_name: &str) -> Result> +where + for<'a> &'a mut H: FileHandler, + O: for<'de> serde::Deserialize<'de> + Id, +{ + let vec = read_objects(file_handler, file_name, true)?; + CollectionWithId::new(vec).map_err(|e| anyhow!("{}", e)) +} + +/// Read a CollectionId from a optional file in a file_handler +pub fn read_opt_collection( + file_handler: &mut H, + file_name: &str, +) -> Result> +where + for<'a> &'a mut H: FileHandler, + O: for<'de> serde::Deserialize<'de> + Id, +{ + let vec = read_objects(file_handler, file_name, false)?; + CollectionWithId::new(vec).map_err(|e| anyhow!("{}", e)) +} diff --git a/src/read_utils.rs b/src/read_utils.rs deleted file mode 100644 index 5b9937803..000000000 --- a/src/read_utils.rs +++ /dev/null @@ -1,347 +0,0 @@ -// Copyright (C) 2017 Kisio Digital and/or its affiliates. -// -// This program is free software: you can redistribute it and/or modify it -// under the terms of the GNU Affero General Public License as published by the -// Free Software Foundation, version 3. - -// This program is distributed in the hope that it will be useful, but WITHOUT -// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -// FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more -// details. - -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see -//! Some utilities for input dataset to the library. - -use crate::{ - objects::{self, Contributor}, - Result, -}; -use anyhow::{anyhow, bail, Context}; -use serde::Deserialize; -use skip_error::SkipError; -use std::path; -use std::path::{Path, PathBuf}; -use std::{collections::BTreeMap, io::Read}; -use std::{fs::File, io::Seek}; -use tracing::info; -use typed_index_collection::{CollectionWithId, Id}; - -#[derive(Deserialize, Debug)] -struct ConfigDataset { - dataset_id: String, -} - -#[derive(Deserialize, Debug)] -struct Config { - contributor: objects::Contributor, - dataset: ConfigDataset, - feed_infos: Option>, -} - -/// Read a JSON configuration file to facilitate the creation of: -/// - a Contributor -/// - a Dataset -/// - a list of key/value which will be used in 'feed_infos.txt' -/// Below is an example of this file -/// ```text -/// { -/// "contributor": { -/// "contributor_id": "contributor_id", -/// "contributor_name": "Contributor Name", -/// "contributor_license": "AGPIT", -/// "contributor_website": "http://www.datasource-website.com" -/// }, -/// "dataset": { -/// "dataset_id": "dataset-id" -/// }, -/// "feed_infos": { -/// "feed_publisher_name": "The Great Data Publisher", -/// "feed_license": "AGPIT", -/// "feed_license_url": "http://www.datasource-website.com", -/// "tartare_platform": "dev", -/// "tartare_contributor_id": "contributor_id" -/// } -/// } -/// ``` -pub fn read_config>( - config_path: Option

, -) -> Result<( - objects::Contributor, - objects::Dataset, - BTreeMap, -)> { - let contributor; - let dataset; - let mut feed_infos = BTreeMap::default(); - - if let Some(config_path) = config_path { - let config_path = config_path.as_ref(); - info!("Reading dataset and contributor from {:?}", config_path); - let json_config_file = File::open(config_path)?; - let config: Config = serde_json::from_reader(json_config_file)?; - - contributor = config.contributor; - dataset = objects::Dataset::new(config.dataset.dataset_id, contributor.id.clone()); - if let Some(config_feed_infos) = config.feed_infos { - feed_infos = config_feed_infos; - } - } else { - contributor = Contributor::default(); - dataset = objects::Dataset::default(); - } - - Ok((contributor, dataset, feed_infos)) -} - -pub(crate) trait FileHandler -where - Self: std::marker::Sized, -{ - type Reader: Read; - - fn get_file_if_exists(self, name: &str) -> Result<(Option, PathBuf)>; - - fn get_file(self, name: &str) -> Result<(Self::Reader, PathBuf)> { - let (reader, path) = self.get_file_if_exists(name)?; - Ok(( - reader.ok_or_else(|| anyhow!("file {:?} not found", path))?, - path, - )) - } - - fn source_name(&self) -> &str; -} - -/// PathFileHandler is used to read files for a directory -pub(crate) struct PathFileHandler> { - base_path: P, -} - -impl> PathFileHandler

{ - pub(crate) fn new(path: P) -> Self { - PathFileHandler { base_path: path } - } -} - -impl<'a, P: AsRef> FileHandler for &'a mut PathFileHandler

{ - type Reader = File; - fn get_file_if_exists(self, name: &str) -> Result<(Option, PathBuf)> { - let f = self.base_path.as_ref().join(name); - if f.exists() { - Ok(( - Some(File::open(&f).with_context(|| format!("Error reading {:?}", &f))?), - f, - )) - } else { - Ok((None, f)) - } - } - fn source_name(&self) -> &str { - self.base_path.as_ref().to_str().unwrap_or_else(|| { - panic!( - "the path '{:?}' should be valid UTF-8", - self.base_path.as_ref() - ) - }) - } -} - -/// ZipHandler is a wrapper around a ZipArchive -/// It provides a way to access the archive's file by their names -/// -/// Unlike ZipArchive, it gives access to a file by its name not regarding its path in the ZipArchive -/// It thus cannot be correct if there are 2 files with the same name in the archive, -/// but for transport data if will make it possible to handle a zip with a sub directory -pub(crate) struct ZipHandler { - archive: zip::ZipArchive, - archive_path: PathBuf, - index_by_name: BTreeMap, -} - -impl ZipHandler -where - R: Seek + Read, -{ - pub(crate) fn new>(r: R, path: P) -> Result { - let mut archive = zip::ZipArchive::new(r)?; - Ok(ZipHandler { - index_by_name: Self::files_by_name(&mut archive), - archive, - archive_path: path.as_ref().to_path_buf(), - }) - } - - fn files_by_name(archive: &mut zip::ZipArchive) -> BTreeMap { - (0..archive.len()) - .filter_map(|i| { - let file = archive.by_index(i).ok()?; - // we get the name of the file, not regarding its path in the ZipArchive - let real_name = Path::new(file.name()).file_name()?; - let real_name: String = real_name.to_str()?.into(); - Some((real_name, i)) - }) - .collect() - } -} - -impl<'a, R> FileHandler for &'a mut ZipHandler -where - R: Seek + Read, -{ - type Reader = zip::read::ZipFile<'a>; - fn get_file_if_exists(self, name: &str) -> Result<(Option, PathBuf)> { - let p = self.archive_path.join(name); - match self.index_by_name.get(name) { - None => Ok((None, p)), - Some(i) => Ok((Some(self.archive.by_index(*i)?), p)), - } - } - fn source_name(&self) -> &str { - self.archive_path - .to_str() - .unwrap_or_else(|| panic!("the path '{:?}' should be valid UTF-8", self.archive_path)) - } -} - -/// Read a vector of objects from a zip in a file_handler -pub(crate) fn read_objects( - file_handler: &mut H, - file_name: &str, - required_file: bool, -) -> Result> -where - for<'a> &'a mut H: FileHandler, - O: for<'de> serde::Deserialize<'de>, -{ - let (reader, path) = file_handler.get_file_if_exists(file_name)?; - let file_name = path.file_name(); - let basename = file_name.map_or(path.to_string_lossy(), |b| b.to_string_lossy()); - - match (reader, required_file) { - (None, false) => { - info!("Skipping {}", basename); - Ok(vec![]) - } - (None, true) => { - bail!("file {:?} not found", path) - } - (Some(reader), _) => { - info!("Reading {}", basename); - let mut rdr = csv::ReaderBuilder::new() - .flexible(true) - .trim(csv::Trim::All) - .from_reader(reader); - Ok(rdr - .deserialize() - .collect::>() - .with_context(|| format!("Error reading {:?}", path))?) - } - } -} - -/// Read a vector of objects from a zip in a file_handler ignoring error -pub(crate) fn read_objects_loose( - file_handler: &mut H, - file_name: &str, - required_file: bool, -) -> Result> -where - for<'a> &'a mut H: FileHandler, - O: for<'de> serde::Deserialize<'de>, -{ - let (reader, path) = file_handler.get_file_if_exists(file_name)?; - let file_name = path.file_name(); - let basename = file_name.map_or(path.to_string_lossy(), |b| b.to_string_lossy()); - - match (reader, required_file) { - (None, false) => { - info!("Skipping {}", basename); - Ok(vec![]) - } - (None, true) => { - bail!("file {:?} not found", path) - } - (Some(reader), _) => { - info!("Reading {}", basename); - let mut rdr = csv::ReaderBuilder::new() - .flexible(true) - .trim(csv::Trim::All) - .from_reader(reader); - let objects = rdr - .deserialize() - .map(|object| object.with_context(|| format!("Error reading {:?}", path))) - .skip_error_and_warn() - .collect(); - Ok(objects) - } - } -} - -/// Read a CollectionId from a zip in a file_handler -pub(crate) fn read_collection( - file_handler: &mut H, - file_name: &str, -) -> Result> -where - for<'a> &'a mut H: FileHandler, - O: for<'de> serde::Deserialize<'de> + Id, -{ - let vec = read_objects(file_handler, file_name, true)?; - CollectionWithId::new(vec).map_err(|e| anyhow!("{}", e)) -} - -pub(crate) fn read_opt_collection( - file_handler: &mut H, - file_name: &str, -) -> Result> -where - for<'a> &'a mut H: FileHandler, - O: for<'de> serde::Deserialize<'de> + Id, -{ - let vec = read_objects(file_handler, file_name, false)?; - CollectionWithId::new(vec).map_err(|e| anyhow!("{}", e)) -} - -#[cfg(test)] -mod tests { - use super::*; - use pretty_assertions::assert_eq; - use std::io::Read; - - #[test] - fn path_file_handler() { - let mut file_handler = PathFileHandler::new(PathBuf::from("tests/fixtures/file-handler")); - - let (mut hello, _) = file_handler.get_file("hello.txt").unwrap(); - let mut hello_str = String::new(); - hello.read_to_string(&mut hello_str).unwrap(); - assert_eq!("hello\n", hello_str); - - let (mut world, _) = file_handler.get_file("folder/world.txt").unwrap(); - let mut world_str = String::new(); - world.read_to_string(&mut world_str).unwrap(); - assert_eq!("world\n", world_str); - } - - #[test] - fn zip_file_handler() { - let p = "tests/fixtures/file-handler.zip"; - let reader = File::open(p).unwrap(); - let mut file_handler = ZipHandler::new(reader, p).unwrap(); - - { - let (mut hello, _) = file_handler.get_file("hello.txt").unwrap(); - let mut hello_str = String::new(); - hello.read_to_string(&mut hello_str).unwrap(); - assert_eq!("hello\n", hello_str); - } - - { - let (mut world, _) = file_handler.get_file("world.txt").unwrap(); - let mut world_str = String::new(); - world.read_to_string(&mut world_str).unwrap(); - assert_eq!("world\n", world_str); - } - } -} diff --git a/src/utils.rs b/src/utils.rs index a38df92d7..9ee949fb6 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -12,7 +12,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see -use crate::read_utils::{read_objects, FileHandler}; +use crate::{file_handler::FileHandler, parser::read_objects}; use anyhow::Context; use skip_error::skip_error_and_warn; use std::{ diff --git a/src/validity_period.rs b/src/validity_period.rs index 130b47f1a..40f400e55 100644 --- a/src/validity_period.rs +++ b/src/validity_period.rs @@ -143,9 +143,7 @@ mod tests { mod compute_dataset_validity_period { use super::super::*; use crate::{ - calendars, - model::Collections, - read_utils::{self, PathFileHandler}, + calendars, configuration::*, file_handler::PathFileHandler, model::Collections, test_utils::*, }; @@ -164,7 +162,7 @@ mod tests { create_file_with_content(path, "calendar_dates.txt", calendar_dates_content); let mut collections = Collections::default(); - let (_, mut dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (_, mut dataset, _) = read_config(None::<&str>).unwrap(); calendars::manage_calendars(&mut handler, &mut collections).unwrap(); compute_dataset_validity_period(&mut dataset, &collections.calendars).unwrap(); @@ -195,7 +193,7 @@ mod tests { create_file_with_content(path, "calendar.txt", calendars_content); let mut collections = Collections::default(); - let (_, mut dataset, _) = read_utils::read_config(None::<&str>).unwrap(); + let (_, mut dataset, _) = read_config(None::<&str>).unwrap(); calendars::manage_calendars(&mut handler, &mut collections).unwrap(); compute_dataset_validity_period(&mut dataset, &collections.calendars).unwrap(); diff --git a/tests/gtfs2ntfs.rs b/tests/gtfs2ntfs.rs index b420a376e..ea1ba1715 100644 --- a/tests/gtfs2ntfs.rs +++ b/tests/gtfs2ntfs.rs @@ -14,9 +14,9 @@ use std::collections::BTreeMap; use transit_model::{ + configuration::read_config, gtfs, ntfs, objects::{Contributor, Dataset}, - read_utils::read_config, test_utils::*, PrefixConfiguration, };