Merge pull request #838 from CanalTP/gtfs_makes_some_items_visible

[tech] make some items public

Cargo.toml

@@ -32,6 +32,8 @@ members = [
 
 [features]
 xmllint = ["proj"]
+gtfs = []
+parser = []
 # Experimental feature, use at your own risks
 mutable-model = []
 

gtfs2netexfr/src/main.rs

@@ -23,7 +23,7 @@ use tracing_subscriber::{
     layer::SubscriberExt as _,
     util::SubscriberInitExt as _,
 };
-use transit_model::{read_utils, Result};
+use transit_model::{configuration, Result};
 
 lazy_static::lazy_static! {
     pub static ref GIT_VERSION: String = transit_model::binary_full_version(env!("CARGO_PKG_VERSION"));
@@ -106,7 +106,7 @@ fn init_logger() {
 fn run(opt: Opt) -> Result<()> {
     info!("Launching gtfs2netexfr...");
 
-    let (contributor, dataset, feed_infos) = read_utils::read_config(opt.config)?;
+    let (contributor, dataset, feed_infos) = configuration::read_config(opt.config)?;
     let configuration = transit_model::gtfs::Configuration {
         contributor,
         dataset,

gtfs2ntfs/src/main.rs

@@ -23,7 +23,7 @@ use tracing_subscriber::{
     layer::SubscriberExt as _,
     util::SubscriberInitExt as _,
 };
-use transit_model::{read_utils, transfers::generates_transfers, PrefixConfiguration, Result};
+use transit_model::{configuration, transfers::generates_transfers, PrefixConfiguration, Result};
 
 lazy_static::lazy_static! {
     pub static ref GIT_VERSION: String = transit_model::binary_full_version(env!("CARGO_PKG_VERSION"));
@@ -100,7 +100,7 @@ struct Opt {
 fn run(opt: Opt) -> Result<()> {
     info!("Launching gtfs2ntfs...");
 
-    let (contributor, dataset, feed_infos) = read_utils::read_config(opt.config)?;
+    let (contributor, dataset, feed_infos) = configuration::read_config(opt.config)?;
     let mut prefix_conf = PrefixConfiguration::default();
     if let Some(data_prefix) = opt.prefix {
         prefix_conf.set_data_prefix(data_prefix);

src/calendars.rs

@@ -17,9 +17,10 @@
 //! - calendar.txt and calendar_dates.txt format are identical between the GTFS
 //!   and NTFS
 
+use crate::file_handler::FileHandler;
 use crate::model::Collections;
 use crate::objects::{self, Date, ExceptionType};
-use crate::read_utils::{read_objects, FileHandler};
+use crate::parser::read_objects;
 use crate::serde_utils::*;
 use crate::vptranslator::translate;
 use crate::Result;
@@ -166,7 +167,10 @@ where
     Ok(())
 }
 
-pub(crate) fn manage_calendars<H>(file_handler: &mut H, collections: &mut Collections) -> Result<()>
+pub(crate) fn _manage_calendars<H>(
+    file_handler: &mut H,
+    collections: &mut Collections,
+) -> Result<()>
 where
     for<'a> &'a mut H: FileHandler,
 {
@@ -188,6 +192,22 @@ where
     Ok(())
 }
 
+#[cfg(not(feature = "parser"))]
+pub(crate) fn manage_calendars<H>(file_handler: &mut H, collections: &mut Collections) -> Result<()>
+where
+    for<'a> &'a mut H: FileHandler,
+{
+    _manage_calendars(file_handler, collections)
+}
+#[cfg(feature = "parser")]
+/// Read calendar_dates.txt and calendar.txt files
+pub fn manage_calendars<H>(file_handler: &mut H, collections: &mut Collections) -> Result<()>
+where
+    for<'a> &'a mut H: FileHandler,
+{
+    _manage_calendars(file_handler, collections)
+}
+
 /// Write the calendar_dates.txt file into a Path from a list of Calendar
 pub fn write_calendar_dates(
     path: &path::Path,

src/configuration.rs

@@ -0,0 +1,91 @@
+// Copyright (C) 2017 Kisio Digital and/or its affiliates.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU Affero General Public License as published by the
+// Free Software Foundation, version 3.
+
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+// FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+// details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+//! Some utilities for input dataset to the library.
+
+use crate::{
+    objects::{self, Contributor},
+    Result,
+};
+use serde::Deserialize;
+use std::collections::BTreeMap;
+use std::fs::File;
+use std::path;
+use tracing::info;
+
+#[derive(Deserialize, Debug)]
+struct ConfigDataset {
+    dataset_id: String,
+}
+
+#[derive(Deserialize, Debug)]
+struct Config {
+    contributor: objects::Contributor,
+    dataset: ConfigDataset,
+    feed_infos: Option<BTreeMap<String, String>>,
+}
+
+/// Read a JSON configuration file to facilitate the creation of:
+/// - a Contributor
+/// - a Dataset
+/// - a list of key/value which will be used in 'feed_infos.txt'
+/// Below is an example of this file
+/// ```text
+/// {
+///     "contributor": {
+///         "contributor_id": "contributor_id",
+///         "contributor_name": "Contributor Name",
+///         "contributor_license": "AGPIT",
+///         "contributor_website": "http://www.datasource-website.com"
+///     },
+///     "dataset": {
+///         "dataset_id": "dataset-id"
+///     },
+///     "feed_infos": {
+///         "feed_publisher_name": "The Great Data Publisher",
+///         "feed_license": "AGPIT",
+///         "feed_license_url": "http://www.datasource-website.com",
+///         "tartare_platform": "dev",
+///         "tartare_contributor_id": "contributor_id"
+///     }
+/// }
+/// ```
+pub fn read_config<P: AsRef<path::Path>>(
+    config_path: Option<P>,
+) -> Result<(
+    objects::Contributor,
+    objects::Dataset,
+    BTreeMap<String, String>,
+)> {
+    let contributor;
+    let dataset;
+    let mut feed_infos = BTreeMap::default();
+
+    if let Some(config_path) = config_path {
+        let config_path = config_path.as_ref();
+        info!("Reading dataset and contributor from {:?}", config_path);
+        let json_config_file = File::open(config_path)?;
+        let config: Config = serde_json::from_reader(json_config_file)?;
+
+        contributor = config.contributor;
+        dataset = objects::Dataset::new(config.dataset.dataset_id, contributor.id.clone());
+        if let Some(config_feed_infos) = config.feed_infos {
+            feed_infos = config_feed_infos;
+        }
+    } else {
+        contributor = Contributor::default();
+        dataset = objects::Dataset::default();
+    }
+
+    Ok((contributor, dataset, feed_infos))
+}

src/file_handler/mod.rs

@@ -0,0 +1,169 @@
+//! Provides an easy way to access directory or flat zip archive
+use crate::Result;
+use anyhow::{anyhow, Context};
+use std::{
+    collections::BTreeMap,
+    fs::File,
+    io::{Read, Seek},
+    path::{Path, PathBuf},
+};
+
+/// Allows files in a directory or ZipArchive to be read either
+pub trait FileHandler
+where
+    Self: std::marker::Sized,
+{
+    /// Reader
+    type Reader: Read;
+
+    /// Return a file if exist
+    fn get_file_if_exists(self, name: &str) -> Result<(Option<Self::Reader>, PathBuf)>;
+
+    /// Return a file or an error if not exist
+    fn get_file(self, name: &str) -> Result<(Self::Reader, PathBuf)> {
+        let (reader, path) = self.get_file_if_exists(name)?;
+        Ok((
+            reader.ok_or_else(|| anyhow!("file {:?} not found", path))?,
+            path,
+        ))
+    }
+
+    /// Allows to have nicer error messages
+    fn source_name(&self) -> &str;
+}
+
+/// PathFileHandler is used to read files for a directory
+pub struct PathFileHandler<P: AsRef<Path>> {
+    base_path: P,
+}
+
+impl<P: AsRef<Path>> PathFileHandler<P> {
+    /// Constructs a new PathFileHandler
+    pub fn new(path: P) -> Self {
+        PathFileHandler { base_path: path }
+    }
+}
+
+impl<'a, P: AsRef<Path>> FileHandler for &'a mut PathFileHandler<P> {
+    type Reader = File;
+    fn get_file_if_exists(self, name: &str) -> Result<(Option<Self::Reader>, PathBuf)> {
+        let f = self.base_path.as_ref().join(name);
+        if f.exists() {
+            Ok((
+                Some(File::open(&f).with_context(|| format!("Error reading {:?}", &f))?),
+                f,
+            ))
+        } else {
+            Ok((None, f))
+        }
+    }
+    fn source_name(&self) -> &str {
+        self.base_path.as_ref().to_str().unwrap_or_else(|| {
+            panic!(
+                "the path '{:?}' should be valid UTF-8",
+                self.base_path.as_ref()
+            )
+        })
+    }
+}
+
+/// ZipHandler is a wrapper around a ZipArchive
+/// It provides a way to access the archive's file by their names
+///
+/// Unlike ZipArchive, it gives access to a file by its name not regarding its path in the ZipArchive
+/// It thus cannot be correct if there are 2 files with the same name in the archive,
+/// but for transport data if will make it possible to handle a zip with a sub directory
+pub struct ZipHandler<R: Seek + Read> {
+    archive: zip::ZipArchive<R>,
+    archive_path: PathBuf,
+    index_by_name: BTreeMap<String, usize>,
+}
+
+/// ZipHandler is used to read files from an archive
+impl<R> ZipHandler<R>
+where
+    R: Seek + Read,
+{
+    /// Constructs a new ZipHandler
+    pub fn new<P: AsRef<Path>>(r: R, path: P) -> Result<Self> {
+        let mut archive = zip::ZipArchive::new(r)?;
+        Ok(ZipHandler {
+            index_by_name: Self::files_by_name(&mut archive),
+            archive,
+            archive_path: path.as_ref().to_path_buf(),
+        })
+    }
+    fn files_by_name(archive: &mut zip::ZipArchive<R>) -> BTreeMap<String, usize> {
+        (0..archive.len())
+            .filter_map(|i| {
+                let file = archive.by_index(i).ok()?;
+                // we get the name of the file, not regarding its path in the ZipArchive
+                let real_name = Path::new(file.name()).file_name()?;
+                let real_name: String = real_name.to_str()?.into();
+                Some((real_name, i))
+            })
+            .collect()
+    }
+}
+
+impl<'a, R> FileHandler for &'a mut ZipHandler<R>
+where
+    R: Seek + Read,
+{
+    type Reader = zip::read::ZipFile<'a>;
+    fn get_file_if_exists(self, name: &str) -> Result<(Option<Self::Reader>, PathBuf)> {
+        let p = self.archive_path.join(name);
+        match self.index_by_name.get(name) {
+            None => Ok((None, p)),
+            Some(i) => Ok((Some(self.archive.by_index(*i)?), p)),
+        }
+    }
+    fn source_name(&self) -> &str {
+        self.archive_path
+            .to_str()
+            .unwrap_or_else(|| panic!("the path '{:?}' should be valid UTF-8", self.archive_path))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use pretty_assertions::assert_eq;
+    use std::io::Read;
+
+    #[test]
+    fn path_file_handler() {
+        let mut file_handler = PathFileHandler::new(PathBuf::from("tests/fixtures/file-handler"));
+
+        let (mut hello, _) = file_handler.get_file("hello.txt").unwrap();
+        let mut hello_str = String::new();
+        hello.read_to_string(&mut hello_str).unwrap();
+        assert_eq!("hello\n", hello_str);
+
+        let (mut world, _) = file_handler.get_file("folder/world.txt").unwrap();
+        let mut world_str = String::new();
+        world.read_to_string(&mut world_str).unwrap();
+        assert_eq!("world\n", world_str);
+    }
+
+    #[test]
+    fn zip_file_handler() {
+        let p = "tests/fixtures/file-handler.zip";
+        let reader = File::open(p).unwrap();
+        let mut file_handler = ZipHandler::new(reader, p).unwrap();
+
+        {
+            let (mut hello, _) = file_handler.get_file("hello.txt").unwrap();
+            let mut hello_str = String::new();
+            hello.read_to_string(&mut hello_str).unwrap();
+            assert_eq!("hello\n", hello_str);
+        }
+
+        {
+            let (mut world, _) = file_handler.get_file("world.txt").unwrap();
+            let mut world_str = String::new();
+            world.read_to_string(&mut world_str).unwrap();
+            assert_eq!("world\n", world_str);
+        }
+    }
+}