Refresh the GTFS bundle every time the program starts.

Cargo.toml

@@ -9,8 +9,10 @@ chrono = "0.4"
 csv = "1.4"
 gtfs-structures = "0.47"
 log = "0.4"
+reqwest = { version = "0.11", features = ["blocking"] }
 sdl3 = {version = "0.17", features = ["ttf"]}
 serde = { version = "1.0", features = ["derive"]}
+time-format = "1.2"
 yaml_serde = "0.10"
 zip = "8.3"
 

resources/gtfs.rs

@@ -1,132 +0,0 @@
-use gtfs_structures::{Agency, Calendar, CalendarDate, Route, Stop};
-use serde::{de::DeserializeOwned};
-use std::{collections::HashMap, fs::File};
-use zip::ZipArchive;
-
-// The main GTFS struct. This is similar to (but not exactly) gtfs-structures::Gtfs because we don't need everything
-#[derive(Debug)]
-pub struct Gtfs {
-    /// All agencies. They can not be read by `agency_id`, as it is not a required field
-    pub agencies: Vec<Agency>,
-    /// All Calendar by `service_id`
-    pub calendar: HashMap<String, Calendar>,
-    /// All calendar dates grouped by service_id
-    pub calendar_dates: HashMap<String, Vec<CalendarDate>>,
-    /// All routes by `route_id`
-    pub routes: HashMap<String, Route>,
-    /// All stop by `stop_id`.
-    pub stops: HashMap<String, Stop>,
-}
-
-// Utility function to load all records in a dataset
-fn load_all<V>(_: &V) -> bool { true } 
-
-// Loads a vector of the selected type
-fn load_vector<T: serde::de::DeserializeOwned>(
-    destination: &mut Vec<T>,
-    zip_reader: &mut ZipArchive<File>,
-    table_name: &str,
-) {
-    let file_reader = zip_reader.by_name(table_name).unwrap();
-    let mut rdr = csv::Reader::from_reader(file_reader);
-
-    for row in rdr.deserialize() {
-        let record: T = row.unwrap();
-        destination.push(record);
-    }
-}
-
-// Loads a HashMap of the selected type, using the provided index function as the key
-fn load_map<'a, V: DeserializeOwned>(
-    destination: &mut HashMap<String, V>,
-    zip_reader: &mut ZipArchive<File>,
-    table_name: &str,
-    index: fn(&V) -> String,
-    accept: fn(&V) -> bool,
-) {
-    let file_reader = zip_reader.by_name(table_name).unwrap();
-    let mut rdr = csv::Reader::from_reader(file_reader);
-
-    for row in rdr.deserialize() {
-        let record: V = row.unwrap();
-        if accept(&record) {
-            let idx: String = index(&record);
-            destination.insert(idx, record);
-        }
-    }
-}
-
-// Loads a HashMap of a vector of the selected type, using the provided index function as the key
-// And a predicate as a filter
-fn load_vector_map<'a, V: DeserializeOwned + Clone>(
-    destination: &mut HashMap<String, Vec<V>>,
-    zip_reader: &mut ZipArchive<File>,
-    table_name: &str,
-    index: fn(&V) -> String,
-    accept: fn(&V) -> bool,
-) {
-    let file_reader = zip_reader.by_name(table_name).unwrap();
-    let mut rdr = csv::Reader::from_reader(file_reader);
-
-    for row in rdr.deserialize() {
-        let record: V = row.unwrap();
-        if accept(&record) {
-            let idx: String = index(&record);
-            destination.entry(idx).or_insert_with(Vec::new).push(record);
-        }
-    }
-}
-
-pub fn init(src_file: &str, routes: Vec<String>) -> Gtfs {
-    // Open zip file
-    let mut zip_reader = zip::ZipArchive::new(File::open(src_file).unwrap()).unwrap();
-
-    let mut gtfs: Gtfs = Gtfs {
-        agencies: Vec::new(),
-        calendar: HashMap::new(),
-        calendar_dates: HashMap::new(),
-        routes: HashMap::new(),
-        stops: HashMap::new(),
-    };
-
-    // Agencies
-    load_vector(&mut gtfs.agencies, &mut zip_reader, "agency.txt");
-
-    // Calendars
-    load_map(
-        &mut gtfs.calendar,
-        &mut zip_reader,
-        "calendar.txt",
-        |c: &Calendar| String::from(&c.id),
-        load_all
-    );
-
-    // Calendar Dates
-    load_vector_map(
-        &mut gtfs.calendar_dates,
-        &mut zip_reader,
-        "calendar_dates.txt",
-        |d: &CalendarDate| String::from(&d.service_id),
-        load_all
-    );
-
-    // Stops
-    load_map(&mut gtfs.stops, 
-        &mut zip_reader, 
-        "stops.txt", 
-        |s: &Stop| {String::from(&s.id)},
-        load_all
-    );
-
-    // Routes
-    let accept_filter: fn(&Route) -> bool = (| rs: Vec<String>, r: &Route | {rs.contains(&r.short_name.unwrap())}).curry(routes);
-    load_map(
-        &mut gtfs.routes,
-        &mut zip_reader,
-        "routes.txt",
-        |r: &Route| String::from(&r.id),
-        accept_filter
-    );
-
-    return gtfs;
-}

src/gtfs/mod.rs

@@ -1,15 +1,13 @@
 mod arrival;
 mod loader;
 mod utils;
+mod refresher;
 pub mod structs;
 use chrono::{DateTime, Local, Timelike};
-use log::{debug};
+use log::{debug, trace};
-use std::{
+use std::{collections::{HashMap, HashSet}, fs::File };
-    collections::{HashMap, HashSet}, fs::File, io::Error
-};
 use gtfs_structures::{Exception, RawTrip};
-
+use crate::gtfs::{loader::load_gtfs, structs::{Arrival, Gtfs, Preferences, Error}};
-use crate::gtfs::{loader::load_gtfs, structs::{Arrival, Gtfs, Preferences}};
 
 
 impl Gtfs {
@@ -75,7 +73,7 @@ impl Gtfs {
                         trip: &trip,
                         departure_time: stop_timestamp.into()
                     };
-                    debug!("{:#?}: Arrival to {:#?} for trip ID {:#?}.", 
+                    trace!("{:#?}: Arrival to {:#?} for trip ID {:#?}.", 
                             format!("{:02}:{:02}", (arrival.departure_time/3600) as u32, ((arrival.departure_time / 60) % 60) as u32),
                             arrival.trip.trip_headsign.as_ref().unwrap(),
                             arrival.trip.id);
@@ -91,9 +89,12 @@ impl Gtfs {
         
 
     /// Load a GTFS structure from a zip file
-    pub fn load(src_file: &str, prefs: &Preferences) -> Result<Gtfs, Error> {
+    pub fn load(prefs: &Preferences) -> Result<Gtfs, Error> {
+
+        _ = refresher::refresh(prefs);
+
         // Open zip file
-        let zip_file = File::open(src_file)?;
+        let zip_file = File::open(prefs.gtfs_file_path()?)?;
         let mut zip_reader = zip::ZipArchive::new(zip_file)?;
 
         let mut gtfs: Gtfs = Gtfs {

src/gtfs/refresher.rs

@@ -0,0 +1,61 @@
+use log::{debug, error, info};
+use time_format::format_common_utc;
+use std::{fs::{self}, time::{Duration, SystemTime, UNIX_EPOCH}};
+use crate::gtfs::structs::{Error, Preferences};
+use reqwest::{StatusCode, blocking::Client};
+
+
+pub(crate) fn refresh(prefs: &Preferences) -> Result<(), Error> {
+
+    let mut modified_timestamp: SystemTime = SystemTime::UNIX_EPOCH;
+    let gtfs_file_name = prefs.gtfs_file_path()?;
+
+    // Obtain the GTFS zip's creation time.
+    let metadata = fs::metadata(&gtfs_file_name);
+    if metadata.is_ok() {
+        modified_timestamp = metadata?.modified()?;
+    }
+
+    let modified_seconds = modified_timestamp.duration_since(UNIX_EPOCH).unwrap().as_secs() as i64;
+    let modified_header =  format_common_utc(modified_seconds, time_format::DateFormat::HTTP).unwrap();
+
+    debug!("Using if-modified-since: {}", modified_header);
+
+    // request updates from the server if-modified-since last time
+    let client = Client::builder()
+        .timeout(Duration::from_secs(30))
+        .connect_timeout(Duration::from_secs(5))
+        .build()?;
+    
+    let mut response = client
+        .get(&prefs.gtfs_url)
+        .header(reqwest::header::IF_MODIFIED_SINCE, modified_header)
+        .send()
+        .unwrap();
+
+    if response.status() == StatusCode::NOT_MODIFIED {
+        debug!("GTFS data is still up-to-date");
+        return Ok(());
+    }
+
+    if response.status() == StatusCode::OK {
+        info!("Refreshing GTFS data");
+        // Stream the response data into a temp file and swap them
+
+        let tmp_full_file_name = format!("{}.new", &gtfs_file_name);
+        let mut tmp_file = fs::File::create(&tmp_full_file_name)?;
+
+        _ = response.copy_to(&mut tmp_file)?;
+        drop(tmp_file);
+
+        if fs::exists(&gtfs_file_name)? {
+            _ = fs::remove_file(&gtfs_file_name);
+        }
+        _ = fs::rename(&tmp_full_file_name, &gtfs_file_name);
+        info!("GTFS data refreshed");
+        return Ok(());
+    }
+
+    let errmsg = format!("GET on {} returned result code {}", prefs.gtfs_url, response.status());
+    return Err(Error { message: errmsg });
+}    

src/gtfs/structs.rs

@@ -1,6 +1,33 @@
 use std::collections::{HashMap, HashSet};
 use gtfs_structures::{Agency, Calendar, CalendarDate, RawStopTime, RawTrip, Route, Stop};
+use log::error;
 use serde::{Serialize, Deserialize};
+use zip::result::ZipError;
+
+
+#[derive(Debug)]
+pub struct Error {
+    pub(crate) message: String,
+}
+
+impl From<std::io::Error> for Error {
+    fn from(value: std::io::Error) -> Self {
+        return Error { message: value.to_string() }
+    }
+}
+
+impl From<reqwest::Error> for Error {
+    fn from(value: reqwest::Error) -> Self {
+        return Error { message: value.to_string() }
+    }
+}
+
+impl From<ZipError> for Error {
+    fn from(value: ZipError) -> Self {
+        return Error { message: value.to_string() }
+    }
+}
+
 
 // This is to store the preferences for the GTFS(-R) side of the code.
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
@@ -27,6 +54,23 @@ pub struct Preferences {
     pub refresh_seconds: u64,
 }
 
+// Utility functions
+impl Preferences {
+    pub fn gtfs_file_name(&self) -> Result<String, Error> {
+        let file_name_part = self.gtfs_url.split("/").last();
+        if file_name_part.is_none() {
+            error!("The config for gtfs-url is not a valid URL: {}", self.gtfs_url);
+            return Err(Error {message: String::from("Failed to refresh GTFS data")});
+        } 
+        return Ok(String::from(file_name_part.unwrap()));
+    }
+
+    pub fn gtfs_file_path(&self) -> Result<String, Error> {
+        return Ok(format!("{}/{}", self.data_folder, self.gtfs_file_name()?));
+    }
+
+}
+
 
 // The main GTFS struct. This is similar to (but not exactly) gtfs-structures::Gtfs because we don't need everything
 #[derive(Debug)]

src/main.rs

@@ -3,11 +3,10 @@ mod renderer;
 mod config;
 use std::{ops::Add, process, thread::Builder, time::SystemTime};
 use chrono::{DateTime, Duration, Local, NaiveTime};
-use log::{Metadata, Record, error, info};
+use log::{Level, Metadata, Record, error, info};
 use sdl3::event::Event;
 use crate::{config::load_config, gtfs::structs::{Arrival, Gtfs}, renderer::structs::{DisplayData, DisplayEntry, Screen}};
 
-const SRC_FILE: &str = "/home/nahuel/Downloads/GTFS_Realtime.zip";
 const NUM_ARRIVALS: usize = 4;
 
 // Custom Event to signal data refresh
@@ -59,8 +58,8 @@ fn main() {
     static MY_LOGGER: MyLogger = MyLogger;
     struct MyLogger;
     impl log::Log for MyLogger {
-        fn enabled(&self, _metadata: &Metadata) -> bool {
+        fn enabled(&self, metadata: &Metadata) -> bool {
-            true
+            return metadata.level() < Level::Trace;
         }
 
         fn log(&self, record: &Record) {
@@ -92,10 +91,10 @@ fn main() {
 
     // Init GTFS static info
     info!("Loading GTFS data...");
-    let res = Gtfs::load(SRC_FILE, &gtfs_prefs);
+    let res = Gtfs::load(&gtfs_prefs);
 
     if let Err(e) = res {
-        error!("Error loading GTFS data: {}", e);
+        error!("Error loading GTFS data: {:#?}", e);
         process::exit(-1);
     } 
     let gtfs = res.unwrap();