From dc368ca81195cd77ad4ea51f0e2b2f41d47df1c7 Mon Sep 17 00:00:00 2001 From: Nahuel Lofeudo Date: Mon, 25 May 2026 07:45:35 +0100 Subject: [PATCH] Refresh the GTFS bundle every time the program starts. --- Cargo.toml | 2 + resources/gtfs.rs | 132 ------------------------------------------ src/gtfs/mod.rs | 19 +++--- src/gtfs/refresher.rs | 61 +++++++++++++++++++ src/gtfs/structs.rs | 44 ++++++++++++++ src/main.rs | 11 ++-- 6 files changed, 122 insertions(+), 147 deletions(-) delete mode 100644 resources/gtfs.rs create mode 100644 src/gtfs/refresher.rs diff --git a/Cargo.toml b/Cargo.toml index 51ef983..89edd9b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,8 +9,10 @@ chrono = "0.4" csv = "1.4" gtfs-structures = "0.47" log = "0.4" +reqwest = { version = "0.11", features = ["blocking"] } sdl3 = {version = "0.17", features = ["ttf"]} serde = { version = "1.0", features = ["derive"]} +time-format = "1.2" yaml_serde = "0.10" zip = "8.3" diff --git a/resources/gtfs.rs b/resources/gtfs.rs deleted file mode 100644 index 6d4db39..0000000 --- a/resources/gtfs.rs +++ /dev/null @@ -1,132 +0,0 @@ -use gtfs_structures::{Agency, Calendar, CalendarDate, Route, Stop}; -use serde::{de::DeserializeOwned}; -use std::{collections::HashMap, fs::File}; -use zip::ZipArchive; - -// The main GTFS struct. This is similar to (but not exactly) gtfs-structures::Gtfs because we don't need everything -#[derive(Debug)] -pub struct Gtfs { - /// All agencies. They can not be read by `agency_id`, as it is not a required field - pub agencies: Vec, - /// All Calendar by `service_id` - pub calendar: HashMap, - /// All calendar dates grouped by service_id - pub calendar_dates: HashMap>, - /// All routes by `route_id` - pub routes: HashMap, - /// All stop by `stop_id`. - pub stops: HashMap, -} - -// Utility function to load all records in a dataset -fn load_all(_: &V) -> bool { true } - -// Loads a vector of the selected type -fn load_vector( - destination: &mut Vec, - zip_reader: &mut ZipArchive, - table_name: &str, -) { - let file_reader = zip_reader.by_name(table_name).unwrap(); - let mut rdr = csv::Reader::from_reader(file_reader); - - for row in rdr.deserialize() { - let record: T = row.unwrap(); - destination.push(record); - } -} - -// Loads a HashMap of the selected type, using the provided index function as the key -fn load_map<'a, V: DeserializeOwned>( - destination: &mut HashMap, - zip_reader: &mut ZipArchive, - table_name: &str, - index: fn(&V) -> String, - accept: fn(&V) -> bool, -) { - let file_reader = zip_reader.by_name(table_name).unwrap(); - let mut rdr = csv::Reader::from_reader(file_reader); - - for row in rdr.deserialize() { - let record: V = row.unwrap(); - if accept(&record) { - let idx: String = index(&record); - destination.insert(idx, record); - } - } -} - -// Loads a HashMap of a vector of the selected type, using the provided index function as the key -// And a predicate as a filter -fn load_vector_map<'a, V: DeserializeOwned + Clone>( - destination: &mut HashMap>, - zip_reader: &mut ZipArchive, - table_name: &str, - index: fn(&V) -> String, - accept: fn(&V) -> bool, -) { - let file_reader = zip_reader.by_name(table_name).unwrap(); - let mut rdr = csv::Reader::from_reader(file_reader); - - for row in rdr.deserialize() { - let record: V = row.unwrap(); - if accept(&record) { - let idx: String = index(&record); - destination.entry(idx).or_insert_with(Vec::new).push(record); - } - } -} - -pub fn init(src_file: &str, routes: Vec) -> Gtfs { - // Open zip file - let mut zip_reader = zip::ZipArchive::new(File::open(src_file).unwrap()).unwrap(); - - let mut gtfs: Gtfs = Gtfs { - agencies: Vec::new(), - calendar: HashMap::new(), - calendar_dates: HashMap::new(), - routes: HashMap::new(), - stops: HashMap::new(), - }; - - // Agencies - load_vector(&mut gtfs.agencies, &mut zip_reader, "agency.txt"); - - // Calendars - load_map( - &mut gtfs.calendar, - &mut zip_reader, - "calendar.txt", - |c: &Calendar| String::from(&c.id), - load_all - ); - - // Calendar Dates - load_vector_map( - &mut gtfs.calendar_dates, - &mut zip_reader, - "calendar_dates.txt", - |d: &CalendarDate| String::from(&d.service_id), - load_all - ); - - // Stops - load_map(&mut gtfs.stops, - &mut zip_reader, - "stops.txt", - |s: &Stop| {String::from(&s.id)}, - load_all - ); - - // Routes - let accept_filter: fn(&Route) -> bool = (| rs: Vec, r: &Route | {rs.contains(&r.short_name.unwrap())}).curry(routes); - load_map( - &mut gtfs.routes, - &mut zip_reader, - "routes.txt", - |r: &Route| String::from(&r.id), - accept_filter - ); - - return gtfs; -} diff --git a/src/gtfs/mod.rs b/src/gtfs/mod.rs index 05bbce7..1d14b94 100644 --- a/src/gtfs/mod.rs +++ b/src/gtfs/mod.rs @@ -1,15 +1,13 @@ mod arrival; mod loader; mod utils; +mod refresher; pub mod structs; use chrono::{DateTime, Local, Timelike}; -use log::{debug}; -use std::{ - collections::{HashMap, HashSet}, fs::File, io::Error -}; +use log::{debug, trace}; +use std::{collections::{HashMap, HashSet}, fs::File }; use gtfs_structures::{Exception, RawTrip}; - -use crate::gtfs::{loader::load_gtfs, structs::{Arrival, Gtfs, Preferences}}; +use crate::gtfs::{loader::load_gtfs, structs::{Arrival, Gtfs, Preferences, Error}}; impl Gtfs { @@ -75,7 +73,7 @@ impl Gtfs { trip: &trip, departure_time: stop_timestamp.into() }; - debug!("{:#?}: Arrival to {:#?} for trip ID {:#?}.", + trace!("{:#?}: Arrival to {:#?} for trip ID {:#?}.", format!("{:02}:{:02}", (arrival.departure_time/3600) as u32, ((arrival.departure_time / 60) % 60) as u32), arrival.trip.trip_headsign.as_ref().unwrap(), arrival.trip.id); @@ -91,9 +89,12 @@ impl Gtfs { /// Load a GTFS structure from a zip file - pub fn load(src_file: &str, prefs: &Preferences) -> Result { + pub fn load(prefs: &Preferences) -> Result { + + _ = refresher::refresh(prefs); + // Open zip file - let zip_file = File::open(src_file)?; + let zip_file = File::open(prefs.gtfs_file_path()?)?; let mut zip_reader = zip::ZipArchive::new(zip_file)?; let mut gtfs: Gtfs = Gtfs { diff --git a/src/gtfs/refresher.rs b/src/gtfs/refresher.rs new file mode 100644 index 0000000..3e45d23 --- /dev/null +++ b/src/gtfs/refresher.rs @@ -0,0 +1,61 @@ +use log::{debug, error, info}; +use time_format::format_common_utc; +use std::{fs::{self}, time::{Duration, SystemTime, UNIX_EPOCH}}; +use crate::gtfs::structs::{Error, Preferences}; +use reqwest::{StatusCode, blocking::Client}; + + +pub(crate) fn refresh(prefs: &Preferences) -> Result<(), Error> { + + let mut modified_timestamp: SystemTime = SystemTime::UNIX_EPOCH; + let gtfs_file_name = prefs.gtfs_file_path()?; + + // Obtain the GTFS zip's creation time. + let metadata = fs::metadata(>fs_file_name); + if metadata.is_ok() { + modified_timestamp = metadata?.modified()?; + } + + let modified_seconds = modified_timestamp.duration_since(UNIX_EPOCH).unwrap().as_secs() as i64; + let modified_header = format_common_utc(modified_seconds, time_format::DateFormat::HTTP).unwrap(); + + debug!("Using if-modified-since: {}", modified_header); + + // request updates from the server if-modified-since last time + let client = Client::builder() + .timeout(Duration::from_secs(30)) + .connect_timeout(Duration::from_secs(5)) + .build()?; + + let mut response = client + .get(&prefs.gtfs_url) + .header(reqwest::header::IF_MODIFIED_SINCE, modified_header) + .send() + .unwrap(); + + if response.status() == StatusCode::NOT_MODIFIED { + debug!("GTFS data is still up-to-date"); + return Ok(()); + } + + if response.status() == StatusCode::OK { + info!("Refreshing GTFS data"); + // Stream the response data into a temp file and swap them + + let tmp_full_file_name = format!("{}.new", >fs_file_name); + let mut tmp_file = fs::File::create(&tmp_full_file_name)?; + + _ = response.copy_to(&mut tmp_file)?; + drop(tmp_file); + + if fs::exists(>fs_file_name)? { + _ = fs::remove_file(>fs_file_name); + } + _ = fs::rename(&tmp_full_file_name, >fs_file_name); + info!("GTFS data refreshed"); + return Ok(()); + } + + let errmsg = format!("GET on {} returned result code {}", prefs.gtfs_url, response.status()); + return Err(Error { message: errmsg }); +} diff --git a/src/gtfs/structs.rs b/src/gtfs/structs.rs index f1bb8f9..5db7ed3 100644 --- a/src/gtfs/structs.rs +++ b/src/gtfs/structs.rs @@ -1,6 +1,33 @@ use std::collections::{HashMap, HashSet}; use gtfs_structures::{Agency, Calendar, CalendarDate, RawStopTime, RawTrip, Route, Stop}; +use log::error; use serde::{Serialize, Deserialize}; +use zip::result::ZipError; + + +#[derive(Debug)] +pub struct Error { + pub(crate) message: String, +} + +impl From for Error { + fn from(value: std::io::Error) -> Self { + return Error { message: value.to_string() } + } +} + +impl From for Error { + fn from(value: reqwest::Error) -> Self { + return Error { message: value.to_string() } + } +} + +impl From for Error { + fn from(value: ZipError) -> Self { + return Error { message: value.to_string() } + } +} + // This is to store the preferences for the GTFS(-R) side of the code. #[derive(Debug, PartialEq, Serialize, Deserialize)] @@ -27,6 +54,23 @@ pub struct Preferences { pub refresh_seconds: u64, } +// Utility functions +impl Preferences { + pub fn gtfs_file_name(&self) -> Result { + let file_name_part = self.gtfs_url.split("/").last(); + if file_name_part.is_none() { + error!("The config for gtfs-url is not a valid URL: {}", self.gtfs_url); + return Err(Error {message: String::from("Failed to refresh GTFS data")}); + } + return Ok(String::from(file_name_part.unwrap())); + } + + pub fn gtfs_file_path(&self) -> Result { + return Ok(format!("{}/{}", self.data_folder, self.gtfs_file_name()?)); + } + +} + // The main GTFS struct. This is similar to (but not exactly) gtfs-structures::Gtfs because we don't need everything #[derive(Debug)] diff --git a/src/main.rs b/src/main.rs index f0c374d..8423616 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,11 +3,10 @@ mod renderer; mod config; use std::{ops::Add, process, thread::Builder, time::SystemTime}; use chrono::{DateTime, Duration, Local, NaiveTime}; -use log::{Metadata, Record, error, info}; +use log::{Level, Metadata, Record, error, info}; use sdl3::event::Event; use crate::{config::load_config, gtfs::structs::{Arrival, Gtfs}, renderer::structs::{DisplayData, DisplayEntry, Screen}}; -const SRC_FILE: &str = "/home/nahuel/Downloads/GTFS_Realtime.zip"; const NUM_ARRIVALS: usize = 4; // Custom Event to signal data refresh @@ -59,8 +58,8 @@ fn main() { static MY_LOGGER: MyLogger = MyLogger; struct MyLogger; impl log::Log for MyLogger { - fn enabled(&self, _metadata: &Metadata) -> bool { - true + fn enabled(&self, metadata: &Metadata) -> bool { + return metadata.level() < Level::Trace; } fn log(&self, record: &Record) { @@ -92,10 +91,10 @@ fn main() { // Init GTFS static info info!("Loading GTFS data..."); - let res = Gtfs::load(SRC_FILE, >fs_prefs); + let res = Gtfs::load(>fs_prefs); if let Err(e) = res { - error!("Error loading GTFS data: {}", e); + error!("Error loading GTFS data: {:#?}", e); process::exit(-1); } let gtfs = res.unwrap();