nahuel
/
dublinbus-display
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: nahuel:main
Branches Tags
nahuel:main
nahuel:gtfs-r
..
compare: nahuel:gtfs-r
Branches Tags
nahuel:main
nahuel:gtfs-r

No commits in common. "main" and "gtfs-r" have entirely different histories.
main ... gtfs-r

12 changed files with 332 additions and 256 deletions
Show Stats Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,6 +1,4 @@
__pycache__/*
.vscode/*
Enclosure/*.stl
*test*
*.svg
*.FCStd1

48
.ipynb_checkpoints/Playground-checkpoint.ipynb Normal file
View File

@ -0,0 +1,48 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"id": "1f552469",
"metadata": {},
"outputs": [],
"source": [
"import gtfs_kit as gk\n",
"import pandas as pd\n",
"import datetime\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "292cc196",
"metadata": {},
"outputs": [],
"source": [
"feed=gk.read_feed('google_transit_combined.zip', dist_units='km')\n",
"feed"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.7"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

16
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,16 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Python: GTFS Client",
"type": "python",
"request": "launch",
"program": "gtfs_client.py",
"console": "integratedTerminal",
"justMyCode": true
}
]
}

BIN
Enclosure/display-enclosure.FCStd
View File

Binary file not shown.

9
README.md
View File

@ -47,16 +47,15 @@ If your display's resolution is not 1920x720, you will also need to change the c
* libspatialindex-c6
* yaml
```shell
$ sudo apt install python3-iso8601 python3-zeep libsdl2-ttf-2.0-0 python3-numpy python3-pandas python3-fiona python3-pyproj libspatialindex-c6 python3-yaml python3-urllib3
```
$ sudo apt install python3-iso8601 python3-zeep libsdl2-ttf-2.0-0 python3-numpy python3-pandas python3-fiona python3-pyproj libspatialindex-c6 python3-yaml
```
* pygame 2
* GTFS-Kit
```shell
$ sudo apt install python3-pip
$ sudo pip3 install pygame gtfs_kit schedule --break-system-packages
```
$ sudo pip3 install pygame gtfs_kit
```

4
arrival_times.py
View File

@ -1,6 +1,6 @@
import datetime
class ArrivalTime:
class ArrivalTime():
""" Represents the arrival times of buses at one of the configured stops """
def __init__(self, stop_id: str, route_id: str, destination: str, due_in_seconds: int, is_added: bool = False) -> None:
@ -14,7 +14,7 @@ class ArrivalTime:
def due_in_minutes(self) -> int:
return int(self.due_in_seconds / 60)
def is_due(self) -> bool:
def isDue(self) -> bool:
return self.due_in_minutes < 1
def due_in_str(self) -> str:

29
config.py
View File

@ -4,45 +4,40 @@ class Config:
def __init__(self):
# Load the config file
with open("config.yaml") as f:
self.config = yaml.safe_load(f.read())
self.__config = yaml.safe_load(f.read())
# Preload some dictionaries to simplify lookups
self.walk_time_by_stop = {}
for s in self.config.get("stops", []):
self.walk_time_by_stop[str(s["stop_id"])] = s["walk_time"]
# Pre-load some dictionaries to simplify lookups
self.__walk_time_by_stop = {}
for s in self.__config.get("stops", []):
self.__walk_time_by_stop[s["stop_id"]] = s["walk_time"]
@property
def gtfs_feed_url(self) -> str:
return self.config.get("gtfs-feed-url")
return self.__config.get("gtfs-feed-url")
@property
def gtfs_api_url(self) -> str:
return self.config.get("gtfs-r-api-url")
return self.__config.get("gtfs-r-api-url")
@property
def gtfs_api_key(self) -> str:
return self.config.get("gtfs-r-api_key")
return self.__config.get("gtfs-r-api_key")
@property
def update_interval_seconds(self) -> int:
return self.config.get("update-interval-seconds")
@property
def font_file(self) -> str:
return self.config.get("font-file")
return self.__config.get("update-interval-seconds")
@property
def stop_codes(self) -> list[str]:
return [str(s["stop_id"]) for s in self.config.get("stops")]
return [str(s["stop_id"]) for s in self.__config.get("stops")]
def minutes_to_stop(self, stop_id) -> int:
minutes = self.walk_time_by_stop.get(stop_id, 0)
return minutes
return self.__walk_time_by_stop.get(stop_id, 0)
def routes_for_stops(self) -> map:
result = {}
for s in self.config.get("stops"):
for s in self.__config.get("stops"):
for r in s.get("routes", []):
routes = (result.get(s.get("stop_id")) or [])
routes.append(r)

5
config.yaml
View File

@ -3,7 +3,7 @@
# URLs and API keys for the different parts of the GTFS-R feed
# You should not change these unless a new version of the API is released
gtfs-feed-url: "https://www.transportforireland.ie/transitData/Data/GTFS_Realtime.zip"
gtfs-r-api-url: "https://api.nationaltransport.ie/gtfsr/v2/TripUpdates?format=json"
gtfs-r-api-url: "https://api.nationaltransport.ie/gtfsr/v2/gtfsr?format=json"
# You should change this one. Use the key you get from TFI when you register for GTFS-R access
gtfs-r-api_key: "API KEY GOES HERE"
@ -12,9 +12,6 @@ gtfs-r-api_key: "API KEY GOES HERE"
# It must be strictly larger than 60 because the GTFS-R API will throttle us otherwise
update-interval-seconds: 62
# The font to use for the display.
font-file: "jd_lcd_rounded.ttf"
stops: [
{
# Route 15A

287
gtfs_client.py
View File

@ -8,11 +8,12 @@ import pandas as pd
import queue
import refresh_feed
import requests
import sys
import time
import threading
import traceback
import zipfile
class GTFSClient:
class GTFSClient():
def __init__(self, feed_url: str, gtfs_r_url: str, gtfs_r_api_key: str,
stop_codes: list[str], routes_for_stops: dict[str, str],
update_queue: queue.Queue, update_interval_seconds: int = 60):
@ -20,20 +21,20 @@ class GTFSClient:
self.stop_codes = stop_codes
self.routes_for_stops = routes_for_stops
feed_name = '/tmp/' + feed_url.split('/')[-1]
feed_name = feed_url.split('/')[-1]
self.gtfs_r_url = gtfs_r_url
self.gtfs_r_api_key = gtfs_r_api_key
# Make sure that the feed file is up to date
try:
last_mtime = int(os.stat(feed_name).st_mtime)
last_mtime = os.stat(feed_name).st_mtime
except:
last_mtime = 0
_, new_mtime = refresh_feed.update_local_file_from_url_v1(last_mtime, feed_name, feed_url)
refreshed, new_mtime = refresh_feed.update_local_file_from_url_v1(last_mtime, feed_name, feed_url)
# Load the feed
self.feed = self._read_feed(feed_name, dist_units='km')
self.feed = self._read_feed(feed_name, dist_units='km', stop_codes = stop_codes)
gc.collect()
self.stop_ids = self.__wanted_stop_ids()
self.deltas = {}
@ -44,8 +45,9 @@ class GTFSClient:
self._update_queue = update_queue
if update_interval_seconds and update_queue:
self._update_interval_seconds = update_interval_seconds
self._refresh_thread = threading.Thread(target=lambda: every(update_interval_seconds, self.refresh))
def _read_feed(self, path: str, dist_units: str) -> gk.Feed:
def _read_feed(self, path: gk.Path, dist_units: str, stop_codes: list[str]) -> gk.Feed:
"""
NOTE: This helper method was extracted from gtfs_kit.feed to modify it
to only load the stop_times for the stops we are interested in,
@ -54,7 +56,7 @@ class GTFSClient:
This version also reads CSV data straight from the zip file to avoid
wearing out the Pi's SD card.
"""
files_to_load = [
FILES_TO_LOAD = [
# List of feed files to load. stop_times.txt is loaded separately.
'trips.txt',
'routes.txt',
@ -64,15 +66,15 @@ class GTFSClient:
'agency.txt'
]
if not os.path.exists(path):
raise ValueError("Path {} does not exist".format(path))
path = gk.Path(path)
if not path.exists():
raise ValueError(f"Path {path} does not exist")
print("Loading GTFS feed {}".format(path), file=sys.stderr)
gc.collect()
feed_dict = {table: None for table in gk.cs.GTFS_REF["table"]}
with zipfile.ZipFile(path) as z:
for filename in files_to_load:
for filename in FILES_TO_LOAD:
table = filename.split(".")[0]
# read the file
with z.open(filename) as f:
@ -214,22 +216,20 @@ class GTFSClient:
next_buses.drop(index=ids_to_delete, inplace=True)
return next_buses
@staticmethod
def __time_to_seconds(s: str) -> int:
def __time_to_seconds(self, s: str) -> int:
sx = s.split(":")
if len(sx) != 3:
print("Malformed timestamp:", s)
return 0
return int(sx[0]) * 3600 + int(sx[1]) * 60 + int (sx[2])
@staticmethod
def __due_in_seconds(time_str: str) -> int:
def __due_in_seconds(self, time_str: str) -> int:
"""
Returns the number of seconds in the future that the time_str (format hh:mm:ss) is
"""
now = datetime.datetime.now().strftime("%H:%M:%S")
tnow = GTFSClient.__time_to_seconds(now)
tstop = GTFSClient.__time_to_seconds(time_str)
tnow = self.__time_to_seconds(now)
tstop = self.__time_to_seconds(time_str)
if tstop > tnow:
return tstop - tnow
else:
@ -242,96 +242,88 @@ class GTFSClient:
Look up a destination string in Trips from the route and direction
"""
trips = self.feed.trips
destination = trips[(trips["route_id"] == route_id) & (trips["direction_id"] == direction_id)].head(1)["trip_headsign"].item()
# For some reason destination sometimes isn't a string. Try to find out why
if not destination.__class__ == str:
sys.stderr.write("Destination not found for route " + str(route_id) + ", direction " + str(direction_id) + "\n")
destination = "---- ?????? ----"
return destination
return trips[(trips["route_id"] == route_id) & (trips["direction_id"] == direction_id)].head(1)["trip_headsign"].item()
def __poll_gtfsr_deltas(self) -> tuple[dict, list, list]:
try:
# Poll GTFS-R API
if self.gtfs_r_api_key != "":
headers = {"x-api-key": self.gtfs_r_api_key}
response = requests.get(url = self.gtfs_r_url, headers = headers, timeout=(2, 10))
if response.status_code != 200:
print("GTFS-R sent non-OK response: {}\n{}".format(response.status_code, response.text))
return {}, [], []
def __poll_gtfsr_deltas(self) -> list[map, set]:
deltas_json = json.loads(response.text)
else:
deltas_json = json.load(open("example.json"))
# Poll GTFS-R API
if self.gtfs_r_api_key != "":
headers = {"x-api-key": self.gtfs_r_api_key}
response = requests.get(url = self.gtfs_r_url, headers = headers)
if response.status_code != 200:
print("GTFS-R sent non-OK response: {}\n{}".format(response.status_code, response.text))
return ({}, [], [])
deltas = {}
canceled_trips = set()
added_stops = []
deltas_json = json.loads(response.text)
else:
deltas_json = json.load(open("example.json"))
# Pre-compute some data to use for added trips:
relevant_service_ids = self.__current_service_ids()
relevant_trips = self.feed.trips[self.feed.trips["service_id"].isin(relevant_service_ids)]
relevant_route_ids = set(relevant_trips["route_id"])
today = datetime.date.today().strftime("%Y%m%d")
deltas = {}
canceled_trips = set()
added_stops = []
for e in deltas_json.get("entity", []):
try:
trip_update = e.get("trip_update")
trip = trip_update.get("trip")
trip_id = trip.get("trip_id")
trip_action = trip.get("schedule_relationship")
if trip_action == "SCHEDULED":
for u in e.get("trip_update", {}).get("stop_time_update", []):
delay = u.get("arrival", u.get("departure", {})).get("delay", 0)
deltas_for_trip = (deltas.get(trip_id) or {})
deltas_for_trip[u.get("stop_id")] = delay
deltas[trip_id] = deltas_for_trip
# Pre-compute some data to use for added trips:
relevant_service_ids = self.__current_service_ids()
relevant_trips = self.feed.trips[self.feed.trips["service_id"].isin(relevant_service_ids)]
relevant_route_ids = set(relevant_trips["route_id"])
today = datetime.date.today().strftime("%Y%m%d")
elif trip_action == "ADDED":
start_date = trip.get("start_date")
start_time = trip.get("start_time")
route_id = trip.get("route_id")
direction_id = trip.get("direction_id")
for e in deltas_json.get("entity", []):
is_deleted = e.get("is_deleted") or False
try:
trip_update = e.get("trip_update")
trip = trip_update.get("trip")
trip_id = trip.get("trip_id")
trip_action = trip.get("schedule_relationship")
if trip_action == "SCHEDULED":
for u in e.get("trip_update", {}).get("stop_time_update", []):
delay = u.get("arrival", u.get("departure", {})).get("delay", 0)
deltas_for_trip = (deltas.get(trip_id) or {})
deltas_for_trip[u.get("stop_id")] = delay
deltas[trip_id] = deltas_for_trip
# Check if the route is part of the routes we care about
if not route_id in relevant_route_ids:
continue
elif trip_action == "ADDED":
start_date = trip.get("start_date")
start_time = trip.get("start_time")
route_id = trip.get("route_id")
direction_id = trip.get("direction_id")
# Check if the route is part of the routes we care about
if not route_id in relevant_route_ids:
continue
# And that it's for today
current_time = datetime.datetime.now().strftime("%H:%M:%S")
if start_date > today or start_time > current_time:
continue
# And that it's for today
current_time = datetime.datetime.now().strftime("%H:%M:%S")
if start_date > today or start_time > current_time:
continue
# Look for the entry for any of the stops we want
wanted_stop_ids = self.__wanted_stop_ids()
for stop_time_update in e.get("trip_update").get("stop_time_update", []):
if stop_time_update.get("stop_id", "") in wanted_stop_ids:
arrival_time = int((stop_time_update.get("arrival", stop_time_update.get("departure", {})).get("time", 0)))
if arrival_time < int(time.time()):
continue
new_arrival = ArrivalTime(
stop_id = stop_time_update.get("stop_code"),
route_id = self.feed.routes[self.feed.routes["route_id"] == route_id]["route_short_name"].item(),
destination = self.__lookup_headsign_by_route(route_id, direction_id),
due_in_seconds = arrival_time - int(time.time()),
is_added = True
)
print("Added route:", new_arrival)
added_stops.append(new_arrival)
# Look for the entry for any of the stops we want
wanted_stop_ids = self.__wanted_stop_ids()
for stop_time_update in e.get("trip_update").get("stop_time_update", []):
if stop_time_update.get("stop_id", "") in wanted_stop_ids:
arrival_time = int((stop_time_update.get("arrival", stop_time_update.get("departure", {})).get("time", 0)))
if arrival_time < int(time.time()):
continue
new_arrival = ArrivalTime(
stop_id = stop_time_update.get("stop_id"),
route_id = self.feed.routes[self.feed.routes["route_id"] == route_id]["route_short_name"].item(),
destination = self.__lookup_headsign_by_route(route_id, direction_id),
due_in_seconds = arrival_time - int(time.time()),
is_added = True
)
print("Added route:", new_arrival)
added_stops.append(new_arrival)
elif trip_action == "CANCELED":
canceled_trips.add(trip_id)
else:
print("Unsupported action:", trip_action)
except Exception as x:
print("Error parsing GTFS-R entry:", str(e))
raise x
return deltas, canceled_trips, added_stops
except Exception as e:
print("Polling for GTFS-R failed:", str(e))
return {}, [], []
elif trip_action == "CANCELED":
canceled_trips.add(trip_id)
else:
print("Unsupported action:", trip_action)
except Exception as x:
print("Error parsing GTFS-R entry:", str(e))
raise(x)
return deltas, canceled_trips, added_stops
def get_next_n_buses(self, num_entries: int) -> pd.core.frame.DataFrame:
@ -346,48 +338,69 @@ class GTFSClient:
return joined_data
def start(self) -> None:
""" Start the refresh thread """
self._refresh_thread.start()
self.refresh()
def refresh(self):
"""
Create and enqueue the refreshed stop data
"""
# Retrieve the GTFS-R deltas
deltas, canceled_trips, added_stops = self.__poll_gtfsr_deltas()
if len(deltas) > 0 or len(canceled_trips) > 0 or len(added_stops) > 0:
# Only update deltas and canceled trips if the API returns data
self.deltas = deltas
self.canceled_trips = canceled_trips
self.added_stops = added_stops
arrivals = []
# take more entries than we need in case there are cancelations
buses = self.get_next_n_buses(15)
for index, bus in buses.iterrows():
if not bus["trip_id"] in self.canceled_trips:
delta = self.deltas.get(bus["trip_id"], {}).get(bus["stop_id"], 0)
if delta != 0:
print("Delta for route {} stop {} is {}".format(bus["route_short_name"], bus["stop_id"], delta))
arrival = ArrivalTime(stop_id = bus["stop_id"],
route_id = bus["route_short_name"],
destination = bus["trip_headsign"],
due_in_seconds = self.__due_in_seconds(bus["arrival_time"]) + delta,
is_added = False
)
arrivals.append(arrival)
if len(self.added_stops) > 0:
# Append the added stops from GTFS-R and re-sort
arrivals.extend(self.added_stops)
arrivals.sort()
# Select the first 5 of what remains
arrivals = arrivals[0:5]
if self._update_queue:
self._update_queue.put(arrivals)
gc.collect()
return arrivals
def every(delay, task) -> None:
""" Auxilliary function to schedule updates.
Taken from https://stackoverflow.com/questions/474528/what-is-the-best-way-to-repeatedly-execute-a-function-every-x-seconds
"""
next_time = time.time() + delay
while True:
time.sleep(max(0, next_time - time.time()))
try:
# Retrieve the GTFS-R deltas
deltas, canceled_trips, added_stops = self.__poll_gtfsr_deltas()
if len(deltas) > 0 or len(canceled_trips) > 0 or len(added_stops) > 0:
# Only update deltas and canceled trips if the API returns data
self.deltas = deltas
self.canceled_trips = canceled_trips
self.added_stops = added_stops
arrivals = []
# take more entries than we need in case there are cancellations
buses = self.get_next_n_buses(15)
for index, bus in buses.iterrows():
if not bus["trip_id"] in self.canceled_trips:
delta = self.deltas.get(bus["trip_id"], {}).get(bus["stop_id"], 0)
if delta != 0:
print("Delta for route {} stop {} is {}".format(bus["route_short_name"], bus["stop_id"], delta))
arrival = ArrivalTime(stop_id = bus["stop_code"],
route_id = bus["route_short_name"],
destination = bus["trip_headsign"],
due_in_seconds = GTFSClient.__due_in_seconds(bus["arrival_time"]) + delta,
is_added = False
)
arrivals.append(arrival)
if len(self.added_stops) > 0:
# Append the added stops from GTFS-R and re-sort
arrivals.extend(self.added_stops)
arrivals.sort()
# Select the first 5 of what remains
arrivals = arrivals[0:5]
if self._update_queue:
self._update_queue.put(arrivals)
gc.collect()
except Exception as e:
print("Exception in refresh: {}".format(str(e)))
task()
except Exception:
traceback.print_exc()
# in production code you might want to have this instead of course:
# logger.exception("Problem while executing repetitive task.")
# skip tasks if we are behind schedule:
next_time += (time.time() - next_time) // delay * delay + delay

100
main.py
View File

@ -7,7 +7,6 @@ import gc
from glob import glob
import pygame
from pygame.locals import *
import schedule
from time import sleep
import queue
from arrival_times import ArrivalTime
@ -72,34 +71,31 @@ def write_line(line: int, text: str, text_color: Color = COLOR_LCD_AMBER):
def update_screen(config: Config, updates: list[ArrivalTime]) -> None:
""" Repaint the screen with the new arrival times """
try:
updates = updates[0:LINE_COUNT] # take the first X lines
for line_num, update in enumerate(updates):
# Find what color we need to use for the ETA
time_to_walk = update.due_in_minutes - (config.minutes_to_stop(update.stop_id) or 0)
lcd_color = None
if time_to_walk > 5:
lcd_color = COLOR_LCD_GREEN
elif time_to_walk > 1:
lcd_color = COLOR_LCD_AMBER
else:
lcd_color = COLOR_LCD_RED
updates = updates[0:LINE_COUNT] # take the first X lines
for line_num, update in enumerate(updates):
# Find what color we need to use for the ETA
time_to_walk = update.due_in_minutes - (config.minutes_to_stop(update.stop_id) or 0)
lcd_color = None
if time_to_walk > 5:
lcd_color = COLOR_LCD_GREEN
elif time_to_walk > 1:
lcd_color = COLOR_LCD_AMBER
else:
lcd_color = COLOR_LCD_RED
# Draw the line
write_entry(
line = line_num,
route = update.route_id,
destination = update.destination,
time_left = 'Due' if update.is_due() else update.due_in_str(),
time_color = lcd_color,
text_color = COLOR_LCD_GREEN if update.is_added else COLOR_LCD_AMBER
)
# Draw the line
write_entry(
line = line_num,
route = update.route_id,
destination = update.destination,
time_left = 'Due' if update.isDue() else update.due_in_str(),
time_color = lcd_color,
text_color = COLOR_LCD_GREEN if update.is_added else COLOR_LCD_AMBER
)
# Add the current time to the bottom line
datetime_text = "Current time: " + datetime.today().strftime("%d/%m/%Y %H:%M")
write_line(5, datetime_text)
except Exception as e:
print("Error updating screen: ", str(e))
# Add the current time to the bottom line
datetime_text = "Current time: " + datetime.today().strftime("%d/%m/%Y %H:%M")
write_line(5, datetime_text)
def clear_screen() -> None:
""" Clear screen """
@ -124,12 +120,10 @@ def main():
config = Config()
# Initialise graphics context
pygame.display.init()
pygame.font.init()
pygame.init()
window = init_screen()
pygame.font.init()
font = pygame.font.Font(config.font_file or TEXT_FONT, TEXT_SIZE)
font = pygame.font.Font(TEXT_FONT, TEXT_SIZE)
# Init screen
clear_screen()
@ -146,39 +140,33 @@ def main():
update_queue=update_queue,
update_interval_seconds=config.update_interval_seconds)
# Schedule feed refresh, and force the first one
schedule.every(config.update_interval_seconds).seconds.do(scheduler.refresh)
scheduler.refresh()
scheduler.start()
# Main event loop
running = True
while running:
try:
# Pygame event handling begins
if pygame.event.peek():
for e in pygame.event.get():
if e.type == pygame.QUIT:
# Pygame event handling begins
if pygame.event.peek():
for e in pygame.event.get():
if e.type == pygame.QUIT:
running = False
elif e.type == pygame.KEYDOWN:
if e.key == pygame.K_ESCAPE:
running = False
elif e.type == pygame.KEYDOWN:
if e.key == pygame.K_ESCAPE:
running = False
pygame.display.flip()
# Pygame event handling ends
pygame.display.flip()
# Pygame event handling ends
# Display update begins
schedule.run_pending()
if update_queue.qsize() > 0:
clear_screen()
updates = update_queue.get()
update_screen(config, updates)
# Display update begins
if update_queue.qsize() > 0:
clear_screen()
updates = update_queue.get()
update_screen(config, updates)
pygame.display.flip()
gc.collect()
# Display update ends
pygame.display.flip()
gc.collect()
# Display update ends
sleep(0.2)
except Exception as e:
print("Exception in main loop: ", str(e))
sleep(0.2)
pygame.quit()
exit(0)

75
refresh_feed.py
View File

@ -7,7 +7,6 @@ import os
import sys
import time
import requests
import urllib3
# First we construct a handful of functions - testing happens down at the end
def httpdate_to_ts(dt):
@ -18,6 +17,16 @@ def httpdate_to_ts(dt):
def ts_to_httpdate(ts):
return email.utils.formatdate(timeval=ts, localtime=False, usegmt=True)
def write_file_with_time(filename, content, timestamp):
# put the content into the file
with open(filename, 'wb') as fp:
fp.write(content)
# Then set the file's timestamps as requested
os.utime(filename, times=(time.time(), timestamp))
# v1: download remote file if HTTP's Last-Modified header indicates that
# the file has been updated. This requires the remote server to support
# sending the Last-Modified header.
@ -42,34 +51,54 @@ def update_local_file_from_url_v1(last_mtime, local_file, url):
# If file is newer than last one we saw, get it
updated = False
print('Comparing feed mtimes: feed: {} vs remote {}'.format(str(last_mtime), str(mtime)), file=sys.stderr)
if not last_mtime or mtime > int(last_mtime):
print('Refreshing feed..', file=sys.stderr)
if mtime > int(last_mtime):
updated = True
# download the new file content
conn = urllib3.connection_from_url(url)
r2 = conn.request(method="GET", url=url, preload_content=False)
if r2.status != 200:
r2 = requests.get(url) # download the new file content
if r2.status_code != requests.codes.ok:
# http request failed
print('HEY! get for {} returned {}'.format(url, r2.status_code),
file=sys.stderr)
try:
r2.release_conn()
except Exception as e:
print('Could not release connection to {}: {}'.format(url, str(e)))
file=sys.stderr)
return False, last_mtime
with open(local_file,'bw') as f:
for chunk in r2.stream(amt=65536, decode_content=True):
f.write(chunk)
# write new content to local file
write_file_with_time(local_file, r2.content, mtime)
r2.release_conn()
# Change the mtime of the file
os.utime(local_file, (mtime, mtime))
return updated, mtime
# v2: download remote file conditionally, with HTTP's If-Modified-Since header.
# This requires the remote server to support both sending the Last-Modified
# header and receiving the If-Modified-Since header.
#
def update_local_file_from_url_v2(last_mtime, local_file, url):
# Get the remote file, but only if it has changed
r = requests.get(url, headers={
'If-Modified-Since': ts_to_httpdate(last_mtime)
})
updated, mtime = False, last_mtime
if r.status_code == requests.codes.ok:
# File is updated and we just downloaded the content
updated = True
# write new content to local file
print('Downloaded {}.'.format(local_file), file=sys.stderr)
else:
print('No need to refresh feed.', file=sys.stderr)
write_file_with_time(local_file, r.content, mtime)
return updated, mtime
# Update our notion of the file's last modification time
if 'Last-Modified' in r.headers:
mtime = httpdate_to_ts(r.headers['Last-Modified'])
else:
print('HEY! no Last-Modified header for {}'.format(url),
file=sys.stderr)
elif r.status_code == requests.codes.not_modified:
# Successful call, but no updates to file
print('As of {}, server says {} is the same'.format(time.ctime(), url))
else:
# http request failed
print('HEY! get for {} returned {}'.format(url, r.status_code),
file=sys.stderr)
return updated, mtime

13
requirements.txt
View File

@ -1,10 +1,3 @@
gtfs_kit
iso8601
pandas
pygame
pyyaml
requests
urllib3
schedule
zeep
iso8601==1.0.2
pygame==2.1.2
zeep==4.1.0