From e4f4e30b27cbc02d3ca47f8085390d1e8087aabf Mon Sep 17 00:00:00 2001 From: Nahuel Lofeudo Date: Sun, 16 Apr 2023 10:13:52 +0100 Subject: [PATCH] Added some code to automatically refresh the feed file if it changed --- .../Playground-checkpoint.ipynb | 48 ++++++++ .vscode/launch.json | 16 +++ gtfs_client.py | 29 ++++- main.py | 9 +- refresh_feed.py | 104 ++++++++++++++++++ 5 files changed, 197 insertions(+), 9 deletions(-) create mode 100644 .ipynb_checkpoints/Playground-checkpoint.ipynb create mode 100644 .vscode/launch.json create mode 100644 refresh_feed.py diff --git a/.ipynb_checkpoints/Playground-checkpoint.ipynb b/.ipynb_checkpoints/Playground-checkpoint.ipynb new file mode 100644 index 0000000..0e8565a --- /dev/null +++ b/.ipynb_checkpoints/Playground-checkpoint.ipynb @@ -0,0 +1,48 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "id": "1f552469", + "metadata": {}, + "outputs": [], + "source": [ + "import gtfs_kit as gk\n", + "import pandas as pd\n", + "import datetime\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "292cc196", + "metadata": {}, + "outputs": [], + "source": [ + "feed=gk.read_feed('google_transit_combined.zip', dist_units='km')\n", + "feed" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..0f5f884 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: GTFS Client", + "type": "python", + "request": "launch", + "program": "gtfs_client.py", + "console": "integratedTerminal", + "justMyCode": true + } + ] +} \ No newline at end of file diff --git a/gtfs_client.py b/gtfs_client.py index d8c8992..86c0bc4 100644 --- a/gtfs_client.py +++ b/gtfs_client.py @@ -1,6 +1,9 @@ + +import refresh_feed from arrival_times import ArrivalTime import datetime import gtfs_kit as gk +import os import pandas as pd import queue import time @@ -8,8 +11,19 @@ import threading import traceback class GTFSClient(): - def __init__(self, feed_name: str, stop_names: list[str], update_queue: queue.Queue, update_interval_seconds: int = 60): + def __init__(self, feed_url: str, stop_names: list[str], update_queue: queue.Queue, update_interval_seconds: int = 60): self.stop_names = stop_names + feed_name = feed_url.split('/')[-1] + + # Make sure that the feed file is up to date + last_mtime = os.stat(feed_name).st_mtime + refreshed, new_mtime = refresh_feed.update_local_file_from_url_v1(last_mtime, feed_name, feed_url) + if refreshed: + print("The feed file was refreshed.") + else: + print("The feed file was up to date") + + # Load the feed self.feed = gk.read_feed(feed_name, dist_units='km') self.stop_ids = self.__wanted_stop_ids() @@ -110,6 +124,11 @@ class GTFSClient(): return joined_data + def start(self) -> None: + """ Start the refresh thread """ + self._refresh_thread.start() + self.refresh() + def refresh(self): """ @@ -149,7 +168,7 @@ def every(delay, task) -> None: # skip tasks if we are behind schedule: next_time += (time.time() - next_time) // delay * delay + delay - -c = GTFSClient('google_transit_combined.zip', ['College Drive, stop 2410', 'Priory Walk, stop 1114'], None, None) - -print(c.refresh()) \ No newline at end of file +if __name__ == "__main__": + c = GTFSClient('https://www.transportforireland.ie/transitData/google_transit_combined.zip', + ['College Drive, stop 2410', 'Priory Walk, stop 1114'], None, None) + print(c.refresh()) \ No newline at end of file diff --git a/main.py b/main.py index 18cc60a..4508d37 100755 --- a/main.py +++ b/main.py @@ -8,6 +8,7 @@ from time import sleep from dublinbus_soap_client import DublinBusSoapClient import queue from arrival_times import ArrivalTime +from gtfs_client import GTFSClient # Constants # The font is JD LCD Rounded by Jecko Development @@ -22,8 +23,8 @@ COLOR_BACKGROUND = pygame.Color(0, 0, 0) UPDATE_INTERVAL_SECONDS = 30 TEXT_SIZE = 160 # Size of the font in pixels STOPS = [ - 2410, # College Drive - 1114 # Priory Walk + 'College Drive, stop 2410', + 'Priory Walk, stop 1114', ] # Define how long it takes to walk to a particular stop @@ -42,8 +43,8 @@ INTER_LINE_SPACE = -20 # 1920x720 -> 0 window : pygame.Surface = None font: pygame.font.Font = None update_queue = queue.Queue(maxsize=10) -dublinbus_client = DublinBusSoapClient(stops=STOPS, update_queue=update_queue, update_interval_seconds=UPDATE_INTERVAL_SECONDS) - +#dublinbus_client = DublinBusSoapClient(stops=STOPS, update_queue=update_queue, update_interval_seconds=UPDATE_INTERVAL_SECONDS) +dublinbus_client = GTFSClient(feed_name='google_transit_combined.zip', stop_names=STOPS, update_queue=update_queue, update_interval_seconds=UPDATE_INTERVAL_SECONDS) def get_line_offset(line: int) -> int: """ Calculate the Y offset within the display for a given text line """ diff --git a/refresh_feed.py b/refresh_feed.py new file mode 100644 index 0000000..abd72c4 --- /dev/null +++ b/refresh_feed.py @@ -0,0 +1,104 @@ +# Refresh the feed file from the original source +# Only download the file if the source is newer than the local copy +# This code was adapted from https://forums.raspberrypi.com/viewtopic.php?t=152226#p998268 + +import email.utils +import os +import sys +import time +import requests + +# First we construct a handful of functions - testing happens down at the end +def httpdate_to_ts(dt): + time_tuple = email.utils.parsedate_tz(dt) + return 0 if time_tuple is None else email.utils.mktime_tz(time_tuple) + + +def ts_to_httpdate(ts): + return email.utils.formatdate(timeval=ts, localtime=False, usegmt=True) + + +def write_file_with_time(filename, content, timestamp): + # put the content into the file + with open(filename, 'wb') as fp: + fp.write(content) + + # Then set the file's timestamps as requested + os.utime(filename, times=(time.time(), timestamp)) + + +# v1: download remote file if HTTP's Last-Modified header indicates that +# the file has been updated. This requires the remote server to support +# sending the Last-Modified header. +# +def update_local_file_from_url_v1(last_mtime, local_file, url): + + # Check the status of the remote file without downloading it + r1 = requests.head(url) + if r1.status_code != requests.codes.ok: + # http request failed + print('HEY! get for {} returned {}'.format(url, r1.status_code), + file=sys.stderr) + return False, last_mtime + + # Get the modification time for the file, if possible + if 'Last-Modified' in r1.headers: + mtime = httpdate_to_ts(r1.headers['Last-Modified']) + else: + print('HEY! no Last-Modified header for {}'.format(url), + file=sys.stderr) + return False, last_mtime + + # If file is newer than last one we saw, get it + updated = False + if mtime > int(last_mtime): + updated = True + r2 = requests.get(url) # download the new file content + if r2.status_code != requests.codes.ok: + # http request failed + print('HEY! get for {} returned {}'.format(url, r2.status_code), + file=sys.stderr) + return False, last_mtime + + # write new content to local file + write_file_with_time(local_file, r2.content, mtime) + + return updated, mtime + + +# v2: download remote file conditionally, with HTTP's If-Modified-Since header. +# This requires the remote server to support both sending the Last-Modified +# header and receiving the If-Modified-Since header. +# +def update_local_file_from_url_v2(last_mtime, local_file, url): + + # Get the remote file, but only if it has changed + r = requests.get(url, headers={ + 'If-Modified-Since': ts_to_httpdate(last_mtime) + }) + + updated, mtime = False, last_mtime + + if r.status_code == requests.codes.ok: + # File is updated and we just downloaded the content + updated = True + + # write new content to local file + write_file_with_time(local_file, r.content, mtime) + + # Update our notion of the file's last modification time + if 'Last-Modified' in r.headers: + mtime = httpdate_to_ts(r.headers['Last-Modified']) + else: + print('HEY! no Last-Modified header for {}'.format(url), + file=sys.stderr) + + elif r.status_code == requests.codes.not_modified: + # Successful call, but no updates to file + print('As of {}, server says {} is the same'.format(time.ctime(), url)) + else: + # http request failed + print('HEY! get for {} returned {}'.format(url, r.status_code), + file=sys.stderr) + + return updated, mtime