diff --git a/README.md b/README.md index be74cbe..28d361e 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ If your display's resolution is not 1920x720, you will also need to change the c * yaml ```shell -$ sudo apt install python3-iso8601 python3-zeep libsdl2-ttf-2.0-0 python3-numpy python3-pandas python3-fiona python3-pyproj libspatialindex-c6 python3-yaml +$ sudo apt install python3-iso8601 python3-zeep libsdl2-ttf-2.0-0 python3-numpy python3-pandas python3-fiona python3-pyproj libspatialindex-c6 python3-yaml python3-urllib3 ``` * pygame 2 diff --git a/refresh_feed.py b/refresh_feed.py index 2d2b94c..18239e4 100644 --- a/refresh_feed.py +++ b/refresh_feed.py @@ -7,6 +7,7 @@ import os import sys import time import requests +import urllib3 # First we construct a handful of functions - testing happens down at the end def httpdate_to_ts(dt): @@ -17,16 +18,6 @@ def httpdate_to_ts(dt): def ts_to_httpdate(ts): return email.utils.formatdate(timeval=ts, localtime=False, usegmt=True) - -def write_file_with_time(filename, content, timestamp): - # put the content into the file - with open(filename, 'wb') as fp: - fp.write(content) - - # Then set the file's timestamps as requested - os.utime(filename, times=(time.time(), timestamp)) - - # v1: download remote file if HTTP's Last-Modified header indicates that # the file has been updated. This requires the remote server to support # sending the Last-Modified header. @@ -55,15 +46,28 @@ def update_local_file_from_url_v1(last_mtime, local_file, url): if not last_mtime or mtime > int(last_mtime): print('Refreshing feed..', file=sys.stderr) updated = True - r2 = requests.get(url) # download the new file content - if r2.status_code != requests.codes.ok: + # download the new file content + conn = urllib3.connection_from_url(url) + r2 = conn.request(method="GET", url=url, preload_content=False) + if r2.status != 200: # http request failed print('HEY! get for {} returned {}'.format(url, r2.status_code), - file=sys.stderr) + file=sys.stderr) + try: + r2.release_conn() + except Exception as e: + print('Could not release connection to {}: {}'.format(url, str(e))) return False, last_mtime + with open(local_file,'bw') as f: + for chunk in r2.stream(amt=65536, decode_content=True): + f.write(chunk) + + r2.release_conn() + # Change the mtime of the file + os.utime(local_file, (mtime, mtime)) + # write new content to local file - write_file_with_time(local_file, r2.content, mtime) print('Downloaded {}.'.format(local_file), file=sys.stderr) else: print('No need to refresh feed.', file=sys.stderr)