diff --git a/micropython/minitz/manifest.py b/micropython/minitz/manifest.py
new file mode 100644
index 000000000..51612c738
--- /dev/null
+++ b/micropython/minitz/manifest.py
@@ -0,0 +1,3 @@
+metadata(description="MiniTZ timezone support.", version="0.1.0")
+
+module("minitz.py", opt=3)
diff --git a/micropython/minitz/minitz/__init__.py b/micropython/minitz/minitz/__init__.py
new file mode 100644
index 000000000..606692e43
--- /dev/null
+++ b/micropython/minitz/minitz/__init__.py
@@ -0,0 +1,43 @@
+from _minitz import Database
+import datetime as _datetime
+
+
+# Wraps a _minitz.Zone, and implements tzinfo
+class tzwrap(_datetime.tzinfo):
+ def __init__(self, mtz_zone):
+ self._mtz_zone = mtz_zone
+
+ def __str__(self):
+ return self.tzname(None)
+
+ # Returns (offset: int, designator: str, is_dst: int)
+ def _lookup_local(self, dt):
+ if dt.tzinfo is not self:
+ raise ValueError()
+ t = dt.replace(tzinfo=_datetime.timezone.utc, fold=0).timestamp()
+ return self._mtz_zone.lookup_local(t, dt.fold)
+
+ def utcoffset(self, dt):
+ return _datetime.timedelta(seconds=self._lookup_local(dt)[0])
+
+ def is_dst(self, dt):
+ # Nonstandard. Returns bool.
+ return bool(self._lookup_local(dt)[2])
+
+ def dst(self, dt):
+ is_dst = self._lookup_local(dt)[2]
+ # TODO in the case of is_dst=1, this is returning
+ # a made-up value that may be wrong.
+ return _datetime.timedelta(hours=is_dst)
+
+ def tzname(self, dt):
+ return self._lookup_local(dt)[1]
+
+ def fromutc(self, dt):
+ if dt.fold != 0:
+ raise ValueError()
+ t = dt.replace(tzinfo=_datetime.timezone.utc).timestamp()
+ offset = self._mtz_zone.lookup_utc(t)[0]
+ _datetime.timedelta(seconds=offset)
+
+ return dt + self._offset
diff --git a/micropython/minitz/minitz/fetch.py b/micropython/minitz/minitz/fetch.py
new file mode 100644
index 000000000..a60b484f0
--- /dev/null
+++ b/micropython/minitz/minitz/fetch.py
@@ -0,0 +1,43 @@
+import requests
+import email.utils
+
+# Configurable settings
+server_url = 'http://tzdata.net/api/1/'
+dataset = '1-15'
+
+
+def fetch(url, last_modified, timeout=None):
+ headers = {}
+ if last_modified:
+ headers['if-modified-since'] = email.utils.formatdate(last_modified, False, True)
+
+ resp = requests.request('GET', url, headers=headers, timeout=timeout, parse_headers=True)
+
+ if resp.status_code == 304:
+ # Not modified
+ resp.close()
+ return None
+ if resp.status_code != 200:
+ resp.close()
+ raise Exception()
+
+ content = resp.content
+
+ last_modified = resp.get_date_as_int('last-modified') or 0
+
+ return (last_modified, content)
+
+
+def fetch_zone(zone_name, last_modified, timeout=None):
+ url = server_url + dataset + '/zones-minitzif/' + zone_name
+ return fetch(url, last_modified, timeout)
+
+
+def fetch_all(last_modified, timeout=None):
+ url = server_url + dataset + '/minitzdb'
+ return fetch(url, last_modified, timeout)
+
+
+def fetch_names(last_modified, timeout=None):
+ url = server_url + 'zone-names-mini'
+ return fetch(url, last_modified, timeout)
diff --git a/micropython/minitz/minitz/persist.py b/micropython/minitz/minitz/persist.py
new file mode 100644
index 000000000..8d428324e
--- /dev/null
+++ b/micropython/minitz/minitz/persist.py
@@ -0,0 +1,231 @@
+import datetime
+import os
+import struct
+import time
+from . import Database, tzwrap
+from .fetch import fetch_zone, fetch_all
+
+_local_zone_name = 'UTC'
+_whole_db = False
+
+_db = None
+_last_modified = None
+_local_zone = None
+_local_tzinfo = datetime.timezone.utc
+
+_last_check = None
+
+path_for_meta_file = 'tzmeta'
+path_for_db_file = 'tzdata'
+
+# Initialise by reading from persistent storage.
+def init(want_zone_name=None, want_whole_db=None):
+ try:
+ with open(path_for_meta_file, 'rb') as fp:
+ last_modified, data_crc = struct.unpack('<QI', fp.read(12))
+ local_zone_name = fp.read().decode()
+
+ if not local_zone_name:
+ # Corrupt file:
+ # Mode should be 1 or 2.
+ # Zone name is mandatory.
+ raise ValueError()
+ if last_modified == 0:
+ last_modified = None
+
+ with open(path_for_db_file, 'rb') as fp:
+ data = fp.read()
+
+ db = Database(data)
+ if db.kind != 1 and db.kind != 2:
+ raise ValueError()
+ if db.crc != data_crc:
+ # The tzdata and tzmeta files do not match.
+ raise ValueError()
+
+ whole_db = (db.kind == 2)
+ if want_whole_db is not None and want_whole_db != whole_db:
+ # Want to download one zone file only, have whole DB
+ # OR want to download whole DB, only have one zone
+ raise ValueError()
+
+ if want_zone_name is not None and want_zone_name != local_zone_name:
+ if not whole_db:
+ # Need to download correct zone file.
+ raise ValueError()
+ local_zone_name = want_zone_name
+
+ # For a TZIF file, the string passed to get_zone_by_name() is ignored.
+ local_zone = db.get_zone_by_name(local_zone_name)
+ local_tzinfo = tzwrap(local_zone)
+
+ # Success.
+ success = True
+ except:
+ # Failed
+ success = False
+
+ db = None
+ last_modified = None
+ local_zone_name = want_zone_name or 'UTC'
+ whole_db = whole_db or False
+ local_zone = None
+ local_tzinfo = datetime.timezone.utc if local_zone_name == 'UTC' else None
+
+ # Save state.
+ global _local_zone_name, _whole_db, _db, _last_modified, _local_zone, _local_tzinfo, _last_check
+ _local_zone_name = local_zone_name
+ _whole_db = whole_db
+ _db = db
+ _last_modified = last_modified
+ _local_zone = local_zone
+ _local_tzinfo = local_tzinfo
+ _last_check = None
+ if success:
+ # Pretend last check was 23.5 hours ago.
+ # That way the next check will be in 30 minutes.
+ # This means that if there are many reboots in a row, we don't flood
+ # the server with requests.
+ # 23.5 * 3600 * 1000 = 84_600_000
+ #
+ # (It would be better if we could use real UTC time to track when the
+ # last check was, and store the last update time in persistent memory.
+ # But we don't necessarily know the real UTC time at init time, and may
+ # not want a Flash write on every update check).
+ _last_check = time.ticks_add(time.ticks_ms(), -84_600_000)
+
+ return success
+
+
+def _force_update_from_internet(zone_name=None, whole_db=None, timeout=None):
+ last_modified = _last_modified
+ if whole_db is None:
+ whole_db = _whole_db
+ elif whole_db != _whole_db:
+ # Force fresh download as it's a different file
+ last_modified = None
+ if zone_name is None:
+ zone_name = _local_zone_name
+ elif zone_name != _local_zone_name and not whole_db:
+ # Force fresh download as it's a different file
+ last_modified = None
+ if not zone_name:
+ # Empty string is not a valid zone name.
+ raise ValueError()
+
+ # We update _last_check even if the HTTP request fails.
+ # This is to comply with the fair usage policy of tzdata.net.
+ global _last_check
+ _last_check = time.ticks_ms()
+
+ if whole_db:
+ last_modified, data = fetch_zone(zone_name, last_modified, timeout)
+ else:
+ last_modified, data = fetch_all(last_modified, timeout)
+
+ if data is None:
+ # Not changed
+ return
+
+ db = Database(data)
+ if db.kind != (2 if whole_db else 1):
+ # Not the kind of file that was expected
+ raise ValueError()
+
+ # For a TZIF file, the string passed to get_zone_by_name() is ignored.
+ local_zone = db.get_zone_by_name(zone_name)
+ local_tzinfo = tzwrap(local_zone)
+
+ # Download success!
+
+ # Save state.
+ global _local_zone_name, _whole_db, _db, _last_modified, _local_zone, _local_tzinfo
+ _local_zone_name = zone_name
+ _whole_db = whole_db
+ _db = db
+ _last_modified = last_modified
+ _local_zone = local_zone
+ _local_tzinfo = local_tzinfo
+
+ # Save the data to persistent storage.
+
+ # Maybe this may make flash wear-levelling easier?
+ # We give the filesystem as much free space as possible
+ # before we start writing to it.
+ os.unlink(path_for_db_file)
+
+ with open(path_for_meta_file, 'wb') as fp:
+ fp.write(struct.pack('<QI', last_modified or 0, db.crc))
+ fp.write(zone_name.encode())
+
+ with open(path_for_db_file, 'wb') as fp:
+ fp.write(data)
+
+
+# Initialise by reading from persistent storage.
+# If that fails, will try to do first-time download of timezone
+# data from the Internet.
+def init_with_download_if_needed(zone_name=None, whole_db=None, timeout=None):
+ if not init(zone_name, whole_db):
+ if whole_db or zone_name != 'UTC':
+ _force_update_from_internet(zone_name, whole_db, timeout)
+
+
+def set_zone(zone_name, can_download, timeout=None):
+ if _local_zone_name == zone_name:
+ # Nothing to do!
+ pass
+ elif _whole_db:
+ local_zone = _db.get_zone_by_name(zone_name)
+ local_tzinfo = tzwrap(local_zone)
+
+ global _local_zone_name, _local_zone, _local_tzinfo
+ _local_zone_name = zone_name
+ _local_zone = local_zone
+ _local_tzinfo = local_tzinfo
+ elif not can_download:
+ raise ValueError("Changing zone without whole DB or Internet")
+ else:
+ _force_update_from_internet(zone_name, _whole_db, timeout)
+
+
+def update_from_internet_if_needed(timeout=None):
+ # Call this regularly. Ideally at least once an hour, but it's fine
+ # to call it much more frequently, even multiple times per second.
+ # This function will do nothing if an update is not needed.
+ #
+ # We attempt an Internet update at most once per day.
+ # This is to comply with the fair usage policy of tzdata.net.
+ if (_last_check is not None and
+ time.ticks_diff(time.ticks_ms(), _last_check) < 24 * 3600 * 1000):
+ # Too soon.
+ return
+ _force_update_from_internet(timeout=timeout)
+
+
+def has_tzinfo():
+ return _local_tzinfo is not None
+
+def get_tzinfo():
+ if _local_tzinfo is None:
+ raise ValueError()
+ return _local_tzinfo
+
+def have_db():
+ return _db is not None
+
+def get_raw_zone():
+ if _local_zone is None:
+ raise ValueError()
+ return _local_zone
+
+def get_db():
+ if _db is None:
+ raise ValueError()
+ return _db
+
+def get_zone_name():
+ return _local_zone_name
+
+def get_last_modified():
+ return datetime.datetime.fromtimestamp(_last_modified, datetime.timezone.utc)
diff --git a/micropython/net/ntptime/ntptime.py b/micropython/net/ntptime/ntptime.py
index ff0d9d202..25cc62ad1 100644
--- a/micropython/net/ntptime/ntptime.py
+++ b/micropython/net/ntptime/ntptime.py
@@ -22,12 +22,37 @@ def time():
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
s.settimeout(timeout)
- res = s.sendto(NTP_QUERY, addr)
+ s.sendto(NTP_QUERY, addr)
msg = s.recv(48)
finally:
s.close()
val = struct.unpack("!I", msg[40:44])[0]
+ # 2024-01-01 00:00:00 converted to an NTP timestamp
+ MIN_NTP_TIMESTAMP = 3913056000
+
+ # Y2036 fix
+ #
+ # The NTP timestamp has a 32-bit count of seconds, which will wrap back
+ # to zero on 7 Feb 2036 at 06:28:16.
+ #
+ # We know that this software was written during 2024 (or later).
+ # So we know that timestamps less than MIN_NTP_TIMESTAMP are impossible.
+ # So if the timestamp is less than MIN_NTP_TIMESTAMP, that probably means
+ # that the NTP time wrapped at 2^32 seconds. (Or someone set the wrong
+ # time on their NTP server, but we can't really do anything about that).
+ #
+ # So in that case, we need to add in those extra 2^32 seconds, to get the
+ # correct timestamp.
+ #
+ # This means that this code will work until the year 2160. More precisely,
+ # this code will not work after 7th Feb 2160 at 06:28:15.
+ #
+ if val < MIN_NTP_TIMESTAMP:
+ val += 0x100000000
+
+ # Convert timestamp from NTP format to our internal format
+
EPOCH_YEAR = utime.gmtime(0)[0]
if EPOCH_YEAR == 2000:
# (date(2000, 1, 1) - date(1900, 1, 1)).days * 24*60*60
diff --git a/python-ecosys/requests/requests/__init__.py b/python-ecosys/requests/requests/__init__.py
index 740102916..9414be3e8 100644
--- a/python-ecosys/requests/requests/__init__.py
+++ b/python-ecosys/requests/requests/__init__.py
@@ -1,6 +1,16 @@
import usocket
+def _make_user_agent():
+ import sys
+ comment = sys.implementation._machine
+ comment = comment.replace('\\', '\\\\').replace('(', '\\(').replace(')', '\\)')
+ ua = "MicroPython/{}.{}.{} ({})".format(*sys.implementation[1][:3], comment)
+ return ua
+
+default_user_agent = _make_user_agent()
+
+
class Response:
def __init__(self, f):
self.raw = f
@@ -32,6 +42,29 @@ def json(self):
return ujson.loads(self.content)
+ def get_date(self, k):
+ import email.utils
+ as_str = self.headers.get(k)
+ if not as_str:
+ return None
+ return email.utils.parsedate_to_datetime(as_str)
+
+ def get_date_as_int(self, k):
+ import datetime
+ dt = self.get_date(k)
+ if dt is None:
+ return None
+ if dt.tzinfo is None:
+ dt = dt.replace(tzinfo=datetime.timezone.utc)
+ else:
+ dt = dt.astimezone(datetime.timezone.utc)
+
+ # dt.timestamp() returns a value of type float, but the number
+ # inside that float will always be an integer in this case,
+ # because the formats supported by parsedate_to_datetime()
+ # only have 1 second resolution.
+ return int(dt.timestamp())
+
def request(
method,
@@ -43,17 +76,39 @@ def request(
auth=None,
timeout=None,
parse_headers=True,
+ user_agent=True
):
redirect = None # redirection url, None means no redirection
chunked_data = data and getattr(data, "__next__", None) and not getattr(data, "__len__", None)
+ # Security and correctness: Can't add extra fields to `headers`,
+ # we have to have separate variables for them.
+ #
+ # This is because have `headers={}` in the function prototype.
+ # So the same dictionary will get reused for every call that doesn't
+ # explicitly specify a `headers` parameter.
+ #
+ # So if we put an Authorization header in `headers`, then following a
+ # request to a site that requires a username and password, that same
+ # plain-text username and password would be sent with every following HTTP
+ # request that used the default for the `headers` parameter.
+ basic_auth_header = None
if auth is not None:
import ubinascii
-
username, password = auth
- formated = b"{}:{}".format(username, password)
- formated = str(ubinascii.b2a_base64(formated)[:-1], "ascii")
- headers["Authorization"] = "Basic {}".format(formated)
+ basic_auth_header = b"{}:{}".format(username, password)
+ basic_auth_header = ubinascii.b2a_base64(basic_auth_header)[:-1]
+
+ if user_agent:
+ if user_agent is True:
+ # Use the standard User-Agent
+ user_agent = default_user_agent
+ elif user_agent[0] == ' ':
+ # It's a suffix to append to the standard User-Agent
+ user_agent = default_user_agent + user_agent
+
+ if user_agent:
+ user_agent = user_agent.encode('ascii')
try:
proto, dummy, host, path = url.split("/", 3)
@@ -91,11 +146,31 @@ def request(
s.connect(ai[-1])
if proto == "https:":
context = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT)
+ # TODO: This is a security vulnerability.
+ # HTTPS is providing nearly zero security, because of the next
+ # line. We disable all the protection against MiTM attacks!
+ #
+ # I mean... with this configuration, HTTPS still provides
+ # protection against passive eavesdropping, so there's that?
+ # But with modern network design, and modern attacks, anyone
+ # able to passively eavesdrop is almost certainly able to MiTM
+ # too. So the safety level is technically not quite zero, but
+ # it is very close to zero, and is far less than people using
+ # HTTPS expect.
context.verify_mode = tls.CERT_NONE
s = context.wrap_socket(s, server_hostname=host)
s.write(b"%s /%s HTTP/1.0\r\n" % (method, path))
if "Host" not in headers:
s.write(b"Host: %s\r\n" % host)
+ if basic_auth_header:
+ s.write(b"Authorization: Basic ")
+ s.write(basic_auth_header)
+ s.write(b"\r\n")
+ if user_agent:
+ s.write(b"User-Agent: ")
+ s.write(user_agent)
+ s.write(b"\r\n")
+
# Iterate over keys to avoid tuple alloc
for k in headers:
s.write(k)
@@ -152,7 +227,10 @@ def request(
elif parse_headers is True:
l = str(l, "utf-8")
k, v = l.split(":", 1)
- resp_d[k] = v.strip()
+ # Headers are case insensitive, so we lowercase them.
+ # This avoids having to do a linear case-insensitive search
+ # through the dictionary later.
+ resp_d[k.lower()] = v.strip()
else:
parse_headers(l, resp_d)
except OSError:
diff --git a/python-stdlib/datetime/datetime.py b/python-stdlib/datetime/datetime.py
index 0f2a89105..d26caa218 100644
--- a/python-stdlib/datetime/datetime.py
+++ b/python-stdlib/datetime/datetime.py
@@ -831,6 +831,7 @@ def timetuple(self):
conv = _tmod.gmtime
epoch = datetime.EPOCH.replace(tzinfo=None)
else:
+ # TODO this is wrong when a timezone is set
conv = _tmod.localtime
epoch = datetime.EPOCH
return conv(round((self - epoch).total_seconds()))