diff --git a/micro/bluetooth.py b/micro/app/bluetooth.py similarity index 98% rename from micro/bluetooth.py rename to micro/app/bluetooth.py index f05a5ba..c423a12 100644 --- a/micro/bluetooth.py +++ b/micro/app/bluetooth.py @@ -4,9 +4,9 @@ import network import ubluetooth import json -from wifi import connect_wifi, scan_wifi -from config import get_device_id, add_config from machine import Pin +from app.wifi import connect_wifi, scan_wifi +from app.config import get_device_id, add_config DEVICE_DATA = 'DEVICE_DATA' REQUEST_NETWORKS = 'REQUEST_NETWORKS' diff --git a/micro/config.py b/micro/app/config.py similarity index 100% rename from micro/config.py rename to micro/app/config.py diff --git a/micro/app/ota/httpclient.py b/micro/app/ota/httpclient.py new file mode 100644 index 0000000..ea7958d --- /dev/null +++ b/micro/app/ota/httpclient.py @@ -0,0 +1,184 @@ +import usocket, os, gc +class Response: + + def __init__(self, socket, saveToFile=None): + self._socket = socket + self._saveToFile = saveToFile + self._encoding = 'utf-8' + if saveToFile is not None: + CHUNK_SIZE = 512 # bytes + with open(saveToFile, 'w') as outfile: + data = self._socket.read(CHUNK_SIZE) + while data: + outfile.write(data) + data = self._socket.read(CHUNK_SIZE) + outfile.close() + + self.close() + + def close(self): + if self._socket: + self._socket.close() + self._socket = None + + @property + def content(self): + if self._saveToFile is not None: + raise SystemError('You cannot get the content from the response as you decided to save it in {}'.format(self._saveToFile)) + + try: + result = self._socket.read() + return result + finally: + self.close() + + @property + def text(self): + return str(self.content, self._encoding) + + def json(self): + try: + import ujson + result = ujson.load(self._socket) + return result + finally: + self.close() + + +class HttpClient: + + def __init__(self, headers={}): + self._headers = headers + + def is_chunked_data(data): + return getattr(data, "__iter__", None) and not getattr(data, "__len__", None) + + def request(self, method, url, data=None, json=None, file=None, custom=None, saveToFile=None, headers={}, stream=None): + chunked = data and self.is_chunked_data(data) + redirect = None #redirection url, None means no redirection + def _write_headers(sock, _headers): + for k in _headers: + sock.write(b'{}: {}\r\n'.format(k, _headers[k])) + + try: + proto, dummy, host, path = url.split('/', 3) + except ValueError: + proto, dummy, host = url.split('/', 2) + path = '' + if proto == 'http:': + port = 80 + elif proto == 'https:': + import ussl + port = 443 + else: + raise ValueError('Unsupported protocol: ' + proto) + + if ':' in host: + host, port = host.split(':', 1) + port = int(port) + + ai = usocket.getaddrinfo(host, port, 0, usocket.SOCK_STREAM) + if len(ai) < 1: + raise ValueError('You are not connected to the internet...') + ai = ai[0] + + s = usocket.socket(ai[0], ai[1], ai[2]) + try: + s.connect(ai[-1]) + if proto == 'https:': + gc.collect() + s = ussl.wrap_socket(s, server_hostname=host) + s.write(b'%s /%s HTTP/1.0\r\n' % (method, path)) + if not 'Host' in headers: + s.write(b'Host: %s\r\n' % host) + # Iterate over keys to avoid tuple alloc + _write_headers(s, self._headers) + _write_headers(s, headers) + + # add user agent + s.write(b'User-Agent: MicroPython Client\r\n') + if json is not None: + assert data is None + import ujson + data = ujson.dumps(json) + s.write(b'Content-Type: application/json\r\n') + + if data: + if chunked: + s.write(b"Transfer-Encoding: chunked\r\n") + else: + s.write(b"Content-Length: %d\r\n" % len(data)) + s.write(b"\r\n") + if data: + if chunked: + for chunk in data: + s.write(b"%x\r\n" % len(chunk)) + s.write(chunk) + s.write(b"\r\n") + s.write("0\r\n\r\n") + else: + s.write(data) + elif file: + s.write(b'Content-Length: %d\r\n' % os.stat(file)[6]) + s.write(b'\r\n') + with open(file, 'r') as file_object: + for line in file_object: + s.write(line + '\n') + elif custom: + custom(s) + else: + s.write(b'\r\n') + + l = s.readline() + #print('l: ', l) + l = l.split(None, 2) + status = int(l[1]) + reason = '' + if len(l) > 2: + reason = l[2].rstrip() + while True: + l = s.readline() + if not l or l == b'\r\n': + break + #print('l: ', l) + if l.startswith(b'Transfer-Encoding:'): + if b'chunked' in l: + raise ValueError('Unsupported ' + l) + elif l.startswith(b'Location:') and not 200 <= status <= 299: + if status in [301, 302, 303, 307, 308]: + redirect = l[10:-2].decode() + else: + raise NotImplementedError("Redirect {} not yet supported".format(status)) + except OSError: + s.close() + raise + + if redirect: + s.close() + if status in [301, 302, 303]: + return self.request('GET', url=redirect) + else: + return self.request(method, redirect) + else: + resp = Response(s,saveToFile) + resp.status_code = status + resp.reason = reason + return resp + + def head(self, url, **kw): + return self.request('HEAD', url, **kw) + + def get(self, url, **kw): + return self.request('GET', url, **kw) + + def post(self, url, **kw): + return self.request('POST', url, **kw) + + def put(self, url, **kw): + return self.request('PUT', url, **kw) + + def patch(self, url, **kw): + return self.request('PATCH', url, **kw) + + def delete(self, url, **kw): + return self.request('DELETE', url, **kw) \ No newline at end of file diff --git a/micro/app/ota/ota_updater.py b/micro/app/ota/ota_updater.py new file mode 100644 index 0000000..af12e7b --- /dev/null +++ b/micro/app/ota/ota_updater.py @@ -0,0 +1,251 @@ +import os, gc +from .httpclient import HttpClient + +class OTAUpdater: + """ + A class to update your MicroController with the latest version from a GitHub tagged release, + optimized for low power usage. + """ + + def __init__(self, github_repo, github_src_dir='', module='', main_dir='main', new_version_dir='next', secrets_file=None, headers={}): + self.http_client = HttpClient(headers=headers) + self.github_repo = github_repo.rstrip('/').replace('https://github.com/', '') + self.github_src_dir = '' if len(github_src_dir) < 1 else github_src_dir.rstrip('/') + '/' + self.module = module.rstrip('/') + self.main_dir = main_dir + self.new_version_dir = new_version_dir + self.secrets_file = secrets_file + + def __del__(self): + self.http_client = None + + def check_for_update_to_install_during_next_reboot(self) -> bool: + """Function which will check the GitHub repo if there is a newer version available. + + This method expects an active internet connection and will compare the current + version with the latest version available on GitHub. + If a newer version is available, the file 'next/.version' will be created + and you need to call machine.reset(). A reset is needed as the installation process + takes up a lot of memory (mostly due to the http stack) + + Returns + ------- + bool: true if a new version is available, false otherwise + """ + + (current_version, latest_version) = self._check_for_new_version() + if latest_version > current_version: + print('New version available, will download and install on next reboot') + self._create_new_version_file(latest_version) + return True + + return False + + def install_update_if_available_after_boot(self, ssid, password) -> bool: + """This method will install the latest version if out-of-date after boot. + + This method, which should be called first thing after booting, will check if the + next/.version' file exists. + + - If yes, it initializes the WIFI connection, downloads the latest version and installs it + - If no, the WIFI connection is not initialized as no new known version is available + """ + + if self.new_version_dir in os.listdir(self.module): + if '.version' in os.listdir(self.modulepath(self.new_version_dir)): + latest_version = self.get_version(self.modulepath(self.new_version_dir), '.version') + print('New update found: ', latest_version) + OTAUpdater._using_network(ssid, password) + self.install_update_if_available() + return True + + print('No new updates found...') + return False + + def install_update_if_available(self) -> bool: + """This method will immediately install the latest version if out-of-date. + + This method expects an active internet connection and allows you to decide yourself + if you want to install the latest version. It is necessary to run it directly after boot + (for memory reasons) and you need to restart the microcontroller if a new version is found. + + Returns + ------- + bool: true if a new version is available, false otherwise + """ + + (current_version, latest_version) = self._check_for_new_version() + if latest_version > current_version: + print('Updating to version {}...'.format(latest_version)) + self._create_new_version_file(latest_version) + self._download_new_version(latest_version) + self._copy_secrets_file() + self._delete_old_version() + self._install_new_version() + return True + + return False + + + @staticmethod + def _using_network(ssid, password): + import network + sta_if = network.WLAN(network.STA_IF) + if not sta_if.isconnected(): + print('connecting to network...') + sta_if.active(True) + sta_if.connect(ssid, password) + while not sta_if.isconnected(): + pass + print('network config:', sta_if.ifconfig()) + + def _check_for_new_version(self): + current_version = self.get_version(self.modulepath(self.main_dir)) + latest_version = self.get_latest_version() + + print('Checking version... ') + print('\tCurrent version: ', current_version) + print('\tLatest version: ', latest_version) + return (current_version, latest_version) + + def _create_new_version_file(self, latest_version): + self.mkdir(self.modulepath(self.new_version_dir)) + with open(self.modulepath(self.new_version_dir + '/.version'), 'w') as versionfile: + versionfile.write(latest_version) + versionfile.close() + + def get_version(self, directory, version_file_name='.version'): + print(directory + '/' + version_file_name) + if version_file_name in os.listdir(directory): + with open(directory + '/' + version_file_name) as f: + version = f.read() + return version + return '0.0' + + def get_latest_version(self): + latest_release = self.http_client.get('https://api.github.com/repos/{}/releases/latest'.format(self.github_repo)) + gh_json = latest_release.json() + try: + version = gh_json['tag_name'] + except KeyError as e: + raise ValueError( + "Release not found: \n", + "Please ensure release as marked as 'latest', rather than pre-release \n", + "github api message: \n {} \n ".format(gh_json) + ) + latest_release.close() + return version + + def _download_new_version(self, version): + print('Downloading version {}'.format(version)) + self._download_all_files(version) + print('Version {} downloaded to {}'.format(version, self.modulepath(self.new_version_dir))) + + def _download_all_files(self, version, sub_dir=''): + url = 'https://api.github.com/repos/{}/contents{}{}{}?ref=refs/tags/{}'.format(self.github_repo, self.github_src_dir, self.main_dir, sub_dir, version) + gc.collect() + file_list = self.http_client.get(url) + file_list_json = file_list.json() + for file in file_list_json: + path = self.modulepath(self.new_version_dir + '/' + file['path'].replace(self.main_dir + '/', '').replace(self.github_src_dir, '')) + if file['type'] == 'file': + gitPath = file['path'] + print('\tDownloading: ', gitPath, 'to', path) + self._download_file(version, gitPath, path) + elif file['type'] == 'dir': + print('Creating dir', path) + self.mkdir(path) + self._download_all_files(version, sub_dir + '/' + file['name']) + gc.collect() + + file_list.close() + + def _download_file(self, version, gitPath, path): + self.http_client.get('https://raw.githubusercontent.com/{}/{}/{}'.format(self.github_repo, version, gitPath), saveToFile=path) + + def _copy_secrets_file(self): + if self.secrets_file: + fromPath = self.modulepath(self.main_dir + '/' + self.secrets_file) + toPath = self.modulepath(self.new_version_dir + '/' + self.secrets_file) + print('Copying secrets file from {} to {}'.format(fromPath, toPath)) + self._copy_file(fromPath, toPath) + print('Copied secrets file from {} to {}'.format(fromPath, toPath)) + + def _delete_old_version(self): + print('Deleting old version at {} ...'.format(self.modulepath(self.main_dir))) + self._rmtree(self.modulepath(self.main_dir)) + print('Deleted old version at {} ...'.format(self.modulepath(self.main_dir))) + + def _install_new_version(self): + print('Installing new version at {} ...'.format(self.modulepath(self.main_dir))) + if self._os_supports_rename(): + os.rename(self.modulepath(self.new_version_dir), self.modulepath(self.main_dir)) + else: + self._copy_directory(self.modulepath(self.new_version_dir), self.modulepath(self.main_dir)) + self._rmtree(self.modulepath(self.new_version_dir)) + print('Update installed, please reboot now') + + def _rmtree(self, directory): + for entry in os.ilistdir(directory): + is_dir = entry[1] == 0x4000 + if is_dir: + self._rmtree(directory + '/' + entry[0]) + else: + os.remove(directory + '/' + entry[0]) + os.rmdir(directory) + + def _os_supports_rename(self) -> bool: + self._mk_dirs('otaUpdater/osRenameTest') + os.rename('otaUpdater', 'otaUpdated') + result = len(os.listdir('otaUpdated')) > 0 + self._rmtree('otaUpdated') + return result + + def _copy_directory(self, fromPath, toPath): + if not self._exists_dir(toPath): + self._mk_dirs(toPath) + + for entry in os.ilistdir(fromPath): + is_dir = entry[1] == 0x4000 + if is_dir: + self._copy_directory(fromPath + '/' + entry[0], toPath + '/' + entry[0]) + else: + self._copy_file(fromPath + '/' + entry[0], toPath + '/' + entry[0]) + + def _copy_file(self, fromPath, toPath): + with open(fromPath) as fromFile: + with open(toPath, 'w') as toFile: + CHUNK_SIZE = 512 # bytes + data = fromFile.read(CHUNK_SIZE) + while data: + toFile.write(data) + data = fromFile.read(CHUNK_SIZE) + toFile.close() + fromFile.close() + + def _exists_dir(self, path) -> bool: + try: + os.listdir(path) + return True + except: + return False + + def _mk_dirs(self, path:str): + paths = path.split('/') + + pathToCreate = '' + for x in paths: + self.mkdir(pathToCreate + x) + pathToCreate = pathToCreate + x + '/' + + # different micropython versions act differently when directory already exists + def mkdir(self, path:str): + try: + os.mkdir(path) + except OSError as exc: + if exc.args[0] == 17: + pass + + + def modulepath(self, path): + return self.module + '/' + path if self.module else path \ No newline at end of file diff --git a/micro/server.py b/micro/app/server.py similarity index 96% rename from micro/server.py rename to micro/app/server.py index 3905e34..cbfeade 100644 --- a/micro/server.py +++ b/micro/app/server.py @@ -1,8 +1,8 @@ import _thread as thread -import uwebsockets.client -from config import get_device_id import json from time import sleep_ms +import app.uwebsockets.client +from app.config import get_device_id # TODO: Move IO operations to a separate file from machine import Pin, TouchPad diff --git a/micro/startup.py b/micro/app/startup.py similarity index 66% rename from micro/startup.py rename to micro/app/startup.py index a34a366..3c7d1b0 100644 --- a/micro/startup.py +++ b/micro/app/startup.py @@ -1,11 +1,23 @@ -from wifi import connect_wifi, disconnect_wifi, connect_wifi_from_config -from bluetooth import run_setup -from config import load_config, get_config_item -from server import Server +import machine +import gc +from app.wifi import connect_wifi, disconnect_wifi, connect_wifi_from_config +from app.bluetooth import run_setup +from app.config import load_config, get_config_item +from app.server import Server +from app.ota.ota_updater import OTAUpdater def start_setup_mode(): run_setup() +def check_update_and_install(): + otaUpdater = OTAUpdater('https://github.com/alexwohlbruck/covalent', github_src_dir='micro', main_dir='app') + hasUpdated = otaUpdater.install_update_if_available() + if hasUpdated: + machine.reset() + else: + del(otaUpdater) + gc.collect() + from time import sleep, sleep_ms from machine import Pin, TouchPad @@ -21,6 +33,9 @@ def run_startup(): if not wifi_success: start_setup_mode() + # Install updates if available + check_update_and_install() + print ('getting lamp id') # Internet successfully connected @@ -49,7 +64,8 @@ def button_released(pin): last_val = False while (True): - val = touchpad.read() < 250 + sensor = touchpad.read() + val = sensor < 250 if val: if not last_val: button_pressed(None) diff --git a/micro/usocketio/client.py b/micro/app/usocketio/client.py similarity index 100% rename from micro/usocketio/client.py rename to micro/app/usocketio/client.py diff --git a/micro/usocketio/protocol.py b/micro/app/usocketio/protocol.py similarity index 100% rename from micro/usocketio/protocol.py rename to micro/app/usocketio/protocol.py diff --git a/micro/usocketio/transport.py b/micro/app/usocketio/transport.py similarity index 99% rename from micro/usocketio/transport.py rename to micro/app/usocketio/transport.py index ed489e1..bbd3f52 100644 --- a/micro/usocketio/transport.py +++ b/micro/app/usocketio/transport.py @@ -3,7 +3,7 @@ import ujson as json import usocket as socket -import uwebsockets.client +import app.uwebsockets.client from .protocol import * class SocketIO: diff --git a/micro/uwebsockets/client.py b/micro/app/uwebsockets/client.py similarity index 100% rename from micro/uwebsockets/client.py rename to micro/app/uwebsockets/client.py diff --git a/micro/uwebsockets/protocol.py b/micro/app/uwebsockets/protocol.py similarity index 99% rename from micro/uwebsockets/protocol.py rename to micro/app/uwebsockets/protocol.py index d7f1a06..18c2eb7 100644 --- a/micro/uwebsockets/protocol.py +++ b/micro/app/uwebsockets/protocol.py @@ -240,6 +240,6 @@ def close(self, code=CLOSE_OK, reason=''): self._close() def _close(self): - if __debug__: LOGGER.debug("Connection closed") + if __debug__: print("Connection closed") self.open = False self.sock.close() \ No newline at end of file diff --git a/micro/wifi.py b/micro/app/wifi.py similarity index 97% rename from micro/wifi.py rename to micro/app/wifi.py index 0ab5dc7..7da88c7 100644 --- a/micro/wifi.py +++ b/micro/app/wifi.py @@ -1,6 +1,6 @@ import network from time import sleep -from config import add_config, load_config +from app.config import add_config, load_config sta_if = network.WLAN(network.STA_IF) sta_if.active(True) diff --git a/micro/main.py b/micro/main.py index 7404982..817a809 100644 --- a/micro/main.py +++ b/micro/main.py @@ -1,4 +1,4 @@ -from startup import run_startup +from app.startup import run_startup def main(): run_startup()