From c42b956c6d57c5d863246a73870be6038f6e4c1e Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Thu, 12 Oct 2023 12:58:07 +0200 Subject: [PATCH 1/6] feat(netsim): added support for playbooks --- netsim/main.py | 80 ++++++++++++++------- netsim/playbooks/get.py | 61 ++++++++++++++++ netsim/playbooks/playbook_client.py | 54 ++++++++++++++ netsim/playbooks/playbook_engine.py | 105 ++++++++++++++++++++++++++++ netsim/playbooks/requirements.txt | 2 + netsim/playbooks/serve.py | 57 +++++++++++++++ netsim/sims/example/playbook.json | 55 +++++++++++++++ netsim/venv/.gitkeep | 0 8 files changed, 388 insertions(+), 26 deletions(-) create mode 100644 netsim/playbooks/get.py create mode 100644 netsim/playbooks/playbook_client.py create mode 100644 netsim/playbooks/playbook_engine.py create mode 100644 netsim/playbooks/requirements.txt create mode 100644 netsim/playbooks/serve.py create mode 100644 netsim/sims/example/playbook.json create mode 100644 netsim/venv/.gitkeep diff --git a/netsim/main.py b/netsim/main.py index c8ae8a5..c1e8cde 100644 --- a/netsim/main.py +++ b/netsim/main.py @@ -33,7 +33,35 @@ def logs_on_error(nodes, prefix): else: print('[WARN] log file missing: %s' % log_name) +def build_cmd(node, i, node_ips, node_params, node_counts): + cmd = node['cmd'] + if 'param' in node: + if node['param'] == 'id': + cmd = cmd % i + if node['connect']['strategy'] == 'plain': + cnt = node_counts[node['connect']['node']] + id = i % cnt + connect_to = '%s_%d' % (node['connect']['node'], id) + ip = node_ips[connect_to] + cmd = cmd % ip + if node['connect']['strategy'] == 'plain_with_id': + cnt = node_counts[node['connect']['node']] + id = i % cnt + connect_to = '%s_%d' % (node['connect']['node'], id) + ip = node_ips[connect_to] + cmd = cmd % (ip, id) + if node['connect']['strategy'] == 'params': + cnt = node_counts[node['connect']['node']] + id = i % cnt + connect_to = '%s_%d' % (node['connect']['node'], id) + param = node_params[connect_to] + cmd = cmd % (param) + return cmd + + def run(nodes, prefix, args, debug=False, full_debug=False, visualize=False): + nodes = sorted(nodes, key=lambda k: (k.get('position', 1000000), k['name'])) + print(nodes) integration = args.integration topo = StarTopo(nodes=nodes) net = Mininet(topo = topo, waitConnected=True, link=TCLink) @@ -72,42 +100,39 @@ def run(nodes, prefix, args, debug=False, full_debug=False, visualize=False): temp_dirs = [] + ftc = [] + for node in nodes: node_counts[node['name']] = int(node['count']) for i in range(int(node['count'])): node_name = '%s_%d' % (node['name'], i) f = open('logs/%s__%s.txt' % (prefix, node_name), 'w+') + ftc.append(f) n = net.get(node_name) node_ips[node_name] = n.IP() - cmd = node['cmd'] - if 'param' in node: - if node['param'] == 'id': - cmd = cmd % i - if node['connect']['strategy'] == 'plain': - cnt = node_counts[node['connect']['node']] - id = i % cnt - connect_to = '%s_%d' % (node['connect']['node'], id) - ip = node_ips[connect_to] - cmd = cmd % ip - if node['connect']['strategy'] == 'plain_with_id': - cnt = node_counts[node['connect']['node']] - id = i % cnt - connect_to = '%s_%d' % (node['connect']['node'], id) - ip = node_ips[connect_to] - cmd = cmd % (ip, id) - if node['connect']['strategy'] == 'params': - cnt = node_counts[node['connect']['node']] - id = i % cnt - connect_to = '%s_%d' % (node['connect']['node'], id) - param = node_params[connect_to] - cmd = cmd % (param) + cmd = "" + if 'cmd' in node: + cmd = build_cmd(node, i, node_ips, node_params, node_counts) + elif 'playbook' in node: + requirements_path = node['playbook']['requirements'] + playbook_path = node['playbook']['path'] + cmd = f"source venv/bin/activate && pip install -r playbooks/{requirements_path} && python3 playbooks/{playbook_path}" + if node['connect']['strategy'] == 'plain': + cnt = node_counts[node['connect']['node']] + id = i % cnt + connect_to = '%s_%d' % (node['connect']['node'], id) + ip = node_ips[connect_to] + cmd = cmd + ' ' + ip + else: + print("error: no command or playbook specified") + exit(1) # cleanup_run = subprocess.run("sudo rm -rf /root/.local/share/iroh", shell=True, capture_output=True) time.sleep(0.1) env_vars['SSLKEYLOGFILE']= './logs/keylog_%s_%s.txt' % (prefix, node_name) temp_dir = tempfile.TemporaryDirectory(prefix='netsim', suffix='{}_{}'.format(prefix, node_name)) temp_dirs.append(temp_dir) - env_vars['IROH_DATA_DIR'] = '{}'.format(temp_dir) + env_vars['IROH_DATA_DIR'] = '{}'.format(temp_dir.name) p = n.popen(cmd, stdout=f, stderr=f, shell=True, env=env_vars) if 'process' in node and node['process'] == 'short': @@ -126,15 +151,15 @@ def run(nodes, prefix, args, debug=False, full_debug=False, visualize=False): found = 0 node_name = '%s_%d' % (node['name'], zz) n = net.get(node_name) - f = open('logs/%s__%s.txt' % (prefix, node_name), 'r') - lines = f.readlines() + fx = open('logs/%s__%s.txt' % (prefix, node_name), 'r') + lines = fx.readlines() for line in lines: if node['param_parser'] == 'iroh_ticket': if line.startswith('All-in-one ticket'): node_params[node_name] = line[len('All-in-one ticket: '):].strip() found+=1 break - f.close() + fx.close() if found == int(node['count']): done_wait = True break @@ -163,6 +188,9 @@ def run(nodes, prefix, args, debug=False, full_debug=False, visualize=False): p.terminate() for p in p_box: p.terminate() + for f in ftc: + f.flush() + f.close() net.stop() sniffer.close() diff --git a/netsim/playbooks/get.py b/netsim/playbooks/get.py new file mode 100644 index 0000000..212119f --- /dev/null +++ b/netsim/playbooks/get.py @@ -0,0 +1,61 @@ +import os +import playbook_client as pbc +import iroh +import time +import argparse +import logging +import sys + +iroh.set_log_level(iroh.LogLevel.DEBUG) + +# Configure the logging system +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + stream=sys.stderr +) + +parser = argparse.ArgumentParser(description="Test param handler") +parser.add_argument("--syncer", help="Syncer address") +args = parser.parse_args() + +# syncer = pbc.Syncer(addr="http://127.0.0.1:8000") +syncer = pbc.Syncer(addr=f"http://{args.syncer}:8000") + +IROH_DATA_DIR = os.environ.get("IROH_DATA_DIR") + +if IROH_DATA_DIR is None: + logging.info("IROH_DATA_DIR is not set") + exit(1) + +node = iroh.IrohNode(IROH_DATA_DIR) +logging.info("Started Iroh node: {}".format(node.node_id())) + +syncer.set("srv_node_id", node.node_id()) + +author = node.author_new() +logging.info("Created author: {}".format(author.to_string())) + +ticket = syncer.wait_for("ticket") + +doc_ticket = iroh.DocTicket.from_string(ticket) +doc = node.doc_join(doc_ticket) +logging.info("Joined doc: {}".format(doc.id())) + +syncer.set("get_ready", 1) + +time.sleep(15) + +keys = doc.keys() +logging.info("Keys:") +for key in keys: + content = doc.get_content_bytes(key) + logging.info("{} : {} (hash: {})".format(key.key(), content.decode("utf8"), key.hash().to_string())) + +logging.info("Done logging.infoing keys...") + +syncer.inc("test_done") +time.sleep(5) +syncer.inc("test_done") + +logging.info("Done") \ No newline at end of file diff --git a/netsim/playbooks/playbook_client.py b/netsim/playbooks/playbook_client.py new file mode 100644 index 0000000..233aa15 --- /dev/null +++ b/netsim/playbooks/playbook_client.py @@ -0,0 +1,54 @@ +import requests +import json +import time +import logging + +# The playbook_engine should always be the first node to spawn +default_url = "http://10.0.0.1:8000" + +class Syncer: + def __init__(self, addr=default_url, run_id="", tick_interval = 0.5, timeout=60): + self.addr = addr + self.run_id = run_id + self.tick_interval = tick_interval + self.timeout = timeout + + def set(self, key, value): + key = f"{self.run_id}_{key}" + data = {"key": key, "value": value} + response = requests.post(self.addr, data=json.dumps(data), headers={"Content-Type": "application/json"}) + if response.status_code != 200: + logging.info("Error setting value:", response.text) + + def inc(self, key): + key = f"{self.run_id}_{key}" + response = requests.put(f"{self.addr}/{key}") + if response.status_code != 200: + logging.info("Error incrementing value:", response.text) + + def wait_for(self, key): + key = f"{self.run_id}_{key}" + start = time.time() + while True: + logging.info("wating 1...") + if time.time() - start > self.timeout: + raise Exception(f"Timeout waiting for {key}") + response = requests.get(f"{self.addr}/{key}") + if response.status_code == 200: + value = json.loads(response.text) + return value + time.sleep(self.tick_interval) + + def wait_for_val(self, key, val): + key = f"{self.run_id}_{key}" + start = time.time() + while True: + logging.info("wating 2...") + if time.time() - start > self.timeout: + raise Exception(f"Timeout waiting for {key} to be {val}") + response = requests.get(f"{self.addr}/{key}") + if response.status_code == 200: + value = json.loads(response.text) + if value == val: + return value + time.sleep(self.tick_interval) \ No newline at end of file diff --git a/netsim/playbooks/playbook_engine.py b/netsim/playbooks/playbook_engine.py new file mode 100644 index 0000000..d42e45d --- /dev/null +++ b/netsim/playbooks/playbook_engine.py @@ -0,0 +1,105 @@ +from http.server import BaseHTTPRequestHandler, HTTPServer +import json +from threading import Lock +import logging +import sys + +# Configure the logging system +logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s [%(levelname)s] %(message)s", + stream=sys.stderr +) + +# Define the key-value store +store = {} +lock = Lock() + +# Define the HTTP request handler +class KeyValueStoreHandler(BaseHTTPRequestHandler): + def do_GET(self): + with lock: + try: + # Get the key from the URL path + key = self.path[1:] + + # Get the value from the store + value = store.get(key) + + if value is None: + # Send a 404 response + self.send_response(404) + self.end_headers() + return + + # Send the value as JSON + self.send_response(200) + self.send_header("Content-type", "application/json") + self.end_headers() + self.wfile.write(json.dumps(value).encode()) + except Exception as e: + logging.error(f"unexpected error: {e}") + + def do_POST(self): + with lock: + try: + # Get the key and value from the request body + content_length = int(self.headers["Content-Length"]) + body = self.rfile.read(content_length) + data = json.loads(body.decode()) + key = data["key"] + value = data["value"] + + # Set the value in the store + store[key] = value + + # Send a success response + self.send_response(200) + self.end_headers() + except: + try: + # Send a 500 response + self.send_response(500) + self.end_headers() + except Exception as e: + logging.error(f"unexpected error: {e}") + + def do_PUT(self): + with lock: + try: + # Get the key from the URL path + key = self.path[1:] + + # Get the value from the store + value = store.get(key) + + if value is None: + value = 0 + + # Increment the value + value += 1 + + # Set the new value in the store + store[key] = value + + # Send the new value as JSON + self.send_response(200) + self.send_header("Content-type", "application/json") + self.end_headers() + self.wfile.write(json.dumps(value).encode()) + except: + try: + # Send a 500 response + self.send_response(500) + self.end_headers() + except Exception as e: + logging.error(f"unexpected error: {e}") + +if __name__ == '__main__': + # Define the HTTP server + server_address = ("", 8000) + httpd = HTTPServer(server_address, KeyValueStoreHandler) + + # Start the HTTP server + logging.info("Starting HTTP server...") + httpd.serve_forever() \ No newline at end of file diff --git a/netsim/playbooks/requirements.txt b/netsim/playbooks/requirements.txt new file mode 100644 index 0000000..02d2c0a --- /dev/null +++ b/netsim/playbooks/requirements.txt @@ -0,0 +1,2 @@ +iroh==0.2.0 +requests==2.31.0 \ No newline at end of file diff --git a/netsim/playbooks/serve.py b/netsim/playbooks/serve.py new file mode 100644 index 0000000..d91d3c7 --- /dev/null +++ b/netsim/playbooks/serve.py @@ -0,0 +1,57 @@ +import os +import playbook_client as pbc +import iroh +import argparse +import logging +import sys + +iroh.set_log_level(iroh.LogLevel.DEBUG) + +# Configure the logging system +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + stream=sys.stderr +) + +parser = argparse.ArgumentParser(description="Test param handler") +parser.add_argument("--syncer", help="Syncer address") +args = parser.parse_args() + +# syncer = pbc.Syncer(addr="http://127.0.0.1:8000") +syncer = pbc.Syncer(addr=f"http://{args.syncer}:8000") + +IROH_DATA_DIR = os.environ.get("IROH_DATA_DIR") +logging.info("IROH_DATA_DIR: {}".format(IROH_DATA_DIR)) + +if IROH_DATA_DIR is None: + logging.info("IROH_DATA_DIR is not set") + exit(1) + +logging.info("Starting Iroh node...") + +node = iroh.IrohNode(IROH_DATA_DIR) +logging.info("Started Iroh node: {}".format(node.node_id())) + +syncer.set("srv_node_id", node.node_id()) + +author = node.author_new() +logging.info("Created author: {}".format(author.to_string())) + +doc = node.doc_new() +logging.info("Created doc: {}".format(doc.id())) + +ticket = doc.share_write() +logging.info("Write-Access Ticket: {}".format(ticket.to_string())) + +syncer.set("ticket", ticket.to_string()) + +syncer.wait_for("get_ready") +logging.info("Ready!") + +hash = doc.set_bytes(author, bytes("foo", "utf8"), bytes("bar", "utf8")) +logging.info("Inserted: {}".format(hash.to_string())) + +syncer.wait_for_val("test_done", "2") + +logging.info("Done") \ No newline at end of file diff --git a/netsim/sims/example/playbook.json b/netsim/sims/example/playbook.json new file mode 100644 index 0000000..2da7f89 --- /dev/null +++ b/netsim/sims/example/playbook.json @@ -0,0 +1,55 @@ +{ + "name": "iroh", + "cases": [ + { + "name": "1_to_1", + "description": "", + "nodes": [ + { + "position": 0, + "name": "syncer", + "count": 1, + "playbook": { + "requirements": "requirements.txt", + "path": "playbook_engine.py" + }, + "type": "public", + "wait": 5, + "connect": { + "strategy": "none" + }, + "param": "id" + }, + { + "position": 1, + "name": "iroh_srv", + "count": 1, + "playbook": { + "requirements": "requirements.txt", + "path": "serve.py --syncer" + }, + "type": "public", + "wait": 5, + "connect": { + "strategy": "plain", + "node": "syncer" + } + }, + { + "name": "iroh_get", + "count": 1, + "playbook": { + "requirements": "requirements.txt", + "path": "get.py --syncer" + }, + "type": "public", + "connect": { + "strategy": "plain", + "node": "syncer" + }, + "process": "short" + } + ] + } + ] +} \ No newline at end of file diff --git a/netsim/venv/.gitkeep b/netsim/venv/.gitkeep new file mode 100644 index 0000000..e69de29 From 7008189c066cdd15427c5902d14f063607d71b4a Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 13 Oct 2023 09:44:28 +0200 Subject: [PATCH 2/6] chore(ci): test ffi --- .github/workflows/netsim_integration.yml | 17 +++++++++++++++++ netsim/playbooks/requirements.txt | 2 +- netsim/sims/example/playbook.json | 2 +- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/netsim_integration.yml b/.github/workflows/netsim_integration.yml index 810e4c3..5a9c2d2 100644 --- a/.github/workflows/netsim_integration.yml +++ b/.github/workflows/netsim_integration.yml @@ -45,11 +45,27 @@ jobs: git clone https://github.com/n0-computer/iroh.git cd iroh cargo build --release + + - name: Fetch and build iroh-ffi + run: | + git clone https://github.com/n0-computer/iroh-ffi.git + cd iroh-ffi + echo "iroh = { path = \"../iroh\" }" >> Cargo.toml + pip3 install maturin uniffi-bindgen + maturin build --release - name: Copy binaries to right location run: | cp target/release/chuck netsim/bins/chuck cp iroh/target/release/iroh netsim/bins/iroh + cp iroh-ffi/target/wheels/iroh-*-py3-none-manylinux_2_34_x86_64.whl ./netsim/bins/ + + - name: Setup python venv + run: | + cd netsim + python3 -m venv venv + source venv/bin/activate + pip3 install bins/iroh-*.whl - name: Run tests run: | @@ -57,6 +73,7 @@ jobs: sudo kill -9 $(pgrep ovs) sudo mn --clean sudo python3 main.py --integration sims/standard/iroh.json + sudo python3 main.py --integration sims/example/playbook.json - name: Setup Environment (PR) if: ${{ github.event_name == 'pull_request' }} diff --git a/netsim/playbooks/requirements.txt b/netsim/playbooks/requirements.txt index 02d2c0a..1301721 100644 --- a/netsim/playbooks/requirements.txt +++ b/netsim/playbooks/requirements.txt @@ -1,2 +1,2 @@ -iroh==0.2.0 +# iroh==0.2.0 requests==2.31.0 \ No newline at end of file diff --git a/netsim/sims/example/playbook.json b/netsim/sims/example/playbook.json index 2da7f89..439df69 100644 --- a/netsim/sims/example/playbook.json +++ b/netsim/sims/example/playbook.json @@ -1,5 +1,5 @@ { - "name": "iroh", + "name": "iroh_playbook", "cases": [ { "name": "1_to_1", From 3da81bfb6ff8a5b930d3f9d22034a45b6f9bc585 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 13 Oct 2023 11:57:29 +0200 Subject: [PATCH 3/6] fix --- netsim/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/netsim/main.py b/netsim/main.py index c1e8cde..ea89d40 100644 --- a/netsim/main.py +++ b/netsim/main.py @@ -60,8 +60,6 @@ def build_cmd(node, i, node_ips, node_params, node_counts): def run(nodes, prefix, args, debug=False, full_debug=False, visualize=False): - nodes = sorted(nodes, key=lambda k: (k.get('position', 1000000), k['name'])) - print(nodes) integration = args.integration topo = StarTopo(nodes=nodes) net = Mininet(topo = topo, waitConnected=True, link=TCLink) From a5b4712be03763b2297f289b7bff16c4dea610bd Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 13 Oct 2023 12:17:14 +0200 Subject: [PATCH 4/6] deps --- netsim/playbooks/requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/netsim/playbooks/requirements.txt b/netsim/playbooks/requirements.txt index 1301721..663bd1f 100644 --- a/netsim/playbooks/requirements.txt +++ b/netsim/playbooks/requirements.txt @@ -1,2 +1 @@ -# iroh==0.2.0 -requests==2.31.0 \ No newline at end of file +requests \ No newline at end of file From 9bf2b44a905e30a2b89fd316374d5c999f601e77 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 13 Oct 2023 12:34:09 +0200 Subject: [PATCH 5/6] deps --- .github/workflows/netsim_integration.yml | 1 + netsim/main.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/netsim_integration.yml b/.github/workflows/netsim_integration.yml index 5a9c2d2..9fc295c 100644 --- a/.github/workflows/netsim_integration.yml +++ b/.github/workflows/netsim_integration.yml @@ -66,6 +66,7 @@ jobs: python3 -m venv venv source venv/bin/activate pip3 install bins/iroh-*.whl + pip3 install -r playbooks/{requirements_path} - name: Run tests run: | diff --git a/netsim/main.py b/netsim/main.py index ea89d40..482f2d3 100644 --- a/netsim/main.py +++ b/netsim/main.py @@ -114,7 +114,7 @@ def run(nodes, prefix, args, debug=False, full_debug=False, visualize=False): elif 'playbook' in node: requirements_path = node['playbook']['requirements'] playbook_path = node['playbook']['path'] - cmd = f"source venv/bin/activate && pip install -r playbooks/{requirements_path} && python3 playbooks/{playbook_path}" + cmd = f"source venv/bin/activate && python3 playbooks/{playbook_path}" if node['connect']['strategy'] == 'plain': cnt = node_counts[node['connect']['node']] id = i % cnt From d58894bc8750cbb76e04e2519d6e31b862dc5022 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 13 Oct 2023 12:36:16 +0200 Subject: [PATCH 6/6] deps --- .github/workflows/netsim_integration.yml | 2 +- netsim/playbooks/requirements.txt | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/netsim_integration.yml b/.github/workflows/netsim_integration.yml index 9fc295c..70decfa 100644 --- a/.github/workflows/netsim_integration.yml +++ b/.github/workflows/netsim_integration.yml @@ -66,7 +66,7 @@ jobs: python3 -m venv venv source venv/bin/activate pip3 install bins/iroh-*.whl - pip3 install -r playbooks/{requirements_path} + pip3 install -r playbooks/requirements.txt - name: Run tests run: | diff --git a/netsim/playbooks/requirements.txt b/netsim/playbooks/requirements.txt index 663bd1f..1301721 100644 --- a/netsim/playbooks/requirements.txt +++ b/netsim/playbooks/requirements.txt @@ -1 +1,2 @@ -requests \ No newline at end of file +# iroh==0.2.0 +requests==2.31.0 \ No newline at end of file