Task 2: External emulator support + SCION artifact export

.github/workflows/on-pull-request.yaml

@@ -4,13 +4,48 @@ on:
   pull_request:
     branches:
       - master
+      - development
+  workflow_dispatch:
+    inputs:
+      run-quick-tests:
+        description: "Run quick tests | ignores other options for testing (they still apply for compile/build)"
+        type: boolean
+        required: false
+        default: "false"
+      run-basic-tests:
+        description: "Run basic tests | Compile and builds all the basic examples - Runs basic tests"
+        type: boolean
+        required: false
+        default: "true"
+      run-internet-tests:
+        description: "Run internet tests | Compile and builds all the internet examples - Runs internet tests"
+        type: boolean
+        required: false
+        default: "true"
+      run-blockchain-tests:
+        description: "Run blockchain tests | Compile and builds all the blockchain examples - Runs blockchain tests"
+        type: boolean
+        required: false
+        default: "false"
+      run-scion-tests:
+        description: "Run SCION tests | Compile and builds all the SCION examples - Runs SCION tests"
+        type: boolean
+        required: false
+        default: "false"
+
 
 jobs:
   compile-examples:
     name: Compile Examples
     runs-on: ubuntu-latest
     permissions:
       contents: read
+    env:
+      SCION_TESTS: ${{ github.event.inputs.run-scion-tests || 'false' }}
+      BASIC_TESTS: ${{ github.event.inputs.run-basic-tests || 'true' }}
+      INTERNET_TESTS: ${{ github.event.inputs.run-internet-tests || 'true' }}
+      BLOCKCHAIN_TESTS: ${{ github.event.inputs.run-blockchain-tests || 'false' }}
+      QUICK_TESTS: ${{ github.event.inputs.run-quick-tests || 'false' }}
     steps:
       - name: Check out the source repository
         uses: actions/checkout@v4
@@ -22,6 +57,7 @@ jobs:
           cache-dependency-path: "**/*requirements.txt"
       - run: pip install -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt
       - name: Get scion-pki
+        if: ${{ env.SCION_TESTS == 'true' }}
         run: |
           curl -fsSL -O https://github.com/scionproto/scion/releases/download/v0.12.0/scion_0.12.0_amd64_linux.tar.gz
           mkdir bin
@@ -30,14 +66,33 @@ jobs:
         run: |
           source development.env
           export PATH=$PATH:$PWD/bin
-          tests/compile-and-build-test/compile-test.py
+          CMD="tests/compile-and-build-test/compile-test.py"
+          if [ "$BASIC_TESTS" == "true" ]; then
+            CMD+=" --basic"
+          fi
+          if [ "$INTERNET_TESTS" == "true" ]; then
+            CMD+=" --internet"
+          fi
+          if [ "$BLOCKCHAIN_TESTS" == "true" ]; then
+            CMD+=" --blockchain"
+          fi
+          if [ "$SCION_TESTS" == "true" ]; then
+            CMD+=" --scion"
+          fi
+          python $CMD
 
   run-tests:
     name: Run Tests
     needs: compile-examples # Only run heavy tests if quick compile test succeeds
     runs-on: ubuntu-latest
     permissions:
       contents: read
+    env:
+      SCION_TESTS: ${{ github.event.inputs.run-scion-tests || 'false' }}
+      BASIC_TESTS: ${{ github.event.inputs.run-basic-tests || 'true' }}
+      INTERNET_TESTS: ${{ github.event.inputs.run-internet-tests || 'true' }}
+      BLOCKCHAIN_TESTS: ${{ github.event.inputs.run-blockchain-tests || 'false' }}
+      QUICK_TESTS: ${{ github.event.inputs.run-quick-tests || 'false' }}
     steps:
       - name: Check out the source repository
         uses: actions/checkout@v4
@@ -49,6 +104,7 @@ jobs:
           cache-dependency-path: "**/*requirements.txt"
       - run: pip install -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt
       - name: Get scion-pki
+        if: ${{ env.SCION_TESTS == 'true' }}
         run: |
           curl -fsSL -O https://github.com/scionproto/scion/releases/download/v0.12.0/scion_0.12.0_amd64_linux.tar.gz
           mkdir bin
@@ -57,7 +113,23 @@ jobs:
         run: |
           source development.env
           export PATH=$PATH:$PWD/bin
-          tests/run-tests.py --ci
+          CMD="tests/run-tests.py"
+          if [ "$QUICK_TESTS" == "true" ]; then
+            CMD+=" --ci"
+          fi
+          if [ "$BASIC_TESTS" == "true" ]; then
+            CMD+=" --basic"
+          fi
+          if [ "$INTERNET_TESTS" == "true" ]; then
+            CMD+=" --internet"
+          fi
+          if [ "$BLOCKCHAIN_TESTS" == "true" ]; then
+            CMD+=" --blockchain"
+          fi
+          if [ "$SCION_TESTS" == "true" ]; then
+            CMD+=" --scion"
+          fi
+          python $CMD
       - name: Archive test results
         uses: actions/upload-artifact@v4
         with:
@@ -75,6 +147,12 @@ jobs:
     runs-on: ubuntu-latest
     permissions:
       contents: read
+    env:
+      SCION_TESTS: ${{ github.event.inputs.run-scion-tests || 'false' }}
+      BASIC_TESTS: ${{ github.event.inputs.run-basic-tests || 'true' }}
+      INTERNET_TESTS: ${{ github.event.inputs.run-internet-tests || 'true' }}
+      BLOCKCHAIN_TESTS: ${{ github.event.inputs.run-blockchain-tests || 'false' }}
+      QUICK_TESTS: ${{ github.event.inputs.run-quick-tests || 'false' }}
     steps:
       - name: Check out the source repository
         uses: actions/checkout@v4
@@ -86,6 +164,7 @@ jobs:
           cache-dependency-path: "**/*requirements.txt"
       - run: pip install -r requirements.txt -r dev-requirements.txt -r tests/requirements.txt
       - name: Get scion-pki
+        if: ${{ github.event.inputs.run-scion-tests == 'true' }}
         run: |
           curl -fsSL -O https://github.com/scionproto/scion/releases/download/v0.12.0/scion_0.12.0_amd64_linux.tar.gz
           mkdir bin
@@ -95,7 +174,20 @@ jobs:
           source development.env
           export PATH=$PATH:$PWD/bin
           cd tests/compile-and-build-test
-          ./compile-and-build-test.py
+          CMD="compile-and-build-test.py"
+          if [ "$BASIC_TESTS" == "true" ]; then
+            CMD+=" --basic"
+          fi
+          if [ "$INTERNET_TESTS" == "true" ]; then
+            CMD+=" --internet"
+          fi
+          if [ "$BLOCKCHAIN_TESTS" == "true" ]; then
+            CMD+=" --blockchain"
+          fi
+          if [ "$SCION_TESTS" == "true" ]; then
+            CMD+=" --scion"
+          fi
+          python $CMD
       - name: Archive test results
         uses: actions/upload-artifact@v4
         with:

.github/workflows/sync-gitee.yaml

@@ -0,0 +1,25 @@
+name: sync2gitee
+on:
+  push:
+    branches:
+      - master
+jobs:
+  repo-sync:
+    env:
+      SSH_PRIVATE_KEY: ${{ secrets.GITEE_SSH_KEY }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+
+      - name: Configure Git
+        run: |
+           git config --global --add safe.directory /github/workspace           
+
+      - name: sync github -> gitee
+        uses: wearerequired/git-mirror-action@master
+        if: env.SSH_PRIVATE_KEY
+        with:
+          source-repo: "git@github.com:${{ github.repository }}.git"
+          destination-repo: "git@gitee.com:seedlab/seed-emulator.git"

VM_images/SeedVM24.04/src-amd/Install_Packages.sh

@@ -1,4 +1,4 @@
 git clone https://github.com/seed-labs/seed-emulator.git ~/seed-emulator
 cd ~/seed-emulator
 pip install -r ~/seed-emulator/requirements.txt
-echo 'export PYTHONPATH="/home/seed/seed-emulator:$PYTHONPATH"' > ~/.bashrc
+echo 'export PYTHONPATH="/home/seed/seed-emulator:$PYTHONPATH"' >> ~/.bashrc

VM_images/SeedVM24.04/src-arm/Install_Packages.sh

@@ -1,4 +1,4 @@
 git clone https://github.com/seed-labs/seed-emulator.git ~/seed-emulator
 cd ~/seed-emulator
 pip install -r ~/seed-emulator/requirements.txt
-echo 'export PYTHONPATH="/home/seed/seed-emulator:$PYTHONPATH"' > ~/.bashrc
+echo 'export PYTHONPATH="/home/seed/seed-emulator:$PYTHONPATH"' >> ~/.bashrc

docs/user_manual/bgp.md

@@ -82,7 +82,7 @@ and `PeerRelationship.Unfiltered` peers.
 
 We show how to allow the nodes inside an emulator to communicate
 with the machines on the real Internet. A complete example
-can be found [here](../../examples/A03-real-world/). 
+can be found [here](../../examples/basic/A03_real_world/).
 To achieve this goal, we first need to create 
 a BGP router to announce the real-world network 
 prefixes inside the emulator, so the packets going to the 

docs/user_manual/node.md

@@ -36,20 +36,33 @@ node.addSoftware("python3")
 
 Some of the software may have to be installed using some ad hoc
 commands, like `curl`, `wget`, etc. We can add the installation commands
-using the `addBuildCommand()` API. 
+using the `addBuildCommand()` and `addBuildCommandAtEnd()` APIs. 
 
 ```python
 node.addBuildCommand("curl http://example.com")
+node.addBuildCommandAtEnd("unzip file.zip")
 ```
 
-This API will lead to the addition of a
-`RUN` command in Dockerfile. When the container is built, the command
+These APIs will lead to the addition of a
+`RUN` command in Dockerfile. When the container is built, the specified command
 will be executed. 
 
 ```
 RUN curl http://example.com
+... omitted ...
+RUN unzip file.zip
 ```
 
+If we also invoke `importFile()` API, the file COPY command caused by 
+this API is actually placed after the RUN command caused by
+the `addBuildCommand()` API. Therefore, if we need to use 
+the imported file in the RUN command, we will not be able to do that.
+Because of this problem, we added a new API called 
+`addBuildCommandAtEnd()`. It will add a RUN command 
+towards the end of the `Dockerfile`, after all the 
+COPY command. 
+
+
 ## Import and create file 
 
 We may also install our own program on a node, or just simply

examples/.gitignore

@@ -0,0 +1 @@
+.ipynb_checkpoints

examples/.not_ready_examples/18-eth-private-network/README.md

@@ -39,7 +39,7 @@ e2.setBootNode(True)
 
 Note the step above is optional. If you do not set any node as boot node, all nodes in the emulation will be each other's boot nodes, which may impact performance if you have a large number of nodes.
 
-To let other nodes know the enode URL of the current node, the boot node hosts a text file with a simple HTTP server containing the URL. By default, it runs on port `8088`. If you have run some other services on the node on that port, you change change the bootnode http server port with `setBootNodeHttpPort`:
+To let other nodes know the enode URL of the current node, the boot node hosts a text file with a simple HTTP server containing the URL. By default, it runs on port `8088`. If you have run some other services on the node on that port, you change the bootnode http server port with `setBootNodeHttpPort`:
 
 ```python3
 # optionally, set boot node http server port

examples/.not_ready_examples/26-proof-of-authority/README.md

@@ -31,7 +31,7 @@
 ## Creating prefunded accounts
 
 ## component-blockchain.py output
-- After configuring your network, the output of this file should should be similar to the picture below
+- After configuring your network, the output of this file should be similar to the picture below
 
 ![Component output](images/component-blockchain-output.png)
 

examples/Z00_external_demo/topology.py

@@ -0,0 +1,61 @@
+from seedemu.compiler import Docker
+from seedemu.core import Emulator
+from seedemu.layers import Base, Routing
+from seedemu.layers.external import ExternalComponent, ExternalComponentLayer
+from seedemu.core.ExternalEmulation import ExternalEmuSpec
+
+
+def main():
+    emu = Emulator()
+    base = Base()
+    routing = Routing()
+    ext_layer = ExternalComponentLayer()
+
+    asn = 65000
+    as0 = base.createAutonomousSystem(asn)
+
+    # Internal network (so r0 has a "normal" interface)
+    as0.createNetwork("net0", prefix="10.65.0.0/24")
+    r0 = as0.createRouter("r0")
+    r0.joinNetwork("net0")
+
+    # Create IX100 and connect r0 to it with a MANUAL IP (no auto ASN->IX mapping)
+    base.createInternetExchange(100)          # creates network name "ix100"
+    r0.joinNetwork("ix100", "10.0.0.1")       # IMPORTANT: no "/24" here
+
+    # External component definition (what Task 2 is about)
+    ext = ExternalComponent(
+        name="ext-r0",
+        role="router",
+        asn=asn,
+        impl_type="generic",
+    )
+
+    ext.addInterface(
+        name="eth0",
+        network="ix100",
+        ip="10.0.0.2/24",
+        mac="02:00:00:00:00:01",
+    )
+
+    ext_layer.addComponent(ext)
+    emu.addLayer(base)
+    emu.addLayer(routing)
+    emu.addLayer(ext_layer)
+
+
+    emu.render()
+    print("Registered externals in Emulator:", list(emu.getExternalComponents().keys()))
+
+    emu.compile(Docker(), output="output", override=True)
+    print("Compiled to .\\output")
+
+    print("External component definition created:")
+    print(f"  Name: {ext.name}")
+    print(f"  Role: {ext.role}")
+    print(f"  ASN:  {ext.asn}")
+    for iface in ext.interfaces:
+        print(f"  Interface {iface.name}: network={iface.network}, ip={iface.ip}, mac={iface.mac}")
+
+if __name__ == "__main__":
+    main()

examples/basic/A00_simple_as/simple_as.py

@@ -107,9 +107,12 @@ def run(dumpfile = None):
     else:
         emu.render()
 
+        # Attach the Internet Map container to the emulator
+        docker = Docker(platform=platform) 
+
         ###############################################################################
         # Compilation
-        emu.compile(Docker(platform=platform), './output', override=True)
+        emu.compile(docker, './output', override=True)
 
 if __name__ == '__main__':
-    run()
+    run()

examples/basic/A08_buildtime_docker/Dockerfile

@@ -2,6 +2,11 @@ FROM python:slim
 
 WORKDIR /app
 
+# Install minimal build tools required to build native dependencies (e.g., ckzg)
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends build-essential && \
+    rm -rf /var/lib/apt/lists/*
+
 RUN PYTHONDONTWRITEBYTECODE=1 pip install --progress-bar off --no-cache-dir eth-account
 
 ENTRYPOINT [ "python", "-c", "import eth_account; print(eth_account.__version__)" ]

examples/basic/A10_add_containers/README.md

@@ -1,5 +1,8 @@
 # Add existing containers
 
+Note: this method is the older approach; we have introduced a new approach (listed
+in another example).
+
 In some cases, we already have a group of existing containers, 
 we would like to add them to the emulator. How do we do this? 
 In this example, the existing containers are put inside the

examples/basic/A11_add_containers_new/README.md

@@ -0,0 +1,26 @@
+# Add existing containers
+
+In some cases, we already have an existing container image, and we want
+some of the containers in the emulator to use this image. There are two
+methods to do this. 
+
+
+## Method 1
+
+In this method, we add existing containers when building the emulator. 
+We have implemented this feature in the `Docker` compiler class. The basic idea
+is to tell the compiler that we want to attach an existing container to 
+a network inside the emulator. The compiler will then generate the corresponding
+entry inside the `docker-compose.yml` file.
+
+See [this example](./method_1/) for details.
+
+
+## Method 2
+
+In this method, we add the existing containers after we have built the emulator.
+Using this method, we first start the emulator, and then start the additional containers
+from the existing images. We attach these containers to the networks created by the emulator.
+This way, the new containers becomes part of the emultion.
+
+See [this example](../A10_add_containers/) for details.