diff --git a/.github/buildfarm/authorized_keys b/.github/buildfarm/authorized_keys new file mode 100644 index 0000000..da19fdc --- /dev/null +++ b/.github/buildfarm/authorized_keys @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3qOXW5twS+gnni5Y4kCU3BBPjTShWsCTW5mQPRNA9GEYlDnBz3Wk0w4hfyI+NcUoSuOxxqrz6NGowfC/F4fYUe53R2/r1hWNuHloArbs9/8yROAaAlqhqmQDVsJxtbZdhh6sgwxcSe65OTYYhsZ8xzD34LBhuyLNaVEZy4WYUyjKjmh0YYSjiO6VFmzdvP3FSAqdFjG/pRs0igum9D8FJhl+FOrZ68fiFd70pyhl/yUsBntvtDbKJ73GKvSV58GytkLaxpDy4KLOVtnNYeqGFlFQ4eBrd+mXeyc2WvwYrn5V7HB2+Q2RTTvpEYazCjAolZ/E0wyqS2fSKFAGcKcDcoWordJ4VXwjiR9k28RPWSk9AcYJU1jI6v0KABOGTR7lNKmQr7fkD0KyZbYIB5EAcOsQHGCCDibFnxglIPFJt/YBZLLTm+ssTQ4ENkVrlS8dmC2drLMN3hsjBagoYD0e7hRARiv7/snREoNrQmKWz21Ft2Y37P7hWKOwQQ/NfZCs= iphy@code.tox.chat diff --git a/.github/buildfarm/generate-wg b/.github/buildfarm/generate-wg new file mode 100755 index 0000000..e20dc05 --- /dev/null +++ b/.github/buildfarm/generate-wg @@ -0,0 +1,15 @@ +#!/bin/sh + +echo '# Peers for the server' >wg0-1.conf + +readarray -t NODES <<<"$(seq 2 9)" + +for NODE in "${NODES[@]}"; do + PRIVATE_KEY="$(wg genkey)" + sed -e "s!@NODE@!$NODE!g;s!@PRIVATE_KEY@!$PRIVATE_KEY!g" wg0.conf.template >"wg0-$NODE.conf" + sed -e "s!@NODE@!$NODE!g" worker.yml.template >"worker-$NODE.yml" + echo >>wg0-1.conf + echo '[Peer]' >>wg0-1.conf + echo "PublicKey = $(echo "$PRIVATE_KEY" | wg pubkey)" >>wg0-1.conf + echo "AllowedIPs = 10.100.0.$NODE/32" >>wg0-1.conf +done diff --git a/.github/buildfarm/server-2.yml b/.github/buildfarm/server-2.yml new file mode 100644 index 0000000..78d85e2 --- /dev/null +++ b/.github/buildfarm/server-2.yml @@ -0,0 +1,12 @@ +backplane: + redisUri: "redis://10.100.0.2:6379" + queues: + - name: "cpu" + properties: + - name: "min-cores" + value: "*" + - name: "max-cores" + value: "*" +server: + publicName: "10.100.0.2:8980" +prometheusPort: 9091 diff --git a/.github/buildfarm/wg0-1.conf b/.github/buildfarm/wg0-1.conf new file mode 100644 index 0000000..aeef1cd --- /dev/null +++ b/.github/buildfarm/wg0-1.conf @@ -0,0 +1,33 @@ +# Peers for the server + +[Peer] +PublicKey = 2UeU4A9G489ts7e0gXucEIXIU+IrtMCJnbnlONu+PlQ= +AllowedIPs = 10.100.0.2/32 + +[Peer] +PublicKey = RCPvqEb+R+pKmZVOqJTHiLHIsb1KKYRNENZ5gtciQAo= +AllowedIPs = 10.100.0.3/32 + +[Peer] +PublicKey = i2dITECpbIX3E7hAMIFTd0fce3uc15Zi4MzeggKF3zQ= +AllowedIPs = 10.100.0.4/32 + +[Peer] +PublicKey = pSpYbq1/gPPqf65nTHgL3/w1xhSOd8sjSV84zYiRCEQ= +AllowedIPs = 10.100.0.5/32 + +[Peer] +PublicKey = eLgqPRirKd5Z7Az1Xe9v5xV1AUq1JxB0jQXBwPOywCA= +AllowedIPs = 10.100.0.6/32 + +[Peer] +PublicKey = z3cUh059Lo9JVf3Ea8r8x88JThVdcnmco63Iz/1dZ1Y= +AllowedIPs = 10.100.0.7/32 + +[Peer] +PublicKey = UON0ft/NzF/0T5aWsZVqO45a4KKmP3pSUoXSkj4Y9kQ= +AllowedIPs = 10.100.0.8/32 + +[Peer] +PublicKey = 594I6K0nCuDEeV3XFubePmR6jT3HVkRIQu4sTrXNZRE= +AllowedIPs = 10.100.0.9/32 diff --git a/.github/buildfarm/wg0-2.conf b/.github/buildfarm/wg0-2.conf new file mode 100644 index 0000000..bc9183c --- /dev/null +++ b/.github/buildfarm/wg0-2.conf @@ -0,0 +1,9 @@ +[Interface] +PrivateKey = WE+bkC80LF5s5+BBZ5TtWMKyBXAiSEdgVRd8ytWvkU0= +Address = 10.100.0.2/24 + +[Peer] +PublicKey = Tb1SOeotVZa6FbEmRq1x4Wo62PxjrjLeHbRYp4kTH2A= +Endpoint = code.tox.chat:51820 +AllowedIPs = 10.100.0.0/24 +PersistentKeepalive = 25 diff --git a/.github/buildfarm/wg0-3.conf b/.github/buildfarm/wg0-3.conf new file mode 100644 index 0000000..097017c --- /dev/null +++ b/.github/buildfarm/wg0-3.conf @@ -0,0 +1,9 @@ +[Interface] +PrivateKey = mDRI1EKXXY8siirWEx+ifXV0zHM3s3yOeS6WBp2VfGE= +Address = 10.100.0.3/24 + +[Peer] +PublicKey = Tb1SOeotVZa6FbEmRq1x4Wo62PxjrjLeHbRYp4kTH2A= +Endpoint = code.tox.chat:51820 +AllowedIPs = 10.100.0.0/24 +PersistentKeepalive = 25 diff --git a/.github/buildfarm/wg0-4.conf b/.github/buildfarm/wg0-4.conf new file mode 100644 index 0000000..0c8559e --- /dev/null +++ b/.github/buildfarm/wg0-4.conf @@ -0,0 +1,9 @@ +[Interface] +PrivateKey = 4P1Ct0HsoLvSz5KXVe8Jt/mNkdvf26H0R51xMzDiFFs= +Address = 10.100.0.4/24 + +[Peer] +PublicKey = Tb1SOeotVZa6FbEmRq1x4Wo62PxjrjLeHbRYp4kTH2A= +Endpoint = code.tox.chat:51820 +AllowedIPs = 10.100.0.0/24 +PersistentKeepalive = 25 diff --git a/.github/buildfarm/wg0-5.conf b/.github/buildfarm/wg0-5.conf new file mode 100644 index 0000000..bc7d33c --- /dev/null +++ b/.github/buildfarm/wg0-5.conf @@ -0,0 +1,9 @@ +[Interface] +PrivateKey = WN5cSY9S9Sh8jlLbR1BDlF/rcRQfRZIX5xIoXZRZJUg= +Address = 10.100.0.5/24 + +[Peer] +PublicKey = Tb1SOeotVZa6FbEmRq1x4Wo62PxjrjLeHbRYp4kTH2A= +Endpoint = code.tox.chat:51820 +AllowedIPs = 10.100.0.0/24 +PersistentKeepalive = 25 diff --git a/.github/buildfarm/wg0-6.conf b/.github/buildfarm/wg0-6.conf new file mode 100644 index 0000000..b35b75b --- /dev/null +++ b/.github/buildfarm/wg0-6.conf @@ -0,0 +1,9 @@ +[Interface] +PrivateKey = sPpPQmp3eZqWv0DyWkSDsJoz5Ofxp7eb2SIT8TZwt3Q= +Address = 10.100.0.6/24 + +[Peer] +PublicKey = Tb1SOeotVZa6FbEmRq1x4Wo62PxjrjLeHbRYp4kTH2A= +Endpoint = code.tox.chat:51820 +AllowedIPs = 10.100.0.0/24 +PersistentKeepalive = 25 diff --git a/.github/buildfarm/wg0-7.conf b/.github/buildfarm/wg0-7.conf new file mode 100644 index 0000000..b7775f1 --- /dev/null +++ b/.github/buildfarm/wg0-7.conf @@ -0,0 +1,9 @@ +[Interface] +PrivateKey = CFyxB7DmHXM1VaF2v8SbLTyRH1eGrfGIPd74Rq3EQFc= +Address = 10.100.0.7/24 + +[Peer] +PublicKey = Tb1SOeotVZa6FbEmRq1x4Wo62PxjrjLeHbRYp4kTH2A= +Endpoint = code.tox.chat:51820 +AllowedIPs = 10.100.0.0/24 +PersistentKeepalive = 25 diff --git a/.github/buildfarm/wg0-8.conf b/.github/buildfarm/wg0-8.conf new file mode 100644 index 0000000..4f5c4bf --- /dev/null +++ b/.github/buildfarm/wg0-8.conf @@ -0,0 +1,9 @@ +[Interface] +PrivateKey = OM17rUuFLFGxhMI0gIW59JCDyITC34eQRPKEx/cpMmQ= +Address = 10.100.0.8/24 + +[Peer] +PublicKey = Tb1SOeotVZa6FbEmRq1x4Wo62PxjrjLeHbRYp4kTH2A= +Endpoint = code.tox.chat:51820 +AllowedIPs = 10.100.0.0/24 +PersistentKeepalive = 25 diff --git a/.github/buildfarm/wg0-9.conf b/.github/buildfarm/wg0-9.conf new file mode 100644 index 0000000..89f4d19 --- /dev/null +++ b/.github/buildfarm/wg0-9.conf @@ -0,0 +1,9 @@ +[Interface] +PrivateKey = yJA/1szzSxqxwSxT3si/1v3Gx0oTDlOS10HZgTwfZFc= +Address = 10.100.0.9/24 + +[Peer] +PublicKey = Tb1SOeotVZa6FbEmRq1x4Wo62PxjrjLeHbRYp4kTH2A= +Endpoint = code.tox.chat:51820 +AllowedIPs = 10.100.0.0/24 +PersistentKeepalive = 25 diff --git a/.github/buildfarm/wg0.conf.template b/.github/buildfarm/wg0.conf.template new file mode 100644 index 0000000..6af9f06 --- /dev/null +++ b/.github/buildfarm/wg0.conf.template @@ -0,0 +1,9 @@ +[Interface] +PrivateKey = @PRIVATE_KEY@ +Address = 10.100.0.@NODE@/24 + +[Peer] +PublicKey = Tb1SOeotVZa6FbEmRq1x4Wo62PxjrjLeHbRYp4kTH2A= +Endpoint = code.tox.chat:51820 +AllowedIPs = 10.100.0.0/24 +PersistentKeepalive = 25 diff --git a/.github/buildfarm/worker-2.yml b/.github/buildfarm/worker-2.yml new file mode 100644 index 0000000..5a3bb28 --- /dev/null +++ b/.github/buildfarm/worker-2.yml @@ -0,0 +1,11 @@ +backplane: + redisUri: "redis://10.100.0.2:6379" + queues: + - name: "cpu" + properties: + - name: "min-cores" + value: "*" + - name: "max-cores" + value: "*" +worker: + publicName: "10.100.0.2:8981" diff --git a/.github/buildfarm/worker-3.yml b/.github/buildfarm/worker-3.yml new file mode 100644 index 0000000..26e4658 --- /dev/null +++ b/.github/buildfarm/worker-3.yml @@ -0,0 +1,11 @@ +backplane: + redisUri: "redis://10.100.0.2:6379" + queues: + - name: "cpu" + properties: + - name: "min-cores" + value: "*" + - name: "max-cores" + value: "*" +worker: + publicName: "10.100.0.3:8981" diff --git a/.github/buildfarm/worker-4.yml b/.github/buildfarm/worker-4.yml new file mode 100644 index 0000000..1ec4f63 --- /dev/null +++ b/.github/buildfarm/worker-4.yml @@ -0,0 +1,11 @@ +backplane: + redisUri: "redis://10.100.0.2:6379" + queues: + - name: "cpu" + properties: + - name: "min-cores" + value: "*" + - name: "max-cores" + value: "*" +worker: + publicName: "10.100.0.4:8981" diff --git a/.github/buildfarm/worker-5.yml b/.github/buildfarm/worker-5.yml new file mode 100644 index 0000000..3ed3fb5 --- /dev/null +++ b/.github/buildfarm/worker-5.yml @@ -0,0 +1,11 @@ +backplane: + redisUri: "redis://10.100.0.2:6379" + queues: + - name: "cpu" + properties: + - name: "min-cores" + value: "*" + - name: "max-cores" + value: "*" +worker: + publicName: "10.100.0.5:8981" diff --git a/.github/buildfarm/worker-6.yml b/.github/buildfarm/worker-6.yml new file mode 100644 index 0000000..8acc898 --- /dev/null +++ b/.github/buildfarm/worker-6.yml @@ -0,0 +1,11 @@ +backplane: + redisUri: "redis://10.100.0.2:6379" + queues: + - name: "cpu" + properties: + - name: "min-cores" + value: "*" + - name: "max-cores" + value: "*" +worker: + publicName: "10.100.0.6:8981" diff --git a/.github/buildfarm/worker-7.yml b/.github/buildfarm/worker-7.yml new file mode 100644 index 0000000..e714eb1 --- /dev/null +++ b/.github/buildfarm/worker-7.yml @@ -0,0 +1,11 @@ +backplane: + redisUri: "redis://10.100.0.2:6379" + queues: + - name: "cpu" + properties: + - name: "min-cores" + value: "*" + - name: "max-cores" + value: "*" +worker: + publicName: "10.100.0.7:8981" diff --git a/.github/buildfarm/worker-8.yml b/.github/buildfarm/worker-8.yml new file mode 100644 index 0000000..dab283e --- /dev/null +++ b/.github/buildfarm/worker-8.yml @@ -0,0 +1,11 @@ +backplane: + redisUri: "redis://10.100.0.2:6379" + queues: + - name: "cpu" + properties: + - name: "min-cores" + value: "*" + - name: "max-cores" + value: "*" +worker: + publicName: "10.100.0.8:8981" diff --git a/.github/buildfarm/worker-9.yml b/.github/buildfarm/worker-9.yml new file mode 100644 index 0000000..3f3da88 --- /dev/null +++ b/.github/buildfarm/worker-9.yml @@ -0,0 +1,11 @@ +backplane: + redisUri: "redis://10.100.0.2:6379" + queues: + - name: "cpu" + properties: + - name: "min-cores" + value: "*" + - name: "max-cores" + value: "*" +worker: + publicName: "10.100.0.9:8981" diff --git a/.github/buildfarm/worker.yml.template b/.github/buildfarm/worker.yml.template new file mode 100644 index 0000000..0efa557 --- /dev/null +++ b/.github/buildfarm/worker.yml.template @@ -0,0 +1,11 @@ +backplane: + redisUri: "redis://10.100.0.2:6379" + queues: + - name: "cpu" + properties: + - name: "min-cores" + value: "*" + - name: "max-cores" + value: "*" +worker: + publicName: "10.100.0.@NODE@:8981" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c7b24d..f90d722 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,15 +12,62 @@ concurrency: jobs: wireguard-vpn: + strategy: + matrix: + node: [2, 3, 4, 5, 6, 7, 8, 9] +# node: [2, 3, 4] runs-on: ubuntu-latest steps: - - run: | - sudo apt install wireguard - echo "UGdw/CjBpnKF/xJ3/orlUty29kYi1NMnXdXVYExFFH8=" > privatekey - sudo ip link add dev wg0 type wireguard - sudo ip address add dev wg0 10.100.0.2 peer 10.100.0.1 - sudo wg set wg0 listen-port 48123 private-key privatekey peer Tb1SOeotVZa6FbEmRq1x4Wo62PxjrjLeHbRYp4kTH2A= allowed-ips 10.100.0.0/24 endpoint code.tox.chat:51820 - sudo ip link set up dev wg0 - #ping 10.100.0.1 - sudo ifconfig wg0 + - uses: actions/checkout@v3 + - name: Setup wireguard + run: | + sudo apt-get -y --no-install-recommends install redis-tools wireguard + sudo cp .github/buildfarm/wg0-${{ matrix.node }}.conf /etc/wireguard/wg0.conf + sudo systemctl enable wg-quick@wg0.service + sudo systemctl start wg-quick@wg0.service || (sudo systemctl status wg-quick@wg0.service && false) + sudo systemctl start ssh + mkdir "$HOME/.ssh" + cp .github/buildfarm/authorized_keys "$HOME/.ssh/" + - name: Start redis + if: matrix.node == 2 + run: | + docker pull "redis:alpine" + docker run --detach --rm -i --network host "redis:alpine" + - name: Wait for VPN to be up + run: | + ifconfig wg0 sudo wg show + # VPN server and redis/buildfarm server need to be up. + for node in `seq 1 9`; do + for i in `seq 0 9`; do + if ping -c1 "10.100.0.$node"; then break; fi + sleep 1 + done + done + # Error if VPN server is still down after the above timeout. + ping -c1 10.100.0.1 + sudo wg show + ip route + # Wait for redis connectivity. + for i in `seq 0 9`; do + if redis-cli -h 10.100.0.2 -p 6379 ping; then break; fi + sleep 1 + done + redis-cli -h 10.100.0.2 -p 6379 ping + - name: Pull latest worker image + run: docker pull "toxchat/buildfarm-worker" + - name: Test connectivity to other nodes + run: | + ping -c1 10.100.0.1 # server + ping -c1 10.100.0.2 # either self or some other node + ping -c1 10.100.0.3 # either some other node or self + - name: Start worker + if: matrix.node != 2 + run: | + docker run --privileged --rm -i --network host -v "$PWD/.github/buildfarm/worker-${{ matrix.node }}.yml:/app/build_buildfarm/examples/config.minimal.yml" "toxchat/buildfarm-worker" + - name: Start server and worker + if: matrix.node == 2 + run: | + docker pull "toxchat/buildfarm-server" + docker run --detach --privileged --rm -i --network host -v "$PWD/.github/buildfarm/worker-${{ matrix.node }}.yml:/app/build_buildfarm/examples/config.minimal.yml" "toxchat/buildfarm-worker" + docker run --rm -i --network host -v "$PWD/.github/buildfarm/server-${{ matrix.node }}.yml:/app/build_buildfarm/examples/config.minimal.yml" "toxchat/buildfarm-server"