patch(main): Live Migration Support

.github/workflows/hydrun.yaml

@@ -0,0 +1,117 @@
+name: hydrun CI
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: "0 0 * * 0"
+
+jobs:
+  build-linux:
+    runs-on: ${{ matrix.target.runner }}
+    permissions:
+      contents: read
+    strategy:
+      matrix:
+        target:
+          # Binaries
+          - id: rust.x86_64
+            src: .
+            os: public.ecr.aws/firecracker/fcuvm:v77
+            flags: ""
+            cmd: ./Hydrunfile rust x86_64
+            dst: out/*
+            runner: depot-ubuntu-22.04-32
+          - id: rust.aarch64
+            src: .
+            os: public.ecr.aws/firecracker/fcuvm:v77
+            flags: ""
+            cmd: ./Hydrunfile rust aarch64
+            dst: out/*
+            runner: depot-ubuntu-22.04-arm-32
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Restore ccache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            /tmp/ccache
+          key: cache-ccache-${{ matrix.target.id }}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Set up hydrun
+        run: |
+          curl -L -o /tmp/hydrun "https://github.com/pojntfx/hydrun/releases/latest/download/hydrun.linux-$(uname -m)"
+          sudo install /tmp/hydrun /usr/local/bin
+      - name: Build with hydrun
+        working-directory: ${{ matrix.target.src }}
+        run: hydrun -o ${{ matrix.target.os }} ${{ matrix.target.flags }} "${{ matrix.target.cmd }}"
+      - name: Fix permissions for output
+        run: sudo chown -R $USER .
+      - name: Save ccache
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            /tmp/ccache
+          key: cache-ccache-${{ matrix.target.id }}
+      - name: Upload output
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.target.id }}
+          path: ${{ matrix.target.dst }}
+
+  publish-linux:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: write
+    needs: build-linux
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Download output
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/out
+      - name: Publish pre-release to GitHub releases
+        if: ${{ github.ref == 'refs/heads/main-live-migration-pvm' || github.ref == 'refs/heads/main-live-migration' || github.ref == 'refs/heads/firecracker-v1.8-live-migration-pvm' || github.ref == 'refs/heads/firecracker-v1.8-live-migration' }}
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: release-${{ github.ref_name }}
+          prerelease: true
+          files: |
+            /tmp/out/*/*
+      - name: Publish release to GitHub releases
+        if: startsWith(github.ref, 'refs/tags/v')
+        uses: softprops/action-gh-release@v2
+        with:
+          prerelease: false
+          files: |
+            /tmp/out/*/*
+
+      - name: "Configure AWS credentials"
+        uses: "aws-actions/configure-aws-credentials@v4"
+        with:
+          aws-region: "${{ vars.AWS_REGION }}"
+          role-to-assume: "${{ vars.AWS_IAM_ROLE }}"
+          role-session-name: "firecracker-hydrun-${{ github.job }}-${{ github.run_id }}"
+          role-duration-seconds: 900
+
+      - name: Upload to S3
+        if: "!startsWith(github.ref, 'refs/pull/')"
+        run: |
+          if [[ "${{ github.ref }}" == refs/tags/* ]]; then
+            UPLOAD_FOLDER="release/${GITHUB_REF#refs/tags/}"
+          elif [[ "${{ github.ref }}" == refs/heads/* ]]; then
+            UPLOAD_FOLDER="dev/${GITHUB_REF#refs/heads/}"
+          else
+            echo "Skipping S3 upload: unsupported ref type $GITHUB_REF"
+            exit 0
+          fi
+          echo "Uploading artifacts to: ${{ vars.S3_BUCKET_URL }}firecracker/${UPLOAD_FOLDER}/"
+          aws s3 cp /tmp/out ${{ vars.S3_BUCKET_URL }}firecracker/${UPLOAD_FOLDER}/ --recursive

Hydrunfile

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -e
+
+# Rust
+if [ "$1" = "rust" ]; then
+    # Configure Git
+    git config --global --add safe.directory '*'
+
+    # Build
+    export RUSTFLAGS='-C target-feature=+crt-static'
+    cargo build --package firecracker --package jailer --package seccompiler --package rebase-snap --package cpu-template-helper --target "$2-unknown-linux-musl" --all-features --release
+
+    # Stage binaries
+    mkdir -p out
+
+    dir="./build/cargo_target/$2-unknown-linux-musl/release"
+    for file in $(ls "$dir"); do
+        if [[ -x "$dir/$file" && ! -d "$dir/$file" ]]; then
+            cp "$dir/$file" "./out/${file}.linux-$2"
+        fi
+    done
+
+    exit 0
+fi

docs/cpu_templates/cpu-template-helper.md

@@ -237,6 +237,7 @@ CPU features to a heterogeneous fleet consisting of multiple CPU models.
 | HV_X64_MSR_SYNDBG_PENDING_BUFFER        | 0x400000f5              |
 | HV_X64_MSR_SYNDBG_OPTIONS               | 0x400000ff              |
 | HV_X64_MSR_TSC_INVARIANT_CONTROL        | 0x40000118              |
+| HV_X64_MSR_TSC_INVARIANT_CONTROL        | 0x40000118              |
 
 ### ARM registers excluded from guest CPU configuration dump
 

resources/seccomp/aarch64-unknown-linux-musl.json

@@ -39,6 +39,10 @@
             {
                 "syscall": "fsync"
             },
+            {
+                "syscall": "msync",
+                "comment": "Used for live migration to sync dirty pages"
+            },
             {
                 "syscall": "close"
             },

resources/seccomp/x86_64-unknown-linux-musl.json

@@ -39,6 +39,10 @@
             {
                 "syscall": "fsync"
             },
+            {
+                "syscall": "msync",
+                "comment": "Used for live migration to sync dirty pages"
+            },
             {
                 "syscall": "close"
             },

src/firecracker/src/api_server/mod.rs

@@ -157,6 +157,14 @@ impl ApiServer {
                     &METRICS.latencies_us.diff_create_snapshot,
                     "create diff snapshot",
                 )),
+                SnapshotType::Msync => Some((
+                    &METRICS.latencies_us.diff_create_snapshot,
+                    "memory synchronization snapshot",
+                )),
+                SnapshotType::MsyncAndState => Some((
+                    &METRICS.latencies_us.diff_create_snapshot,
+                    "memory synchronization and state snapshot",
+                )),
             },
             VmmAction::LoadSnapshot(_) => {
                 Some((&METRICS.latencies_us.load_snapshot, "load snapshot"))

src/firecracker/src/api_server/request/snapshot.rs

@@ -105,6 +105,7 @@ fn parse_put_snapshot_load(body: &Body) -> Result<ParsedRequest, RequestError> {
         mem_backend,
         enable_diff_snapshots: snapshot_config.enable_diff_snapshots,
         resume_vm: snapshot_config.resume_vm,
+        shared: snapshot_config.shared,
     };
 
     // Construct the `ParsedRequest` object.
@@ -181,6 +182,7 @@ mod tests {
             },
             enable_diff_snapshots: false,
             resume_vm: false,
+            shared: false,
         };
         let mut parsed_request = parse_put_snapshot(&Body::new(body), Some("load")).unwrap();
         assert!(parsed_request
@@ -208,6 +210,7 @@ mod tests {
             },
             enable_diff_snapshots: true,
             resume_vm: false,
+            shared: false,
         };
         let mut parsed_request = parse_put_snapshot(&Body::new(body), Some("load")).unwrap();
         assert!(parsed_request
@@ -235,6 +238,7 @@ mod tests {
             },
             enable_diff_snapshots: false,
             resume_vm: true,
+            shared: false,
         };
         let mut parsed_request = parse_put_snapshot(&Body::new(body), Some("load")).unwrap();
         assert!(parsed_request
@@ -259,6 +263,7 @@ mod tests {
             },
             enable_diff_snapshots: false,
             resume_vm: true,
+            shared: false,
         };
         let parsed_request = parse_put_snapshot(&Body::new(body), Some("load")).unwrap();
         assert_eq!(

src/firecracker/swagger/firecracker.yaml

@@ -1212,6 +1212,8 @@ definitions:
         enum:
           - Full
           - Diff
+          - Msync
+          - MsyncAndState
         description:
           Type of snapshot to create. It is optional and by default, a full
           snapshot is created.
@@ -1247,6 +1249,11 @@ definitions:
         type: boolean
         description:
           When set to true, the vm is also resumed if the snapshot load is successful.
+      shared:
+        type: boolean
+        description: When set to true and the guest memory backend is a file,
+          changes to the memory are asynchronously written back to the
+          backend as the VM is running.
 
   TokenBucket:
     type: object

src/vmm/src/logger/metrics.rs

@@ -604,6 +604,10 @@ pub struct PerformanceMetrics {
     pub full_create_snapshot: SharedStoreMetric,
     /// Measures the snapshot diff create time, at the API (user) level, in microseconds.
     pub diff_create_snapshot: SharedStoreMetric,
+     /// Measures the snapshot memory synchronization time, at the VMM level, in microseconds.
+    pub msync_create_snapshot: SharedStoreMetric,
+    /// Measures the snapshot memory synchronization and state time, at the VMM level, in microseconds.
+    pub msync_and_state_create_snapshot: SharedStoreMetric,
     /// Measures the snapshot load time, at the API (user) level, in microseconds.
     pub load_snapshot: SharedStoreMetric,
     /// Measures the microVM pausing duration, at the API (user) level, in microseconds.
@@ -627,6 +631,8 @@ impl PerformanceMetrics {
         Self {
             full_create_snapshot: SharedStoreMetric::new(),
             diff_create_snapshot: SharedStoreMetric::new(),
+            msync_create_snapshot: SharedStoreMetric::new(),
+            msync_and_state_create_snapshot: SharedStoreMetric::new(),
             load_snapshot: SharedStoreMetric::new(),
             pause_vm: SharedStoreMetric::new(),
             resume_vm: SharedStoreMetric::new(),