test-cloud-hypervisor.yml

name: Cloud Hypervisor Preview Integration

on:
  workflow_dispatch:
    inputs:
      run_live_kvm:
        description: Run the live KVM job on GitHub-hosted x64 Ubuntu 24.04
        required: true
        type: boolean
        default: true
  pull_request:
    types: [opened, synchronize, reopened, labeled]
    paths:
      - '.github/workflows/test-cloud-hypervisor.yml'
      - 'guest/cloud-hypervisor/**'
      - 'containers/build-tools/**'
      - 'containers/cloud-hypervisor/**'
      - 'containers/enclave/**'
      - 'guest/microvm-supervisor/**'
      - 'src/cloud-hypervisor/**'
      - 'src/cloud-hypervisor-runtime-backend.ts'
      - 'src/cloud-hypervisor-runtime-backend.test.ts'
      - 'src/microvm/**'
      - 'src/filesystem-policy.ts'
      - 'src/types/runtime-options.ts'
      - 'scripts/ci/cloud-hypervisor-*.sh'
      - 'scripts/ci/configure-docker-mirror.sh'
      - 'scripts/ci/docker-registry-ci.test.ts'
      - 'docs/cloud-hypervisor-foundation.md'

permissions:
  contents: read
  id-token: write
  attestations: write

concurrency:
  group: cloud-hypervisor-preview-${{ github.ref }}
  cancel-in-progress: false

jobs:
  build-test-artifacts:
    name: Build deterministic test guest
    if: github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request'
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4

      - name: Free unused hosted-runner disk space
        # Kernel, virtiofsd, three ext4 images and their bundles need room on the
        # hosted runner. Remove preinstalled SDKs this job never uses.
        run: |
          df -h /
          sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup \
            /opt/hostedtoolcache/CodeQL
          df -h /

      - name: Configure Docker Hub cache
        run: bash scripts/ci/configure-docker-mirror.sh

      - name: Set up Go
        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
        with:
          go-version: '1.25.0'
          cache-dependency-path: guest/microvm-supervisor/go.mod

      - name: Run guest supervisor unit tests
        working-directory: guest/microvm-supervisor
        # Running unit tests here (not just building it) catches
        # defects like an incorrect syscall.Mount() fstype before they
        # only surface as a guest kernel panic during the live-KVM job
        # below, which is much slower to diagnose.
        run: go test ./...

      - name: Set up Node.js
        uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
        with:
          node-version: '22'
          cache: npm

      - name: Install deterministic guest build prerequisites
        run: |
          sudo apt-get update
          sudo apt-get install --yes --no-install-recommends \
            bc \
            binutils \
            bison \
            build-essential \
            ca-certificates \
            cpio \
            e2fsprogs \
            file \
            flex \
            libcap2-bin \
            libcap-ng-dev \
            libelf-dev \
            libseccomp-dev \
            libssl-dev \
            pkg-config \
            rsync \
            xz-utils
          # virtiofsd is built from pinned upstream source by
          # build-test-artifacts.sh with the rustup toolchain it pins.
          rustup --version

      - name: Build and verify pinned artifacts
        run: |
          docker build \
            --platform linux/amd64 \
            --tag awf-cloud-hypervisor-build-tools:test \
            containers/build-tools
          docker build --platform linux/amd64 \
            --file containers/enclave/Dockerfile --target enclave-agent \
            --build-arg BUILD_TOOLS_IMAGE=awf-cloud-hypervisor-build-tools:test \
            --tag awf-cloud-hypervisor-enclave-agent:test containers
          BUILD_TOOLS_IMAGE=awf-cloud-hypervisor-build-tools:test \
            ENCLAVE_AGENT_IMAGE=awf-cloud-hypervisor-enclave-agent:test \
            AWF_CH_PRUNE_DOCKER_BUILD_CACHE=1 \
            ./guest/cloud-hypervisor/build-test-artifacts.sh
          ./guest/cloud-hypervisor/verify-test-artifacts.sh \
            release/cloud-hypervisor-test-x86_64

      - name: Attest guest artifact provenance
        uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
        with:
          subject-path: release/cloud-hypervisor-test-x86_64/awf-cloud-hypervisor-test-x86_64.tar.gz

      - name: Attest enclave rootfs bundle provenance
        uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
        with:
          subject-path: release/cloud-hypervisor-test-x86_64/awf-cloud-hypervisor-enclave-rootfs-x86_64.tar.gz

      - name: Attest shared enclave rootfs provenance
        id: attest_enclave_rootfs
        uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
        with:
          subject-path: release/cloud-hypervisor-test-x86_64/enclave-rootfs.ext4

      - name: Attest enclave rootfs manifest
        id: attest_enclave_manifest
        uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
        with:
          subject-path: release/cloud-hypervisor-test-x86_64/enclave-manifest.json

      - name: Bundle enclave attestations for offline verification
        env:
          ROOTFS_BUNDLE: ${{ steps.attest_enclave_rootfs.outputs.bundle-path }}
          MANIFEST_BUNDLE: ${{ steps.attest_enclave_manifest.outputs.bundle-path }}
        run: |
          cp "$ROOTFS_BUNDLE" \
            release/cloud-hypervisor-test-x86_64/enclave-rootfs.provenance.sigstore.jsonl
          cp "$MANIFEST_BUNDLE" \
            release/cloud-hypervisor-test-x86_64/enclave-manifest.sigstore.jsonl

      - name: Upload guest artifacts
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: cloud-hypervisor-test-x86_64
          path: release/cloud-hypervisor-test-x86_64/
          if-no-files-found: error
          retention-days: 7

  live-kvm:
    name: Live Cloud Hypervisor KVM smoke/security
    needs: build-test-artifacts
    if: >-
      (github.event_name == 'workflow_dispatch' && inputs.run_live_kvm) ||
      (github.event_name == 'pull_request' &&
       contains(github.event.pull_request.labels.*.name, 'cloud-hypervisor-kvm'))
    runs-on: ubuntu-24.04
    # 16 cases, each with up to a 90s boot budget and (since the guest
    # connectivity probe raised its own timeout for the same nested-KVM
    # scheduling reasons) up to a further 90s probe budget in the worst
    # case; 60 minutes leaves headroom without masking a genuine hang.
    timeout-minutes: 60
    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v4

      - name: Set up Node.js
        uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
        with:
          node-version: '22'
          cache: npm

      - name: Download verified guest artifacts
        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
        with:
          name: cloud-hypervisor-test-x86_64
          path: ${{ runner.temp }}/cloud-hypervisor-test-x86_64

      - name: Restore artifact executable permissions
        run: |
          # actions/upload-artifact + actions/download-artifact do not
          # reliably preserve the executable bit on binary files (a known
          # GitHub Actions artifact limitation). Digest verification below
          # still proves file integrity; this only restores the mode bits
          # needed to exec the pinned, already-verified binaries.
          chmod 0755 \
            "$RUNNER_TEMP/cloud-hypervisor-test-x86_64/cloud-hypervisor" \
            "$RUNNER_TEMP/cloud-hypervisor-test-x86_64/virtiofsd" \
            "$RUNNER_TEMP/cloud-hypervisor-test-x86_64/awf-supervisor"

      - name: Verify capable host and artifact digests
        run: |
          ./scripts/ci/cloud-hypervisor-host-preflight.sh \
            "$RUNNER_TEMP/cloud-hypervisor-test-x86_64"

      - name: Install, build, and prepare infrastructure images
        run: |
          npm ci
          npm run build
          docker build -t ghcr.io/github/gh-aw-firewall/squid:latest containers/squid
          docker build -t ghcr.io/github/gh-aw-firewall/api-proxy:latest containers/api-proxy

      - name: Run live fail-closed smoke/security coverage
        run: |
          ./scripts/ci/cloud-hypervisor-live-smoke.sh \
            "$RUNNER_TEMP/cloud-hypervisor-test-x86_64"

      - name: Print guest boot diagnostics on failure
        if: failure()
        run: |
          set -uo pipefail
          # Cloud Hypervisor's own collectDiagnostics() runs before the run
          # directory is torn down on a startup failure (see
          # src/cloud-hypervisor-runtime-backend.ts), so the guest serial
          # console and Cloud Hypervisor log land under each case's
          # audit/cloud-hypervisor/ directory. Print them directly in the
          # job log for fast triage, in addition to the uploaded artifact.
          # Redact the test-only secret sentinel defensively, matching the
          # scan the next step performs on the copied artifact.
          for f in "$RUNNER_TEMP"/awf-cloud-hypervisor-live/*/audit/cloud-hypervisor/{serial.log,cloud-hypervisor.log,network-diagnostics.txt,vm-info.json,counters.json}; do
            [ -f "$f" ] || continue
            echo "--- $f ---"
            sed 's/awf-cloud-hypervisor-real-secret-do-not-expose/[REDACTED]/g' "$f"
          done

      - name: Collect redacted diagnostics
        if: always()
        run: |
          set -euo pipefail
          source_root="$RUNNER_TEMP/awf-cloud-hypervisor-live"
          destination="$RUNNER_TEMP/cloud-hypervisor-diagnostics-safe"
          rm -rf "$destination"
          mkdir -p "$destination"
          if [ -d "$source_root" ]; then
            # --keep-containers (the keep-containers/preserve-diagnostics
            # live case) intentionally leaves its own work/audit files
            # root-owned (they are written by the CLI process, itself run
            # via sudo, and deliberately never cleaned up so the case's own
            # assertions can inspect them afterward). Every other case's
            # files are owned by the runner user and readable without
            # sudo, but find/cp must run as root here to reach the
            # keep-containers case's own preserved files too -- reading a
            # root-owned file as the runner user is a permission error,
            # not evidence of a missing file.
            while IFS= read -r -d '' file; do
              relative=${file#"$source_root/"}
              mkdir -p "$destination/$(dirname "$relative")"
              sudo cp "$file" "$destination/$relative"
            done < <(
              sudo find "$source_root" -type f \
                \( -path '*/audit/*' \
                -o -path '*/proxy-logs/*' \
                -o -name 'stdout.log' \
                -o -name 'stderr.log' \) \
                -print0
            )
          fi
          # Hand ownership of the copied destination tree back to the
          # runner user: sudo cp above creates new destination files as
          # root, which the secret-sentinel scan below and the later
          # upload-artifact step (both running as the runner user, not
          # root) need to be able to read.
          sudo chown -R "$(id -u):$(id -g)" "$destination"
          # awf-resolved-config.json's agentCommand field always contains the
          # smoke test's own shell command verbatim -- and this specific
          # command (the api-proxy-reflect case) intentionally references
          # the sentinel string itself as the pattern it greps for, to
          # assert the sentinel is absent from `env`. That is expected,
          # self-referential test source text, not a leak of the sentinel
          # *value* into somewhere it shouldn't be (guest console output,
          # network captures, proxy logs, etc. are all still fully scanned
          # below) -- so this one, known, always-matching file is excluded
          # from the scan rather than silently disabling it everywhere.
          if grep -R --binary-files=without-match \
            -F 'awf-cloud-hypervisor-real-secret-do-not-expose' \
            --exclude='awf-resolved-config.json' \
            "$destination"; then
            echo "::error::Secret sentinel found in diagnostic artifacts"
            rm -rf "$destination"
            exit 1
          fi

      - name: Upload actionable diagnostics
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: cloud-hypervisor-live-diagnostics
          path: ${{ runner.temp }}/cloud-hypervisor-diagnostics-safe/
          if-no-files-found: warn
          retention-days: 7

      - name: Enforce final residue cleanup
        if: always()
        run: |
          set -euo pipefail
          if sudo ip netns list | grep -q '^awfvm-'; then
            sudo ip netns list >&2
            echo "::error::Cloud Hypervisor namespace residue remains after cleanup"
            exit 1
          fi
          if sudo iptables -S DOCKER-USER | grep -q -- '--comment awf:awf_vm_'; then
            sudo iptables -S DOCKER-USER >&2
            echo "::error::Cloud Hypervisor bridge-rule residue remains after cleanup"
            exit 1
          fi
          # /sys/fs/cgroup/awf-cloud-hypervisor is a parent cgroup that
          # persists across the whole job; only per-run sub-cgroups are
          # created one level inside it (see cgroupPath in
          # src/cloud-hypervisor/manager.ts). Any cgroup v2 directory --
          # including this parent itself -- always contains standard
          # controller interface files (cpu.max, memory.max,
          # cgroup.controllers, ...) simply by virtue of existing;
          # matching all entries here (not just directories) made this
          # check a guaranteed false positive the moment the live-KVM job
          # ever actually completed successfully. See the identical fix
          # in scripts/ci/cloud-hypervisor-live-smoke.sh's own
          # assert_no_residue for the same root cause.
          if [ -d /sys/fs/cgroup/awf-cloud-hypervisor ] && \
             [ -n "$(sudo find /sys/fs/cgroup/awf-cloud-hypervisor -mindepth 1 -maxdepth 1 -type d 2>/dev/null)" ]; then
            echo "::error::Cloud Hypervisor cgroup residue remains after cleanup"
            exit 1
          fi
          if sudo pgrep -f '/var/lib/awf-cloud-hypervisor/trusted-artifacts/run-[^/]*/[c]loud-hypervisor --api-socket' >/dev/null 2>&1; then
            echo "::error::Cloud Hypervisor process residue remains after cleanup"
            exit 1
          fi