Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 27 additions & 15 deletions .github/actions/setup-bink/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,10 @@ runs:
- name: Configure kernel for nested containers
shell: bash
run: |
sudo systemctl disable --now apparmor 2>/dev/null || true
sudo aa-teardown 2>/dev/null || true
echo "AppArmor status after teardown:"
sudo aa-status 2>/dev/null || echo "(aa-status failed)"
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1
Expand Down Expand Up @@ -49,6 +52,8 @@ runs:
libbtrfs-dev \
libdevmapper-dev \
pkg-config
# passt's postinst reloads its AppArmor profile in enforce mode; remove it
sudo aa-teardown 2>/dev/null || true

- name: Set up KVM
shell: bash
Expand All @@ -60,24 +65,32 @@ runs:
shell: bash
run: |
podman --version
sudo mkdir -p /etc/containers
echo '{"defaultAction":"SCMP_ACT_ALLOW"}' | sudo tee /etc/containers/seccomp.json
printf '[containers]\napparmor_profile = "unconfined"\nseccomp_profile = "/etc/containers/seccomp.json"\n' | sudo tee /etc/containers/containers.conf
grep -q '^root:' /etc/subuid || echo 'root:100000:65536' | sudo tee -a /etc/subuid
grep -q '^root:' /etc/subgid || echo 'root:100000:65536' | sudo tee -a /etc/subgid
printf 'unqualified-search-registries = ["docker.io"]\n' | sudo tee /etc/containers/registries.conf
sudo systemctl start podman.socket
sudo podman info --format '{{.Store.GraphRoot}}'
mkdir -p ~/.config/containers
echo '{"defaultAction":"SCMP_ACT_ALLOW"}' > ~/.config/containers/seccomp.json
printf '[containers]\napparmor_profile = "unconfined"\nseccomp_profile = "%s/.config/containers/seccomp.json"\n' "$HOME" > ~/.config/containers/containers.conf
USER_NAME=$(id -un)
grep -q "^${USER_NAME}:" /etc/subuid || echo "${USER_NAME}:100000:65536" | sudo tee -a /etc/subuid
grep -q "^${USER_NAME}:" /etc/subgid || echo "${USER_NAME}:100000:65536" | sudo tee -a /etc/subgid
printf 'unqualified-search-registries = ["docker.io"]\n' > ~/.config/containers/registries.conf
systemctl --user enable --now podman.socket
echo "CONTAINER_HOST=unix:///run/user/$(id -u)/podman/podman.sock" >> "$GITHUB_ENV"
podman info --format '{{.Store.GraphRoot}}'

- name: Build bink binary
shell: bash
run: sudo make build-bink
run: make build-bink

- name: Build cluster and DNS images
shell: bash
run: |
make build-cluster-image
make build-dns-image

- name: Verify prerequisites
shell: bash
run: |
test -f ./bink
sudo podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}"
podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}"
df -h /
free -h

Expand Down Expand Up @@ -105,21 +118,20 @@ runs:
shell: bash
run: |
for f in /tmp/podman-image-cache/*.tar; do
sudo podman load -i "$f"
podman load -i "$f"
done
sudo podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}"
podman images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}"

- name: Pre-pull container images
if: steps.image-cache.outputs.cache-hit != 'true'
shell: bash
run: |
mkdir -p /tmp/podman-image-cache
for img in $BINK_IMAGES $EXTERNAL_IMAGES; do
sudo podman pull "$img"
podman pull "$img"
name=$(echo "$img" | sed 's|[/:]|_|g')
sudo podman save -o "/tmp/podman-image-cache/${name}.tar" "$img"
podman save -o "/tmp/podman-image-cache/${name}.tar" "$img"
done
sudo chown -R $(id -u):$(id -g) /tmp/podman-image-cache

- name: Save image cache
if: steps.image-cache.outputs.cache-hit != 'true'
Expand Down
18 changes: 11 additions & 7 deletions .github/collect-logs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,34 +2,38 @@
# Collect logs from bink test containers for CI debugging
# Writes per-container log files to $LOG_DIR (default: /tmp/bink-logs)

set -x

LOG_DIR="${LOG_DIR:-/tmp/bink-logs}"
mkdir -p "$LOG_DIR"

SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -i /var/run/cluster/cluster.key -p 2222"

ssh_vm() {
local ctr="$1" cmd="$2"
sudo podman exec "$ctr" ssh $SSH_OPTS core@localhost "$cmd" 2>&1
podman exec "$ctr" ssh $SSH_OPTS core@localhost "$cmd" 2>&1
}

sudo podman ps -a 2>/dev/null > "$LOG_DIR/podman-ps.txt" || true
podman ps -a 2>/dev/null > "$LOG_DIR/podman-ps.txt" || true

for ctr in $(sudo podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' 2>/dev/null); do
for ctr in $(podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' 2>/dev/null); do
echo "Collecting logs for $ctr"
dir="$LOG_DIR/$ctr"
mkdir -p "$dir"

sudo podman logs "$ctr" > "$dir/container.log" 2>&1 || true
podman logs "$ctr" > "$dir/container.log" 2>&1 || true
ssh_vm "$ctr" "sudo journalctl -n 200 --no-pager" > "$dir/journal.log" || echo "(VM not reachable)" > "$dir/journal.log"
ssh_vm "$ctr" "sudo journalctl -u kubelet -n 100 --no-pager" > "$dir/kubelet.log" || true
ssh_vm "$ctr" "sudo journalctl -u crio -n 100 --no-pager" > "$dir/crio.log" || true
ssh_vm "$ctr" "cloud-init status --long" > "$dir/cloud-init.log" || true
ssh_vm "$ctr" "sudo dmesg" > "$dir/dmesg.log" || true
done

df -h | sudo tee "$LOG_DIR/disk.txt" > /dev/null || true
free -h | sudo tee "$LOG_DIR/memory.txt" > /dev/null || true
sudo dmesg | tail -100 | sudo tee "$LOG_DIR/host-dmesg.txt" > /dev/null || true
df -h > "$LOG_DIR/disk.txt" 2>&1 || true
free -h > "$LOG_DIR/memory.txt" 2>&1 || true
sudo dmesg > "$LOG_DIR/host-dmesg.txt" 2>&1 || true
sudo journalctl --no-pager > "$LOG_DIR/host-journal.log" 2>&1 || true
sudo cp /var/log/audit/audit.log "$LOG_DIR/audit.log" 2>/dev/null || true

for bcvk_dir in /tmp/bcvk-logs*; do
[ -d "$bcvk_dir" ] || continue
Expand Down
34 changes: 15 additions & 19 deletions .github/workflows/integration-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ jobs:

- name: Load node image
if: needs.build-node-images.result == 'success'
run: sudo podman load -i node-image.tar
run: podman load -i node-image.tar

- name: Setup bink
uses: ./.github/actions/setup-bink
Expand All @@ -184,10 +184,8 @@ jobs:
node-image: ${{ needs.build-node-images.result == 'skipped' && format('ghcr.io/bootc-dev/bink/node:v{0}-fedora-{1}-disk', needs.supported-versions.outputs.default-version, needs.supported-versions.outputs.fedora-version) || '' }}

- name: Run integration tests
run: sudo make test-integration TEST_PROCS=2
run: make test-integration TEST_PROCS=2
timeout-minutes: 90
env:
CONTAINER_HOST: unix:///run/podman/podman.sock

- name: Collect logs
if: failure()
Expand All @@ -203,9 +201,9 @@ jobs:
- name: Cleanup test clusters
if: always()
run: |
sudo podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' | \
xargs -r sudo podman rm -f 2>/dev/null || true
sudo podman volume prune -f 2>/dev/null || true
podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' | \
xargs -r podman rm -f 2>/dev/null || true
podman volume prune -f 2>/dev/null || true

integration-tests-composefs:
needs: [changes, supported-versions, build-node-images]
Expand All @@ -226,7 +224,7 @@ jobs:

- name: Load composefs node image
if: needs.build-node-images.result == 'success'
run: sudo podman load -i node-image-composefs.tar
run: podman load -i node-image-composefs.tar

- name: Setup bink
uses: ./.github/actions/setup-bink
Expand All @@ -235,10 +233,9 @@ jobs:
node-image: ${{ needs.build-node-images.result == 'skipped' && format('ghcr.io/bootc-dev/bink/node:v{0}-fedora-{1}-disk-composefs', needs.supported-versions.outputs.default-version, needs.supported-versions.outputs.fedora-version) || '' }}

- name: Run composefs integration tests
run: sudo make test-integration-composefs
run: make test-integration-composefs
timeout-minutes: 90
env:
CONTAINER_HOST: unix:///run/podman/podman.sock
BINK_NODE_IMAGE: ghcr.io/bootc-dev/bink/node:v${{ needs.supported-versions.outputs.default-version }}-fedora-${{ needs.supported-versions.outputs.fedora-version }}-disk-composefs

- name: Collect logs
Expand All @@ -255,9 +252,9 @@ jobs:
- name: Cleanup test clusters
if: always()
run: |
sudo podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' | \
xargs -r sudo podman rm -f 2>/dev/null || true
sudo podman volume prune -f 2>/dev/null || true
podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' | \
xargs -r podman rm -f 2>/dev/null || true
podman volume prune -f 2>/dev/null || true

integration-tests-k8s-versions:
needs: [changes, supported-versions, build-node-images]
Expand Down Expand Up @@ -285,7 +282,7 @@ jobs:

- name: Load node image
if: needs.build-node-images.result == 'success'
run: sudo podman load -i node-image.tar
run: podman load -i node-image.tar

- name: Setup bink
uses: ./.github/actions/setup-bink
Expand All @@ -294,10 +291,9 @@ jobs:
node-image: ${{ needs.build-node-images.result == 'skipped' && format('ghcr.io/bootc-dev/bink/node:v{0}-fedora-{1}-disk', matrix.kube-minor, needs.supported-versions.outputs.fedora-version) || '' }}

- name: Run integration tests (K8s ${{ matrix.kube-minor }})
run: sudo make test-integration GINKGO_FOCUS="should create and initialize a complete Kubernetes cluster"
run: make test-integration GINKGO_FOCUS="should create and initialize a complete Kubernetes cluster"
timeout-minutes: 90
env:
CONTAINER_HOST: unix:///run/podman/podman.sock
BINK_NODE_IMAGE: ${{ env.BINK_NODE_IMAGE }}

- name: Collect logs
Expand All @@ -314,9 +310,9 @@ jobs:
- name: Cleanup test clusters
if: always()
run: |
sudo podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' | \
xargs -r sudo podman rm -f 2>/dev/null || true
sudo podman volume prune -f 2>/dev/null || true
podman ps -a --filter "name=k8s-test-bink" --format '{{.Names}}' | \
xargs -r podman rm -f 2>/dev/null || true
podman volume prune -f 2>/dev/null || true

push-node-images:
needs: [supported-versions, build-node-images, integration-tests, integration-tests-composefs, integration-tests-k8s-versions]
Expand Down
1 change: 1 addition & 0 deletions containerfiles/cluster-image/qemu.conf
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ dynamic_ownership = 1
remember_owner = 0
namespaces = [ ]
cgroup_controllers = [ ]
security_driver = "none"

# virtiofsd configuration - use wrapper for container compatibility
virtiofsd_path = "/usr/local/bin/virtiofsd-wrapper"
4 changes: 4 additions & 0 deletions test/integration/images_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
specs "github.com/opencontainers/runtime-spec/specs-go"
"go.podman.io/podman/v6/pkg/specgen"

"github.com/bootc-dev/bink/internal/cluster"
Expand Down Expand Up @@ -125,6 +126,9 @@ var _ = Describe("Cluster Images Volume", Serial, func() {
Name: volumeName,
Dest: "/var/lib/containers/storage",
}},
CapAdd: []string{"SYS_ADMIN"},
Devices: []specs.LinuxDevice{{Path: "/dev/fuse"}},
SelinuxOpts: []string{"disable"},
})
Expect(err).ToNot(HaveOccurred())

Expand Down
Loading