From 5b54bb242985d3f645ebdab90e5719449f3ab8c6 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 5 Mar 2026 22:02:55 +0800 Subject: [PATCH] Enhance CI workflows for Tai service with k3d integration - Add installation and setup steps for k3d to create a local Kubernetes cluster for testing. - Implement health checks for both Docker and K8s instances of Tai, improving readiness verification. - Generate kubeconfig for K8s benchmarks, ensuring proper configuration for testing environments. - Update environment variables to accommodate new K8s service configurations, enhancing overall CI reliability. These changes improve the testing framework for the Tai service by integrating Kubernetes support and refining service readiness checks. --- .github/workflows/pr-test.yml | 72 ++++++++++++++++++++++++++++----- .github/workflows/unit-test.yml | 56 +++++++++++++++++++++++++ 2 files changed, 118 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index d27936f5..63fd4e9a 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -1924,36 +1924,88 @@ jobs: docker pull yaoapp/tai:latest docker pull alpine:latest - - name: Start Tai (Docker proxy for benchmarks) + - name: Install k3d + run: curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash + + - name: Create k3d cluster run: | - docker run -d --name tai \ + k3d cluster create tai-test --no-lb --wait --api-port 16443 + kubectl wait --for=condition=Ready node --all --timeout=60s + k3d image import alpine:latest -c tai-test + + - name: Start Tai Docker instance (benchmarks) + run: | + docker run -d --name tai-docker \ -v /var/run/docker.sock:/var/run/docker.sock \ -p 8080:8080 -p 9100:9100 -p 2375:2375 -p 6080:6080 \ yaoapp/tai:latest for i in $(seq 1 30); do if curl -sf http://127.0.0.1:8080/healthz > /dev/null 2>&1; then - echo "Tai HTTP is ready" - break + echo "Tai Docker HTTP ready"; break fi - echo "Waiting for Tai HTTP... ($i)" - sleep 1 + echo "Waiting for Tai Docker HTTP... ($i)"; sleep 1 done + curl -sf http://127.0.0.1:8080/healthz > /dev/null 2>&1 || { + echo "::error::Tai Docker HTTP failed"; docker logs tai-docker 2>&1; exit 1 + } for i in $(seq 1 15); do if nc -z 127.0.0.1 9100 2>/dev/null; then - echo "Tai gRPC is ready" - break + echo "Tai Docker gRPC ready"; break fi - echo "Waiting for Tai gRPC... ($i)" - sleep 1 + echo "Waiting for Tai Docker gRPC... ($i)"; sleep 1 done + nc -z 127.0.0.1 9100 2>/dev/null || { + echo "::error::Tai Docker gRPC failed"; docker logs tai-docker 2>&1; exit 1 + } + + - name: Start Tai K8s instance (benchmarks) + run: | + K3D_IP=$(docker inspect k3d-tai-test-server-0 | jq -r '.[0].NetworkSettings.Networks["k3d-tai-test"].IPAddress') + echo "k3d server IP: ${K3D_IP}" + + docker run -d --name tai-k8s \ + --network k3d-tai-test \ + -p 8081:8080 -p 9101:9100 -p 6443:6443 -p 6081:6080 \ + -e TAI_K8S_UPSTREAM="tcp://${K3D_IP}:6443" \ + yaoapp/tai:latest + + for i in $(seq 1 30); do + if curl -sf http://127.0.0.1:8081/healthz > /dev/null 2>&1; then + echo "Tai K8s HTTP ready"; break + fi + echo "Waiting for Tai K8s HTTP... ($i)"; sleep 1 + done + curl -sf http://127.0.0.1:8081/healthz > /dev/null 2>&1 || { + echo "::error::Tai K8s HTTP failed"; docker logs tai-k8s 2>&1; exit 1 + } + + for i in $(seq 1 15); do + if nc -z 127.0.0.1 9101 2>/dev/null; then + echo "Tai K8s gRPC ready"; break + fi + echo "Waiting for Tai K8s gRPC... ($i)"; sleep 1 + done + nc -z 127.0.0.1 9101 2>/dev/null || { + echo "::error::Tai K8s gRPC failed"; docker logs tai-k8s 2>&1; exit 1 + } + + - name: Generate kubeconfig for benchmarks + run: | + k3d kubeconfig get tai-test > /tmp/kubeconfig-k3d.yml + sed 's|server: .*|server: https://127.0.0.1:6443|' /tmp/kubeconfig-k3d.yml \ + > ${{ runner.temp }}/kubeconfig-tai.yml - name: Run Benchmarks env: TAI_TEST_HOST: "127.0.0.1" TAI_TEST_DOCKER: "tcp://127.0.0.1:2375" TAI_TEST_GRPC_PORT: "9100" + TAI_TEST_K8S_HOST: "127.0.0.1" + TAI_TEST_K8S_PORT: "6443" + TAI_TEST_K8S_GRPC_PORT: "9101" + TAI_TEST_KUBECONFIG: "${{ runner.temp }}/kubeconfig-tai.yml" TAI_TEST_HOST_IP: "172.17.0.1" SANDBOX_TEST_REMOTE_ADDR: "tai://127.0.0.1:9100" SANDBOX_TEST_IMAGE: "yaoapp/sandbox-v2-test:latest" diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 56e93df5..a6d24efd 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -1430,6 +1430,15 @@ jobs: docker pull yaoapp/tai:latest docker pull alpine:latest + - name: Install k3d + run: curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash + + - name: Create k3d cluster + run: | + k3d cluster create tai-test --no-lb --wait --api-port 16443 + kubectl wait --for=condition=Ready node --all --timeout=60s + k3d image import alpine:latest -c tai-test + - name: Start Tai Docker instance (benchmarks) run: | docker run -d --name tai-docker \ @@ -1443,6 +1452,9 @@ jobs: fi echo "Waiting for Tai Docker HTTP... ($i)"; sleep 1 done + curl -sf http://127.0.0.1:8080/healthz > /dev/null 2>&1 || { + echo "::error::Tai Docker HTTP failed"; docker logs tai-docker 2>&1; exit 1 + } for i in $(seq 1 15); do if nc -z 127.0.0.1 9100 2>/dev/null; then @@ -1450,12 +1462,56 @@ jobs: fi echo "Waiting for Tai Docker gRPC... ($i)"; sleep 1 done + nc -z 127.0.0.1 9100 2>/dev/null || { + echo "::error::Tai Docker gRPC failed"; docker logs tai-docker 2>&1; exit 1 + } + + - name: Start Tai K8s instance (benchmarks) + run: | + K3D_IP=$(docker inspect k3d-tai-test-server-0 | jq -r '.[0].NetworkSettings.Networks["k3d-tai-test"].IPAddress') + echo "k3d server IP: ${K3D_IP}" + + docker run -d --name tai-k8s \ + --network k3d-tai-test \ + -p 8081:8080 -p 9101:9100 -p 6443:6443 -p 6081:6080 \ + -e TAI_K8S_UPSTREAM="tcp://${K3D_IP}:6443" \ + yaoapp/tai:latest + + for i in $(seq 1 30); do + if curl -sf http://127.0.0.1:8081/healthz > /dev/null 2>&1; then + echo "Tai K8s HTTP ready"; break + fi + echo "Waiting for Tai K8s HTTP... ($i)"; sleep 1 + done + curl -sf http://127.0.0.1:8081/healthz > /dev/null 2>&1 || { + echo "::error::Tai K8s HTTP failed"; docker logs tai-k8s 2>&1; exit 1 + } + + for i in $(seq 1 15); do + if nc -z 127.0.0.1 9101 2>/dev/null; then + echo "Tai K8s gRPC ready"; break + fi + echo "Waiting for Tai K8s gRPC... ($i)"; sleep 1 + done + nc -z 127.0.0.1 9101 2>/dev/null || { + echo "::error::Tai K8s gRPC failed"; docker logs tai-k8s 2>&1; exit 1 + } + + - name: Generate kubeconfig for benchmarks + run: | + k3d kubeconfig get tai-test > /tmp/kubeconfig-k3d.yml + sed 's|server: .*|server: https://127.0.0.1:6443|' /tmp/kubeconfig-k3d.yml \ + > ${{ runner.temp }}/kubeconfig-tai.yml - name: Run Benchmarks env: TAI_TEST_HOST: "127.0.0.1" TAI_TEST_DOCKER: "tcp://127.0.0.1:2375" TAI_TEST_GRPC_PORT: "9100" + TAI_TEST_K8S_HOST: "127.0.0.1" + TAI_TEST_K8S_PORT: "6443" + TAI_TEST_K8S_GRPC_PORT: "9101" + TAI_TEST_KUBECONFIG: "${{ runner.temp }}/kubeconfig-tai.yml" TAI_TEST_HOST_IP: "172.17.0.1" SANDBOX_TEST_REMOTE_ADDR: "tai://127.0.0.1:9100" SANDBOX_TEST_IMAGE: "yaoapp/sandbox-v2-test:latest"