fix(ci): 加固流水线工具链检查与部署回滚

- 新增 Preflight runner toolchain:显式校验 docker/curl/kubectl,缺失即报错并尝试 apk 自愈
  (2026-11-22 事故:act_runner 官方镜像无 docker CLI,host 模式下 docker login 直接 127)
- Checkout 改为按 SHA 浅拉取,不支持时回退完整克隆
- Setup kubectl 复用镜像内置 kubectl,缺省时才下载
- 移除已废弃的 kubectl --record(1.30+ 已删除),改用 annotate 记录 lpt/deployed-image
- Rollback 增加 deploy_attempted 标记,避免前置步骤失败时回滚掉上一次成功部署
This commit is contained in:
cat-shark
2026-11-22 22:45:40 +08:00
parent 2505edeae7
commit 830134ef82
+99 -36
View File
@@ -15,17 +15,67 @@ env:
REGISTRY: 192.168.123.199:5000 REGISTRY: 192.168.123.199:5000
APP: lpt-fe APP: lpt-fe
REPO_URL: https://git.cat-shark.xyz/cat-shark/lpt-fe.git REPO_URL: https://git.cat-shark.xyz/cat-shark/lpt-fe.git
# 仅在 runner 镜像里没有 kubectl 时才会触发下载(自定义镜像已内置)
KUBECTL_VERSION: v1.32.4
jobs: jobs:
build-and-deploy: build-and-deploy:
runs-on: ubuntu-latest runs-on: ubuntu-latest
env:
KUBECONFIG: /tmp/kubeconfig
steps: steps:
# 2026-11-22 事故复盘:runner 以 host 模式运行(labels 带 :host),
# 官方 act_runner 镜像不含 docker/curl/kubectl,导致 docker login 直接 127。
# 这里前置检查 + 自愈,缺工具就明确报错,而不是在第二步莫名其妙失败。
- name: Preflight runner toolchain
run: |
set -euo pipefail
echo "host=$(uname -sm) uid=$(id -u)"
missing() {
local need=""
for b in docker curl base64 git; do
command -v "$b" >/dev/null 2>&1 || need="$need $b"
done
printf '%s' "$need"
}
NEED="$(missing)"
if [ -n "$NEED" ]; then
echo "runner 缺少工具:$NEED —— 尝试自愈安装(apk)"
if command -v apk >/dev/null 2>&1 && [ "$(id -u)" = 0 ]; then
apk add --no-cache docker-cli curl ca-certificates || true
fi
NEED="$(missing)"
fi
for b in docker curl kubectl git base64; do
if command -v "$b" >/dev/null 2>&1; then
echo "OK $b -> $(command -v "$b")"
else
echo "MISS $b"
fi
done
if [ -n "$NEED" ]; then
echo ""
echo "ERROR: runner 缺少必需工具:$NEED"
echo " 由于 labels 全为 :host,job 直接在 runner 容器内执行,"
echo " docker / curl / kubectl 必须存在于 runner 镜像里。"
echo " 修复:docker build -f /vol1/1000/gitea/runner.Dockerfile -t lpt-runner:0.6.1-tools /vol1/1000/gitea"
echo " cd /vol1/1000/gitea && docker compose up -d runner"
exit 1
fi
docker version --format 'docker client {{.Client.Version}} / server {{.Server.Version}}'
docker info >/dev/null
echo "OK docker daemon reachable"
- name: Validate branch ↔ environment - name: Validate branch ↔ environment
run: | run: |
set -euo pipefail set -euo pipefail
REF="${{ gitea.ref }}" REF="${{ gitea.ref }}"
BRANCH="${REF#refs/heads/}" BRANCH="${REF#refs/heads/}"
# tag 触发时拒绝
if [[ "$REF" == refs/tags/* ]]; then if [[ "$REF" == refs/tags/* ]]; then
echo "ERROR: 请从分支触发部署,不要用 tag。当前 ref=$REF" echo "ERROR: 请从分支触发部署,不要用 tag。当前 ref=$REF"
exit 1 exit 1
@@ -63,8 +113,17 @@ jobs:
- name: Checkout code - name: Checkout code
run: | run: |
set -euo pipefail set -euo pipefail
git clone "$REPO_URL" . SHA="${{ gitea.sha }}"
git checkout "${{ gitea.sha }}" # 优先只取目标提交(省 10s+);服务器不支持任意 SHA 时回退完整克隆
if ! { git init -q . && git remote add origin "$REPO_URL" \
&& git fetch -q --depth 1 origin "$SHA" && git checkout -q FETCH_HEAD; } 2>/dev/null; then
echo "浅拉取失败,回退完整克隆"
rm -rf .git
git clone -q "$REPO_URL" .
git checkout -q "$SHA"
fi
echo "checked out $(git rev-parse HEAD)"
test "$(git rev-parse HEAD)" = "$SHA"
- name: Login to Docker Registry - name: Login to Docker Registry
run: | run: |
@@ -88,57 +147,56 @@ jobs:
-t "$REGISTRY/$APP:$ENV" . -t "$REGISTRY/$APP:$ENV" .
docker push "$REGISTRY/$APP:$TAG" docker push "$REGISTRY/$APP:$TAG"
docker push "$REGISTRY/$APP:$ENV" docker push "$REGISTRY/$APP:$ENV"
# 持久化 TAG:兼容 Gitea / GitHub Actions;文件兜底避免 env 文件不支持 # 同时打 :$TAG(可追溯)与 :$ENV(k8s 环境指针)两个 tag
if [ -n "${GITHUB_ENV:-}" ]; then mkdir -p /tmp/lpt-ci
echo "IMAGE_TAG=$TAG" >> "$GITHUB_ENV"
fi
if [ -n "${GITEA_ENV:-}" ]; then
echo "IMAGE_TAG=$TAG" >> "$GITEA_ENV"
fi
echo "$TAG" > image_tag.txt echo "$TAG" > image_tag.txt
if [ -n "${{ runner.temp }}" ]; then echo "$TAG" > /tmp/lpt-ci/image_tag.txt
echo "$TAG" > "${{ runner.temp }}/image_tag.txt" || true
fi
echo "IMAGE_TAG=$TAG" echo "IMAGE_TAG=$TAG"
- name: Setup kubectl - name: Setup kubectl
run: | run: |
set -euo pipefail set -euo pipefail
curl -sLO "https://dl.k8s.io/release/v1.30.0/bin/linux/amd64/kubectl" if ! command -v kubectl >/dev/null 2>&1; then
chmod +x kubectl echo "kubectl 不在 runner 镜像中,临时下载 $KUBECTL_VERSION"
mv kubectl /usr/local/bin/ curl -fsSLo /usr/local/bin/kubectl \
"https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
chmod +x /usr/local/bin/kubectl
fi
echo "${{ secrets.KUBECONFIG_B64 }}" | base64 -d > /tmp/kubeconfig echo "${{ secrets.KUBECONFIG_B64 }}" | base64 -d > /tmp/kubeconfig
chmod 600 /tmp/kubeconfig
kubectl version --client
kubectl cluster-info --request-timeout=20s
- name: Create/Update imagePullSecret - name: Create/Update imagePullSecret
run: | run: |
set -euo pipefail set -euo pipefail
kubectl --kubeconfig=/tmp/kubeconfig create secret docker-registry regcred \ kubectl create secret docker-registry regcred \
--docker-server="$REGISTRY" \ --docker-server="$REGISTRY" \
--docker-username="${{ secrets.REGISTRY_USERNAME }}" \ --docker-username="${{ secrets.REGISTRY_USERNAME }}" \
--docker-password="${{ secrets.REGISTRY_PASSWORD }}" \ --docker-password="${{ secrets.REGISTRY_PASSWORD }}" \
-n "lpt-${{ inputs.environment }}" \ -n "lpt-${{ inputs.environment }}" \
--dry-run=client -o yaml | kubectl --kubeconfig=/tmp/kubeconfig apply -f - --dry-run=client -o yaml | kubectl apply -f -
- name: Deploy to K8s - name: Deploy to K8s
run: | run: |
set -euo pipefail set -euo pipefail
ENV="${{ inputs.environment }}" ENV="${{ inputs.environment }}"
IMAGE_TAG="${IMAGE_TAG:-}" IMAGE_TAG="$(cat /tmp/lpt-ci/image_tag.txt 2>/dev/null || echo '')"
if [ -z "$IMAGE_TAG" ] && [ -f image_tag.txt ]; then
IMAGE_TAG="$(cat image_tag.txt)"
fi
if [ -z "$IMAGE_TAG" ] && [ -f "${{ runner.temp }}/image_tag.txt" ]; then
IMAGE_TAG="$(cat "${{ runner.temp }}/image_tag.txt")"
fi
if [ -z "$IMAGE_TAG" ]; then if [ -z "$IMAGE_TAG" ]; then
echo "ERROR: IMAGE_TAG 为空,无法部署" echo "ERROR: IMAGE_TAG 为空,无法部署"
exit 1 exit 1
fi fi
# 只有真正进入部署阶段才允许回滚,避免 build/checkout 失败时回滚掉上一次成功的版本
mkdir -p /tmp/lpt-ci
: > /tmp/lpt-ci/deploy_attempted
echo "Deploying $REGISTRY/$APP:$IMAGE_TAG -> namespace lpt-$ENV" echo "Deploying $REGISTRY/$APP:$IMAGE_TAG -> namespace lpt-$ENV"
kubectl --kubeconfig=/tmp/kubeconfig set image "deployment/$APP" \ # kubectl 1.30+ 已移除 --record,用 annotate 记录本次部署的镜像
kubectl set image "deployment/$APP" \
"$APP=$REGISTRY/$APP:$IMAGE_TAG" \ "$APP=$REGISTRY/$APP:$IMAGE_TAG" \
-n "lpt-$ENV" --record -n "lpt-$ENV"
kubectl --kubeconfig=/tmp/kubeconfig rollout status "deployment/$APP" \ kubectl annotate "deployment/$APP" -n "lpt-$ENV" \
"lpt/deployed-image=$IMAGE_TAG" --overwrite || true
kubectl rollout status "deployment/$APP" \
-n "lpt-$ENV" --timeout=5m -n "lpt-$ENV" --timeout=5m
- name: Debug on failure - name: Debug on failure
@@ -146,21 +204,21 @@ jobs:
run: | run: |
ENV="${{ inputs.environment }}" ENV="${{ inputs.environment }}"
echo "=== Deployment Status ===" echo "=== Deployment Status ==="
kubectl --kubeconfig=/tmp/kubeconfig get deployment "$APP" -n "lpt-$ENV" || true kubectl get deployment "$APP" -n "lpt-$ENV" || true
echo "" echo ""
echo "=== Pod Status ===" echo "=== Pod Status ==="
kubectl --kubeconfig=/tmp/kubeconfig get pods -n "lpt-$ENV" -l "app=$APP" || true kubectl get pods -n "lpt-$ENV" -l "app=$APP" || true
echo "" echo ""
echo "=== Recent Events ===" echo "=== Recent Events ==="
kubectl --kubeconfig=/tmp/kubeconfig get events -n "lpt-$ENV" --sort-by='.lastTimestamp' | tail -20 || true kubectl get events -n "lpt-$ENV" --sort-by='.lastTimestamp' | tail -20 || true
echo "" echo ""
echo "=== Pod Describe (latest) ===" echo "=== Pod Describe (latest) ==="
POD=$(kubectl --kubeconfig=/tmp/kubeconfig get pods -n "lpt-$ENV" -l "app=$APP" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1].metadata.name}' 2>/dev/null || true) POD=$(kubectl get pods -n "lpt-$ENV" -l "app=$APP" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1].metadata.name}' 2>/dev/null || true)
if [ -n "${POD:-}" ]; then if [ -n "${POD:-}" ]; then
kubectl --kubeconfig=/tmp/kubeconfig describe pod "$POD" -n "lpt-$ENV" || true kubectl describe pod "$POD" -n "lpt-$ENV" || true
echo "" echo ""
echo "=== Pod Logs (latest) ===" echo "=== Pod Logs (latest) ==="
kubectl --kubeconfig=/tmp/kubeconfig logs "$POD" -n "lpt-$ENV" --tail=50 || true kubectl logs "$POD" -n "lpt-$ENV" --tail=50 || true
fi fi
- name: Rollback on failure - name: Rollback on failure
@@ -168,5 +226,10 @@ jobs:
run: | run: |
set -euo pipefail set -euo pipefail
ENV="${{ inputs.environment }}" ENV="${{ inputs.environment }}"
kubectl --kubeconfig=/tmp/kubeconfig rollout undo "deployment/$APP" -n "lpt-$ENV" || true if [ ! -f /tmp/lpt-ci/deploy_attempted ]; then
kubectl --kubeconfig=/tmp/kubeconfig rollout status "deployment/$APP" -n "lpt-$ENV" || true echo "本次运行未进入部署阶段(失败在 checkout/build/login 等前置步骤),跳过回滚"
echo "避免把上一次成功的部署回滚掉"
exit 0
fi
kubectl rollout undo "deployment/$APP" -n "lpt-$ENV" || true
kubectl rollout status "deployment/$APP" -n "lpt-$ENV" || true