fix(ci): 加固流水线工具链检查与部署回滚

- 新增 Preflight runner toolchain:显式校验 docker/curl/kubectl,缺失即报错并尝试 apk 自愈
  (2026-11-22 事故:act_runner 官方镜像无 docker CLI,host 模式下 docker login 直接 127)
- Checkout 改为按 SHA 浅拉取,不支持时回退完整克隆
- Setup kubectl 复用镜像内置 kubectl,缺省时才下载
- 移除已废弃的 kubectl --record(1.30+ 已删除),改用 annotate 记录 lpt/deployed-image
- Rollback 增加 deploy_attempted 标记,避免前置步骤失败时回滚掉上一次成功部署
This commit is contained in:
cat-shark
2026-11-22 22:45:55 +08:00
parent 0d27ee84d9
commit c575469f02
+87 -26
View File
@@ -15,6 +15,8 @@ env:
REGISTRY: 192.168.123.199:5000
APP: lpt-ai
REPO_URL: https://git.cat-shark.xyz/cat-shark/lpt-ai.git
# 仅在 runner 镜像里没有 kubectl 时才会触发下载(自定义镜像已内置)
KUBECTL_VERSION: v1.32.4
jobs:
build-and-deploy:
@@ -22,6 +24,53 @@ jobs:
env:
KUBECONFIG: /tmp/kubeconfig
steps:
# 2026-11-22 事故复盘:runner 以 host 模式运行(labels 带 :host),
# 官方 act_runner 镜像不含 docker/curl/kubectl,导致 docker login 直接 127。
# 这里前置检查 + 自愈,缺工具就明确报错,而不是在第二步莫名其妙失败。
- name: Preflight runner toolchain
run: |
set -euo pipefail
echo "host=$(uname -sm) uid=$(id -u)"
missing() {
local need=""
for b in docker curl base64 git; do
command -v "$b" >/dev/null 2>&1 || need="$need $b"
done
printf '%s' "$need"
}
NEED="$(missing)"
if [ -n "$NEED" ]; then
echo "runner 缺少工具:$NEED —— 尝试自愈安装(apk)"
if command -v apk >/dev/null 2>&1 && [ "$(id -u)" = 0 ]; then
apk add --no-cache docker-cli curl ca-certificates || true
fi
NEED="$(missing)"
fi
for b in docker curl kubectl git base64; do
if command -v "$b" >/dev/null 2>&1; then
echo "OK $b -> $(command -v "$b")"
else
echo "MISS $b"
fi
done
if [ -n "$NEED" ]; then
echo ""
echo "ERROR: runner 缺少必需工具:$NEED"
echo " 由于 labels 全为 :host,job 直接在 runner 容器内执行,"
echo " docker / curl / kubectl 必须存在于 runner 镜像里。"
echo " 修复:docker build -f /vol1/1000/gitea/runner.Dockerfile -t lpt-runner:0.6.1-tools /vol1/1000/gitea"
echo " cd /vol1/1000/gitea && docker compose up -d runner"
exit 1
fi
docker version --format 'docker client {{.Client.Version}} / server {{.Server.Version}}'
docker info >/dev/null
echo "OK docker daemon reachable"
- name: Validate branch ↔ environment
run: |
set -euo pipefail
@@ -33,9 +82,6 @@ jobs:
fi
ENV="${{ inputs.environment }}"
echo "branch=$BRANCH environment=$ENV sha=${{ gitea.sha }}"
# lpt-ai 与 lpt-be/lpt-fe 同规则:master 发 prod,dev 发 dev。
# 不允许跨环境出包:本服务的入参/出参结构与 lpt-be 的版本强耦合,
# 把 dev 的代码推到 prod 会让旧版 lpt-be 读到不认识的字段(静默降级)。
case "$ENV" in
prod)
case "$BRANCH" in
@@ -67,8 +113,17 @@ jobs:
- name: Checkout code
run: |
set -euo pipefail
git clone "$REPO_URL" .
git checkout "${{ gitea.sha }}"
SHA="${{ gitea.sha }}"
# 优先只取目标提交(省 10s+);服务器不支持任意 SHA 时回退完整克隆
if ! { git init -q . && git remote add origin "$REPO_URL" \
&& git fetch -q --depth 1 origin "$SHA" && git checkout -q FETCH_HEAD; } 2>/dev/null; then
echo "浅拉取失败,回退完整克隆"
rm -rf .git
git clone -q "$REPO_URL" .
git checkout -q "$SHA"
fi
echo "checked out $(git rev-parse HEAD)"
test "$(git rev-parse HEAD)" = "$SHA"
- name: Login to Docker Registry
run: |
@@ -81,29 +136,30 @@ jobs:
ENV="${{ inputs.environment }}"
TAG="${{ gitea.sha }}-$(date +%s)"
echo "Building $REGISTRY/$APP:$TAG (env tag=$ENV)"
docker build -t "$REGISTRY/$APP:$TAG" -t "$REGISTRY/$APP:$ENV" .
docker build \
-t "$REGISTRY/$APP:$TAG" \
-t "$REGISTRY/$APP:$ENV" .
docker push "$REGISTRY/$APP:$TAG"
docker push "$REGISTRY/$APP:$ENV"
if [ -n "${GITHUB_ENV:-}" ]; then
echo "IMAGE_TAG=$TAG" >> "$GITHUB_ENV"
fi
if [ -n "${GITEA_ENV:-}" ]; then
echo "IMAGE_TAG=$TAG" >> "$GITEA_ENV"
fi
# 同时打 :$TAG(可追溯)与 :$ENV(k8s 环境指针)两个 tag
mkdir -p /tmp/lpt-ci
echo "$TAG" > image_tag.txt
if [ -n "${{ runner.temp }}" ]; then
echo "$TAG" > "${{ runner.temp }}/image_tag.txt" || true
fi
echo "$TAG" > /tmp/lpt-ci/image_tag.txt
echo "IMAGE_TAG=$TAG"
- name: Setup kubectl
run: |
set -euo pipefail
curl -sLO "https://dl.k8s.io/release/v1.30.0/bin/linux/amd64/kubectl"
chmod +x kubectl
mv kubectl /usr/local/bin/
if ! command -v kubectl >/dev/null 2>&1; then
echo "kubectl 不在 runner 镜像中,临时下载 $KUBECTL_VERSION"
curl -fsSLo /usr/local/bin/kubectl \
"https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
chmod +x /usr/local/bin/kubectl
fi
echo "${{ secrets.KUBECONFIG_B64 }}" | base64 -d > /tmp/kubeconfig
chmod 600 /tmp/kubeconfig
kubectl version --client
kubectl cluster-info --request-timeout=20s
- name: Create/Update imagePullSecret
run: |
@@ -119,21 +175,21 @@ jobs:
run: |
set -euo pipefail
ENV="${{ inputs.environment }}"
IMAGE_TAG="${IMAGE_TAG:-}"
if [ -z "$IMAGE_TAG" ] && [ -f image_tag.txt ]; then
IMAGE_TAG="$(cat image_tag.txt)"
fi
if [ -z "$IMAGE_TAG" ] && [ -f "${{ runner.temp }}/image_tag.txt" ]; then
IMAGE_TAG="$(cat "${{ runner.temp }}/image_tag.txt")"
fi
IMAGE_TAG="$(cat /tmp/lpt-ci/image_tag.txt 2>/dev/null || echo '')"
if [ -z "$IMAGE_TAG" ]; then
echo "ERROR: IMAGE_TAG 为空,无法部署"
exit 1
fi
# 只有真正进入部署阶段才允许回滚,避免 build/checkout 失败时回滚掉上一次成功的版本
mkdir -p /tmp/lpt-ci
: > /tmp/lpt-ci/deploy_attempted
echo "Deploying $REGISTRY/$APP:$IMAGE_TAG -> namespace lpt-$ENV"
# kubectl 1.30+ 已移除 --record,用 annotate 记录本次部署的镜像
kubectl set image "deployment/$APP" \
"$APP=$REGISTRY/$APP:$IMAGE_TAG" \
-n "lpt-$ENV" --record
-n "lpt-$ENV"
kubectl annotate "deployment/$APP" -n "lpt-$ENV" \
"lpt/deployed-image=$IMAGE_TAG" --overwrite || true
kubectl rollout status "deployment/$APP" \
-n "lpt-$ENV" --timeout=5m
@@ -164,5 +220,10 @@ jobs:
run: |
set -euo pipefail
ENV="${{ inputs.environment }}"
if [ ! -f /tmp/lpt-ci/deploy_attempted ]; then
echo "本次运行未进入部署阶段(失败在 checkout/build/login 等前置步骤),跳过回滚"
echo "避免把上一次成功的部署回滚掉"
exit 0
fi
kubectl rollout undo "deployment/$APP" -n "lpt-$ENV" || true
kubectl rollout status "deployment/$APP" -n "lpt-$ENV" || true