fix(ci): 加固流水线工具链检查与部署回滚
- 新增 Preflight runner toolchain:显式校验 docker/curl/kubectl,缺失即报错并尝试 apk 自愈 (2026-11-22 事故:act_runner 官方镜像无 docker CLI,host 模式下 docker login 直接 127) - Checkout 改为按 SHA 浅拉取,不支持时回退完整克隆 - Setup kubectl 复用镜像内置 kubectl,缺省时才下载 - 移除已废弃的 kubectl --record(1.30+ 已删除),改用 annotate 记录 lpt/deployed-image - Rollback 增加 deploy_attempted 标记,避免前置步骤失败时回滚掉上一次成功部署
This commit is contained in:
+99
-36
@@ -15,17 +15,67 @@ env:
|
||||
REGISTRY: 192.168.123.199:5000
|
||||
APP: lpt-fe
|
||||
REPO_URL: https://git.cat-shark.xyz/cat-shark/lpt-fe.git
|
||||
# 仅在 runner 镜像里没有 kubectl 时才会触发下载(自定义镜像已内置)
|
||||
KUBECTL_VERSION: v1.32.4
|
||||
|
||||
jobs:
|
||||
build-and-deploy:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
KUBECONFIG: /tmp/kubeconfig
|
||||
steps:
|
||||
# 2026-11-22 事故复盘:runner 以 host 模式运行(labels 带 :host),
|
||||
# 官方 act_runner 镜像不含 docker/curl/kubectl,导致 docker login 直接 127。
|
||||
# 这里前置检查 + 自愈,缺工具就明确报错,而不是在第二步莫名其妙失败。
|
||||
- name: Preflight runner toolchain
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "host=$(uname -sm) uid=$(id -u)"
|
||||
|
||||
missing() {
|
||||
local need=""
|
||||
for b in docker curl base64 git; do
|
||||
command -v "$b" >/dev/null 2>&1 || need="$need $b"
|
||||
done
|
||||
printf '%s' "$need"
|
||||
}
|
||||
|
||||
NEED="$(missing)"
|
||||
if [ -n "$NEED" ]; then
|
||||
echo "runner 缺少工具:$NEED —— 尝试自愈安装(apk)"
|
||||
if command -v apk >/dev/null 2>&1 && [ "$(id -u)" = 0 ]; then
|
||||
apk add --no-cache docker-cli curl ca-certificates || true
|
||||
fi
|
||||
NEED="$(missing)"
|
||||
fi
|
||||
|
||||
for b in docker curl kubectl git base64; do
|
||||
if command -v "$b" >/dev/null 2>&1; then
|
||||
echo "OK $b -> $(command -v "$b")"
|
||||
else
|
||||
echo "MISS $b"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -n "$NEED" ]; then
|
||||
echo ""
|
||||
echo "ERROR: runner 缺少必需工具:$NEED"
|
||||
echo " 由于 labels 全为 :host,job 直接在 runner 容器内执行,"
|
||||
echo " docker / curl / kubectl 必须存在于 runner 镜像里。"
|
||||
echo " 修复:docker build -f /vol1/1000/gitea/runner.Dockerfile -t lpt-runner:0.6.1-tools /vol1/1000/gitea"
|
||||
echo " cd /vol1/1000/gitea && docker compose up -d runner"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker version --format 'docker client {{.Client.Version}} / server {{.Server.Version}}'
|
||||
docker info >/dev/null
|
||||
echo "OK docker daemon reachable"
|
||||
|
||||
- name: Validate branch ↔ environment
|
||||
run: |
|
||||
set -euo pipefail
|
||||
REF="${{ gitea.ref }}"
|
||||
BRANCH="${REF#refs/heads/}"
|
||||
# tag 触发时拒绝
|
||||
if [[ "$REF" == refs/tags/* ]]; then
|
||||
echo "ERROR: 请从分支触发部署,不要用 tag。当前 ref=$REF"
|
||||
exit 1
|
||||
@@ -63,8 +113,17 @@ jobs:
|
||||
- name: Checkout code
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git clone "$REPO_URL" .
|
||||
git checkout "${{ gitea.sha }}"
|
||||
SHA="${{ gitea.sha }}"
|
||||
# 优先只取目标提交(省 10s+);服务器不支持任意 SHA 时回退完整克隆
|
||||
if ! { git init -q . && git remote add origin "$REPO_URL" \
|
||||
&& git fetch -q --depth 1 origin "$SHA" && git checkout -q FETCH_HEAD; } 2>/dev/null; then
|
||||
echo "浅拉取失败,回退完整克隆"
|
||||
rm -rf .git
|
||||
git clone -q "$REPO_URL" .
|
||||
git checkout -q "$SHA"
|
||||
fi
|
||||
echo "checked out $(git rev-parse HEAD)"
|
||||
test "$(git rev-parse HEAD)" = "$SHA"
|
||||
|
||||
- name: Login to Docker Registry
|
||||
run: |
|
||||
@@ -88,57 +147,56 @@ jobs:
|
||||
-t "$REGISTRY/$APP:$ENV" .
|
||||
docker push "$REGISTRY/$APP:$TAG"
|
||||
docker push "$REGISTRY/$APP:$ENV"
|
||||
# 持久化 TAG:兼容 Gitea / GitHub Actions;文件兜底避免 env 文件不支持
|
||||
if [ -n "${GITHUB_ENV:-}" ]; then
|
||||
echo "IMAGE_TAG=$TAG" >> "$GITHUB_ENV"
|
||||
fi
|
||||
if [ -n "${GITEA_ENV:-}" ]; then
|
||||
echo "IMAGE_TAG=$TAG" >> "$GITEA_ENV"
|
||||
fi
|
||||
# 同时打 :$TAG(可追溯)与 :$ENV(k8s 环境指针)两个 tag
|
||||
mkdir -p /tmp/lpt-ci
|
||||
echo "$TAG" > image_tag.txt
|
||||
if [ -n "${{ runner.temp }}" ]; then
|
||||
echo "$TAG" > "${{ runner.temp }}/image_tag.txt" || true
|
||||
fi
|
||||
echo "$TAG" > /tmp/lpt-ci/image_tag.txt
|
||||
echo "IMAGE_TAG=$TAG"
|
||||
|
||||
- name: Setup kubectl
|
||||
run: |
|
||||
set -euo pipefail
|
||||
curl -sLO "https://dl.k8s.io/release/v1.30.0/bin/linux/amd64/kubectl"
|
||||
chmod +x kubectl
|
||||
mv kubectl /usr/local/bin/
|
||||
if ! command -v kubectl >/dev/null 2>&1; then
|
||||
echo "kubectl 不在 runner 镜像中,临时下载 $KUBECTL_VERSION"
|
||||
curl -fsSLo /usr/local/bin/kubectl \
|
||||
"https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
|
||||
chmod +x /usr/local/bin/kubectl
|
||||
fi
|
||||
echo "${{ secrets.KUBECONFIG_B64 }}" | base64 -d > /tmp/kubeconfig
|
||||
chmod 600 /tmp/kubeconfig
|
||||
kubectl version --client
|
||||
kubectl cluster-info --request-timeout=20s
|
||||
|
||||
- name: Create/Update imagePullSecret
|
||||
run: |
|
||||
set -euo pipefail
|
||||
kubectl --kubeconfig=/tmp/kubeconfig create secret docker-registry regcred \
|
||||
kubectl create secret docker-registry regcred \
|
||||
--docker-server="$REGISTRY" \
|
||||
--docker-username="${{ secrets.REGISTRY_USERNAME }}" \
|
||||
--docker-password="${{ secrets.REGISTRY_PASSWORD }}" \
|
||||
-n "lpt-${{ inputs.environment }}" \
|
||||
--dry-run=client -o yaml | kubectl --kubeconfig=/tmp/kubeconfig apply -f -
|
||||
--dry-run=client -o yaml | kubectl apply -f -
|
||||
|
||||
- name: Deploy to K8s
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ENV="${{ inputs.environment }}"
|
||||
IMAGE_TAG="${IMAGE_TAG:-}"
|
||||
if [ -z "$IMAGE_TAG" ] && [ -f image_tag.txt ]; then
|
||||
IMAGE_TAG="$(cat image_tag.txt)"
|
||||
fi
|
||||
if [ -z "$IMAGE_TAG" ] && [ -f "${{ runner.temp }}/image_tag.txt" ]; then
|
||||
IMAGE_TAG="$(cat "${{ runner.temp }}/image_tag.txt")"
|
||||
fi
|
||||
IMAGE_TAG="$(cat /tmp/lpt-ci/image_tag.txt 2>/dev/null || echo '')"
|
||||
if [ -z "$IMAGE_TAG" ]; then
|
||||
echo "ERROR: IMAGE_TAG 为空,无法部署"
|
||||
exit 1
|
||||
fi
|
||||
# 只有真正进入部署阶段才允许回滚,避免 build/checkout 失败时回滚掉上一次成功的版本
|
||||
mkdir -p /tmp/lpt-ci
|
||||
: > /tmp/lpt-ci/deploy_attempted
|
||||
echo "Deploying $REGISTRY/$APP:$IMAGE_TAG -> namespace lpt-$ENV"
|
||||
kubectl --kubeconfig=/tmp/kubeconfig set image "deployment/$APP" \
|
||||
# kubectl 1.30+ 已移除 --record,用 annotate 记录本次部署的镜像
|
||||
kubectl set image "deployment/$APP" \
|
||||
"$APP=$REGISTRY/$APP:$IMAGE_TAG" \
|
||||
-n "lpt-$ENV" --record
|
||||
kubectl --kubeconfig=/tmp/kubeconfig rollout status "deployment/$APP" \
|
||||
-n "lpt-$ENV"
|
||||
kubectl annotate "deployment/$APP" -n "lpt-$ENV" \
|
||||
"lpt/deployed-image=$IMAGE_TAG" --overwrite || true
|
||||
kubectl rollout status "deployment/$APP" \
|
||||
-n "lpt-$ENV" --timeout=5m
|
||||
|
||||
- name: Debug on failure
|
||||
@@ -146,21 +204,21 @@ jobs:
|
||||
run: |
|
||||
ENV="${{ inputs.environment }}"
|
||||
echo "=== Deployment Status ==="
|
||||
kubectl --kubeconfig=/tmp/kubeconfig get deployment "$APP" -n "lpt-$ENV" || true
|
||||
kubectl get deployment "$APP" -n "lpt-$ENV" || true
|
||||
echo ""
|
||||
echo "=== Pod Status ==="
|
||||
kubectl --kubeconfig=/tmp/kubeconfig get pods -n "lpt-$ENV" -l "app=$APP" || true
|
||||
kubectl get pods -n "lpt-$ENV" -l "app=$APP" || true
|
||||
echo ""
|
||||
echo "=== Recent Events ==="
|
||||
kubectl --kubeconfig=/tmp/kubeconfig get events -n "lpt-$ENV" --sort-by='.lastTimestamp' | tail -20 || true
|
||||
kubectl get events -n "lpt-$ENV" --sort-by='.lastTimestamp' | tail -20 || true
|
||||
echo ""
|
||||
echo "=== Pod Describe (latest) ==="
|
||||
POD=$(kubectl --kubeconfig=/tmp/kubeconfig get pods -n "lpt-$ENV" -l "app=$APP" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1].metadata.name}' 2>/dev/null || true)
|
||||
POD=$(kubectl get pods -n "lpt-$ENV" -l "app=$APP" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1].metadata.name}' 2>/dev/null || true)
|
||||
if [ -n "${POD:-}" ]; then
|
||||
kubectl --kubeconfig=/tmp/kubeconfig describe pod "$POD" -n "lpt-$ENV" || true
|
||||
kubectl describe pod "$POD" -n "lpt-$ENV" || true
|
||||
echo ""
|
||||
echo "=== Pod Logs (latest) ==="
|
||||
kubectl --kubeconfig=/tmp/kubeconfig logs "$POD" -n "lpt-$ENV" --tail=50 || true
|
||||
kubectl logs "$POD" -n "lpt-$ENV" --tail=50 || true
|
||||
fi
|
||||
|
||||
- name: Rollback on failure
|
||||
@@ -168,5 +226,10 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ENV="${{ inputs.environment }}"
|
||||
kubectl --kubeconfig=/tmp/kubeconfig rollout undo "deployment/$APP" -n "lpt-$ENV" || true
|
||||
kubectl --kubeconfig=/tmp/kubeconfig rollout status "deployment/$APP" -n "lpt-$ENV" || true
|
||||
if [ ! -f /tmp/lpt-ci/deploy_attempted ]; then
|
||||
echo "本次运行未进入部署阶段(失败在 checkout/build/login 等前置步骤),跳过回滚"
|
||||
echo "避免把上一次成功的部署回滚掉"
|
||||
exit 0
|
||||
fi
|
||||
kubectl rollout undo "deployment/$APP" -n "lpt-$ENV" || true
|
||||
kubectl rollout status "deployment/$APP" -n "lpt-$ENV" || true
|
||||
|
||||
Reference in New Issue
Block a user