fix(ci): 加固流水线工具链检查与部署回滚
- 新增 Preflight runner toolchain:显式校验 docker/curl/kubectl,缺失即报错并尝试 apk 自愈 (2026-11-22 事故:act_runner 官方镜像无 docker CLI,host 模式下 docker login 直接 127) - Checkout 改为按 SHA 浅拉取,不支持时回退完整克隆 - Setup kubectl 复用镜像内置 kubectl,缺省时才下载 - 移除已废弃的 kubectl --record(1.30+ 已删除),改用 annotate 记录 lpt/deployed-image - Rollback 增加 deploy_attempted 标记,避免前置步骤失败时回滚掉上一次成功部署
This commit is contained in:
+99
-36
@@ -15,17 +15,67 @@ env:
|
|||||||
REGISTRY: 192.168.123.199:5000
|
REGISTRY: 192.168.123.199:5000
|
||||||
APP: lpt-fe
|
APP: lpt-fe
|
||||||
REPO_URL: https://git.cat-shark.xyz/cat-shark/lpt-fe.git
|
REPO_URL: https://git.cat-shark.xyz/cat-shark/lpt-fe.git
|
||||||
|
# 仅在 runner 镜像里没有 kubectl 时才会触发下载(自定义镜像已内置)
|
||||||
|
KUBECTL_VERSION: v1.32.4
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build-and-deploy:
|
build-and-deploy:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
env:
|
||||||
|
KUBECONFIG: /tmp/kubeconfig
|
||||||
steps:
|
steps:
|
||||||
|
# 2026-11-22 事故复盘:runner 以 host 模式运行(labels 带 :host),
|
||||||
|
# 官方 act_runner 镜像不含 docker/curl/kubectl,导致 docker login 直接 127。
|
||||||
|
# 这里前置检查 + 自愈,缺工具就明确报错,而不是在第二步莫名其妙失败。
|
||||||
|
- name: Preflight runner toolchain
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
echo "host=$(uname -sm) uid=$(id -u)"
|
||||||
|
|
||||||
|
missing() {
|
||||||
|
local need=""
|
||||||
|
for b in docker curl base64 git; do
|
||||||
|
command -v "$b" >/dev/null 2>&1 || need="$need $b"
|
||||||
|
done
|
||||||
|
printf '%s' "$need"
|
||||||
|
}
|
||||||
|
|
||||||
|
NEED="$(missing)"
|
||||||
|
if [ -n "$NEED" ]; then
|
||||||
|
echo "runner 缺少工具:$NEED —— 尝试自愈安装(apk)"
|
||||||
|
if command -v apk >/dev/null 2>&1 && [ "$(id -u)" = 0 ]; then
|
||||||
|
apk add --no-cache docker-cli curl ca-certificates || true
|
||||||
|
fi
|
||||||
|
NEED="$(missing)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
for b in docker curl kubectl git base64; do
|
||||||
|
if command -v "$b" >/dev/null 2>&1; then
|
||||||
|
echo "OK $b -> $(command -v "$b")"
|
||||||
|
else
|
||||||
|
echo "MISS $b"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -n "$NEED" ]; then
|
||||||
|
echo ""
|
||||||
|
echo "ERROR: runner 缺少必需工具:$NEED"
|
||||||
|
echo " 由于 labels 全为 :host,job 直接在 runner 容器内执行,"
|
||||||
|
echo " docker / curl / kubectl 必须存在于 runner 镜像里。"
|
||||||
|
echo " 修复:docker build -f /vol1/1000/gitea/runner.Dockerfile -t lpt-runner:0.6.1-tools /vol1/1000/gitea"
|
||||||
|
echo " cd /vol1/1000/gitea && docker compose up -d runner"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
docker version --format 'docker client {{.Client.Version}} / server {{.Server.Version}}'
|
||||||
|
docker info >/dev/null
|
||||||
|
echo "OK docker daemon reachable"
|
||||||
|
|
||||||
- name: Validate branch ↔ environment
|
- name: Validate branch ↔ environment
|
||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
REF="${{ gitea.ref }}"
|
REF="${{ gitea.ref }}"
|
||||||
BRANCH="${REF#refs/heads/}"
|
BRANCH="${REF#refs/heads/}"
|
||||||
# tag 触发时拒绝
|
|
||||||
if [[ "$REF" == refs/tags/* ]]; then
|
if [[ "$REF" == refs/tags/* ]]; then
|
||||||
echo "ERROR: 请从分支触发部署,不要用 tag。当前 ref=$REF"
|
echo "ERROR: 请从分支触发部署,不要用 tag。当前 ref=$REF"
|
||||||
exit 1
|
exit 1
|
||||||
@@ -63,8 +113,17 @@ jobs:
|
|||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
git clone "$REPO_URL" .
|
SHA="${{ gitea.sha }}"
|
||||||
git checkout "${{ gitea.sha }}"
|
# 优先只取目标提交(省 10s+);服务器不支持任意 SHA 时回退完整克隆
|
||||||
|
if ! { git init -q . && git remote add origin "$REPO_URL" \
|
||||||
|
&& git fetch -q --depth 1 origin "$SHA" && git checkout -q FETCH_HEAD; } 2>/dev/null; then
|
||||||
|
echo "浅拉取失败,回退完整克隆"
|
||||||
|
rm -rf .git
|
||||||
|
git clone -q "$REPO_URL" .
|
||||||
|
git checkout -q "$SHA"
|
||||||
|
fi
|
||||||
|
echo "checked out $(git rev-parse HEAD)"
|
||||||
|
test "$(git rev-parse HEAD)" = "$SHA"
|
||||||
|
|
||||||
- name: Login to Docker Registry
|
- name: Login to Docker Registry
|
||||||
run: |
|
run: |
|
||||||
@@ -88,57 +147,56 @@ jobs:
|
|||||||
-t "$REGISTRY/$APP:$ENV" .
|
-t "$REGISTRY/$APP:$ENV" .
|
||||||
docker push "$REGISTRY/$APP:$TAG"
|
docker push "$REGISTRY/$APP:$TAG"
|
||||||
docker push "$REGISTRY/$APP:$ENV"
|
docker push "$REGISTRY/$APP:$ENV"
|
||||||
# 持久化 TAG:兼容 Gitea / GitHub Actions;文件兜底避免 env 文件不支持
|
# 同时打 :$TAG(可追溯)与 :$ENV(k8s 环境指针)两个 tag
|
||||||
if [ -n "${GITHUB_ENV:-}" ]; then
|
mkdir -p /tmp/lpt-ci
|
||||||
echo "IMAGE_TAG=$TAG" >> "$GITHUB_ENV"
|
|
||||||
fi
|
|
||||||
if [ -n "${GITEA_ENV:-}" ]; then
|
|
||||||
echo "IMAGE_TAG=$TAG" >> "$GITEA_ENV"
|
|
||||||
fi
|
|
||||||
echo "$TAG" > image_tag.txt
|
echo "$TAG" > image_tag.txt
|
||||||
if [ -n "${{ runner.temp }}" ]; then
|
echo "$TAG" > /tmp/lpt-ci/image_tag.txt
|
||||||
echo "$TAG" > "${{ runner.temp }}/image_tag.txt" || true
|
|
||||||
fi
|
|
||||||
echo "IMAGE_TAG=$TAG"
|
echo "IMAGE_TAG=$TAG"
|
||||||
|
|
||||||
- name: Setup kubectl
|
- name: Setup kubectl
|
||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
curl -sLO "https://dl.k8s.io/release/v1.30.0/bin/linux/amd64/kubectl"
|
if ! command -v kubectl >/dev/null 2>&1; then
|
||||||
chmod +x kubectl
|
echo "kubectl 不在 runner 镜像中,临时下载 $KUBECTL_VERSION"
|
||||||
mv kubectl /usr/local/bin/
|
curl -fsSLo /usr/local/bin/kubectl \
|
||||||
|
"https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
|
||||||
|
chmod +x /usr/local/bin/kubectl
|
||||||
|
fi
|
||||||
echo "${{ secrets.KUBECONFIG_B64 }}" | base64 -d > /tmp/kubeconfig
|
echo "${{ secrets.KUBECONFIG_B64 }}" | base64 -d > /tmp/kubeconfig
|
||||||
|
chmod 600 /tmp/kubeconfig
|
||||||
|
kubectl version --client
|
||||||
|
kubectl cluster-info --request-timeout=20s
|
||||||
|
|
||||||
- name: Create/Update imagePullSecret
|
- name: Create/Update imagePullSecret
|
||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
kubectl --kubeconfig=/tmp/kubeconfig create secret docker-registry regcred \
|
kubectl create secret docker-registry regcred \
|
||||||
--docker-server="$REGISTRY" \
|
--docker-server="$REGISTRY" \
|
||||||
--docker-username="${{ secrets.REGISTRY_USERNAME }}" \
|
--docker-username="${{ secrets.REGISTRY_USERNAME }}" \
|
||||||
--docker-password="${{ secrets.REGISTRY_PASSWORD }}" \
|
--docker-password="${{ secrets.REGISTRY_PASSWORD }}" \
|
||||||
-n "lpt-${{ inputs.environment }}" \
|
-n "lpt-${{ inputs.environment }}" \
|
||||||
--dry-run=client -o yaml | kubectl --kubeconfig=/tmp/kubeconfig apply -f -
|
--dry-run=client -o yaml | kubectl apply -f -
|
||||||
|
|
||||||
- name: Deploy to K8s
|
- name: Deploy to K8s
|
||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
ENV="${{ inputs.environment }}"
|
ENV="${{ inputs.environment }}"
|
||||||
IMAGE_TAG="${IMAGE_TAG:-}"
|
IMAGE_TAG="$(cat /tmp/lpt-ci/image_tag.txt 2>/dev/null || echo '')"
|
||||||
if [ -z "$IMAGE_TAG" ] && [ -f image_tag.txt ]; then
|
|
||||||
IMAGE_TAG="$(cat image_tag.txt)"
|
|
||||||
fi
|
|
||||||
if [ -z "$IMAGE_TAG" ] && [ -f "${{ runner.temp }}/image_tag.txt" ]; then
|
|
||||||
IMAGE_TAG="$(cat "${{ runner.temp }}/image_tag.txt")"
|
|
||||||
fi
|
|
||||||
if [ -z "$IMAGE_TAG" ]; then
|
if [ -z "$IMAGE_TAG" ]; then
|
||||||
echo "ERROR: IMAGE_TAG 为空,无法部署"
|
echo "ERROR: IMAGE_TAG 为空,无法部署"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
# 只有真正进入部署阶段才允许回滚,避免 build/checkout 失败时回滚掉上一次成功的版本
|
||||||
|
mkdir -p /tmp/lpt-ci
|
||||||
|
: > /tmp/lpt-ci/deploy_attempted
|
||||||
echo "Deploying $REGISTRY/$APP:$IMAGE_TAG -> namespace lpt-$ENV"
|
echo "Deploying $REGISTRY/$APP:$IMAGE_TAG -> namespace lpt-$ENV"
|
||||||
kubectl --kubeconfig=/tmp/kubeconfig set image "deployment/$APP" \
|
# kubectl 1.30+ 已移除 --record,用 annotate 记录本次部署的镜像
|
||||||
|
kubectl set image "deployment/$APP" \
|
||||||
"$APP=$REGISTRY/$APP:$IMAGE_TAG" \
|
"$APP=$REGISTRY/$APP:$IMAGE_TAG" \
|
||||||
-n "lpt-$ENV" --record
|
-n "lpt-$ENV"
|
||||||
kubectl --kubeconfig=/tmp/kubeconfig rollout status "deployment/$APP" \
|
kubectl annotate "deployment/$APP" -n "lpt-$ENV" \
|
||||||
|
"lpt/deployed-image=$IMAGE_TAG" --overwrite || true
|
||||||
|
kubectl rollout status "deployment/$APP" \
|
||||||
-n "lpt-$ENV" --timeout=5m
|
-n "lpt-$ENV" --timeout=5m
|
||||||
|
|
||||||
- name: Debug on failure
|
- name: Debug on failure
|
||||||
@@ -146,21 +204,21 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
ENV="${{ inputs.environment }}"
|
ENV="${{ inputs.environment }}"
|
||||||
echo "=== Deployment Status ==="
|
echo "=== Deployment Status ==="
|
||||||
kubectl --kubeconfig=/tmp/kubeconfig get deployment "$APP" -n "lpt-$ENV" || true
|
kubectl get deployment "$APP" -n "lpt-$ENV" || true
|
||||||
echo ""
|
echo ""
|
||||||
echo "=== Pod Status ==="
|
echo "=== Pod Status ==="
|
||||||
kubectl --kubeconfig=/tmp/kubeconfig get pods -n "lpt-$ENV" -l "app=$APP" || true
|
kubectl get pods -n "lpt-$ENV" -l "app=$APP" || true
|
||||||
echo ""
|
echo ""
|
||||||
echo "=== Recent Events ==="
|
echo "=== Recent Events ==="
|
||||||
kubectl --kubeconfig=/tmp/kubeconfig get events -n "lpt-$ENV" --sort-by='.lastTimestamp' | tail -20 || true
|
kubectl get events -n "lpt-$ENV" --sort-by='.lastTimestamp' | tail -20 || true
|
||||||
echo ""
|
echo ""
|
||||||
echo "=== Pod Describe (latest) ==="
|
echo "=== Pod Describe (latest) ==="
|
||||||
POD=$(kubectl --kubeconfig=/tmp/kubeconfig get pods -n "lpt-$ENV" -l "app=$APP" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1].metadata.name}' 2>/dev/null || true)
|
POD=$(kubectl get pods -n "lpt-$ENV" -l "app=$APP" --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1].metadata.name}' 2>/dev/null || true)
|
||||||
if [ -n "${POD:-}" ]; then
|
if [ -n "${POD:-}" ]; then
|
||||||
kubectl --kubeconfig=/tmp/kubeconfig describe pod "$POD" -n "lpt-$ENV" || true
|
kubectl describe pod "$POD" -n "lpt-$ENV" || true
|
||||||
echo ""
|
echo ""
|
||||||
echo "=== Pod Logs (latest) ==="
|
echo "=== Pod Logs (latest) ==="
|
||||||
kubectl --kubeconfig=/tmp/kubeconfig logs "$POD" -n "lpt-$ENV" --tail=50 || true
|
kubectl logs "$POD" -n "lpt-$ENV" --tail=50 || true
|
||||||
fi
|
fi
|
||||||
|
|
||||||
- name: Rollback on failure
|
- name: Rollback on failure
|
||||||
@@ -168,5 +226,10 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
ENV="${{ inputs.environment }}"
|
ENV="${{ inputs.environment }}"
|
||||||
kubectl --kubeconfig=/tmp/kubeconfig rollout undo "deployment/$APP" -n "lpt-$ENV" || true
|
if [ ! -f /tmp/lpt-ci/deploy_attempted ]; then
|
||||||
kubectl --kubeconfig=/tmp/kubeconfig rollout status "deployment/$APP" -n "lpt-$ENV" || true
|
echo "本次运行未进入部署阶段(失败在 checkout/build/login 等前置步骤),跳过回滚"
|
||||||
|
echo "避免把上一次成功的部署回滚掉"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
kubectl rollout undo "deployment/$APP" -n "lpt-$ENV" || true
|
||||||
|
kubectl rollout status "deployment/$APP" -n "lpt-$ENV" || true
|
||||||
|
|||||||
Reference in New Issue
Block a user