749cddf561
Some checks failed
Build and Deploy Log Center / build-and-deploy (push) Failing after 5s
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
248 lines
11 KiB
YAML
248 lines
11 KiB
YAML
name: Build and Deploy Log Center
|
||
|
||
on:
|
||
push:
|
||
branches:
|
||
- master
|
||
- dev
|
||
|
||
jobs:
|
||
build-and-deploy:
|
||
runs-on: ubuntu-latest
|
||
steps:
|
||
- name: Checkout
|
||
run: |
|
||
git clone --depth=1 --branch=${{ github.ref_name }} https://gitea.airlabs.art/${{ github.repository }}.git .
|
||
|
||
- name: Set environment by branch
|
||
run: |
|
||
# 版本标签：环境-日期-commit
|
||
SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
|
||
BUILD_DATE=$(date +%Y%m%d)
|
||
|
||
if [[ "${{ github.ref_name }}" == "master" ]]; then
|
||
echo "IMAGE_TAG=prod-${BUILD_DATE}-${SHORT_SHA}" >> $GITHUB_ENV
|
||
echo "CR_ORG=prod" >> $GITHUB_ENV
|
||
echo "DEPLOY_ENV=production" >> $GITHUB_ENV
|
||
echo "DOMAIN_API=qiyuan-log-center-api.airlabs.art" >> $GITHUB_ENV
|
||
echo "DOMAIN_WEB=qiyuan-log-center-web.airlabs.art" >> $GITHUB_ENV
|
||
elif [[ "${{ github.ref_name }}" == "dev" ]]; then
|
||
echo "IMAGE_TAG=dev-${BUILD_DATE}-${SHORT_SHA}" >> $GITHUB_ENV
|
||
echo "CR_ORG=dev" >> $GITHUB_ENV
|
||
echo "DEPLOY_ENV=development" >> $GITHUB_ENV
|
||
echo "DOMAIN_API=qiyuan-log-center-api.test.airlabs.art" >> $GITHUB_ENV
|
||
echo "DOMAIN_WEB=qiyuan-log-center-web.test.airlabs.art" >> $GITHUB_ENV
|
||
fi
|
||
|
||
- name: Login to Volcano Engine CR
|
||
run: |
|
||
echo "${{ secrets.CR_PASSWORD }}" | docker login --username "${{ secrets.CR_USERNAME }}" --password-stdin ${{ secrets.CR_SERVER }}
|
||
|
||
# Build API Image
|
||
- name: Build and Push API
|
||
id: build-api
|
||
run: |
|
||
set -o pipefail
|
||
DOCKER_BUILDKIT=0 docker build \
|
||
--tag ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/log-center-api:${{ env.IMAGE_TAG }} \
|
||
--tag ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/log-center-api:latest \
|
||
-f ./Dockerfile \
|
||
. 2>&1 | tee /tmp/build-api.log
|
||
docker push ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/log-center-api:${{ env.IMAGE_TAG }} 2>&1 | tee -a /tmp/build-api.log
|
||
docker push ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/log-center-api:latest 2>&1 | tee -a /tmp/build-api.log
|
||
|
||
# Build Web Image
|
||
- name: Build and Push Web
|
||
id: build-web
|
||
run: |
|
||
set -o pipefail
|
||
DOCKER_BUILDKIT=0 docker build \
|
||
--build-arg VITE_API_BASE_URL=https://${{ env.DOMAIN_API }} \
|
||
--tag ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/log-center-web:${{ env.IMAGE_TAG }} \
|
||
--tag ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/log-center-web:latest \
|
||
-f ./web/Dockerfile \
|
||
./web 2>&1 | tee /tmp/build-web.log
|
||
docker push ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/log-center-web:${{ env.IMAGE_TAG }} 2>&1 | tee -a /tmp/build-web.log
|
||
docker push ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/log-center-web:latest 2>&1 | tee -a /tmp/build-web.log
|
||
|
||
# Build K8s Monitor Image
|
||
- name: Build and Push K8s Monitor
|
||
id: build-monitor
|
||
run: |
|
||
set -o pipefail
|
||
DOCKER_BUILDKIT=0 docker build \
|
||
--tag ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/k8s-pod-monitor:${{ env.IMAGE_TAG }} \
|
||
--tag ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/k8s-pod-monitor:latest \
|
||
-f ./k8s-monitor/Dockerfile \
|
||
./k8s-monitor 2>&1 | tee /tmp/build-monitor.log
|
||
docker push ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/k8s-pod-monitor:${{ env.IMAGE_TAG }} 2>&1 | tee -a /tmp/build-monitor.log
|
||
docker push ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/k8s-pod-monitor:latest 2>&1 | tee -a /tmp/build-monitor.log
|
||
|
||
- name: Setup Kubectl
|
||
run: kubectl version --client
|
||
|
||
- name: Set kubeconfig
|
||
run: |
|
||
mkdir -p $HOME/.kube
|
||
if [[ "${{ github.ref_name }}" == "master" ]]; then
|
||
echo "${{ secrets.VOLCANO_PROD_KUBE_CONFIG }}" > $HOME/.kube/config
|
||
elif [[ "${{ github.ref_name }}" == "dev" ]]; then
|
||
echo "${{ secrets.VOLCANO_TEST_KUBE_CONFIG }}" > $HOME/.kube/config
|
||
fi
|
||
chmod 600 $HOME/.kube/config
|
||
|
||
- name: Deploy to K3s
|
||
id: deploy
|
||
run: |
|
||
echo "Environment: ${{ env.DEPLOY_ENV }}"
|
||
echo "API Domain: ${{ env.DOMAIN_API }}"
|
||
echo "Web Domain: ${{ env.DOMAIN_WEB }}"
|
||
|
||
# Replace image placeholders with versioned tag
|
||
sed -i "s|\${CI_REGISTRY_IMAGE}/log-center-api:latest|${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/log-center-api:${{ env.IMAGE_TAG }}|g" k8s/api-deployment-prod.yaml
|
||
sed -i "s|\${CI_REGISTRY_IMAGE}/log-center-web:latest|${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/log-center-web:${{ env.IMAGE_TAG }}|g" k8s/web-deployment-prod.yaml
|
||
sed -i "s|\${CI_REGISTRY_IMAGE}/k8s-pod-monitor:latest|${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/k8s-pod-monitor:${{ env.IMAGE_TAG }}|g" k8s/monitor-cronjob.yaml
|
||
|
||
# Replace domain placeholders in ingress
|
||
sed -i "s|qiyuan-log-center-api.airlabs.art|${{ env.DOMAIN_API }}|g" k8s/ingress.yaml
|
||
sed -i "s|qiyuan-log-center-web.airlabs.art|${{ env.DOMAIN_WEB }}|g" k8s/ingress.yaml
|
||
|
||
# Replace LOG_CENTER_URL in monitor
|
||
sed -i "s|https://qiyuan-log-center-api.airlabs.art|https://${{ env.DOMAIN_API }}|g" k8s/monitor-cronjob.yaml
|
||
|
||
# Apply configurations and capture output
|
||
set -o pipefail
|
||
{
|
||
kubectl apply -f k8s/api-deployment-prod.yaml
|
||
kubectl apply -f k8s/web-deployment-prod.yaml
|
||
kubectl apply -f k8s/ingress.yaml
|
||
kubectl apply -f k8s/monitor-cronjob.yaml
|
||
|
||
# Restart deployments
|
||
kubectl rollout restart deployment/log-center-api
|
||
kubectl rollout restart deployment/log-center-web
|
||
} 2>&1 | tee /tmp/deploy.log
|
||
|
||
# ==================== 清理旧镜像（保留最近5个版本） ====================
|
||
|
||
- name: Clean old images
|
||
if: success()
|
||
run: |
|
||
KEEP=5
|
||
CR="${{ secrets.CR_SERVER }}"
|
||
ORG="${{ env.CR_ORG }}"
|
||
AUTH="${{ secrets.CR_USERNAME }}:${{ secrets.CR_PASSWORD }}"
|
||
|
||
for REPO in log-center-api log-center-web k8s-pod-monitor; do
|
||
echo "Cleaning ${ORG}/${REPO}..."
|
||
|
||
# 获取 Harbor token
|
||
TOKEN=$(curl -s -u "${AUTH}" "https://${CR}/service/token?service=harbor-registry&scope=repository:${ORG}/${REPO}:pull,delete" | \
|
||
python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null)
|
||
|
||
# 列出所有标签，排序后取要删除的
|
||
TAGS=$(curl -s -H "Authorization: Bearer ${TOKEN}" "https://${CR}/v2/${ORG}/${REPO}/tags/list" | \
|
||
python3 -c "
|
||
import sys,json
|
||
try:
|
||
data = json.load(sys.stdin)
|
||
tags = [t for t in data.get('tags',[]) if t != 'latest']
|
||
tags.sort(reverse=True)
|
||
for t in tags[${KEEP}:]:
|
||
print(t)
|
||
except: pass
|
||
" 2>/dev/null)
|
||
|
||
for TAG in $TAGS; do
|
||
DIGEST=$(curl -s -H "Authorization: Bearer ${TOKEN}" \
|
||
-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
|
||
"https://${CR}/v2/${ORG}/${REPO}/manifests/${TAG}" -o /dev/null -D - | \
|
||
grep -i docker-content-digest | awk '{print $2}' | tr -d '\r')
|
||
if [ -n "$DIGEST" ]; then
|
||
curl -s -X DELETE -H "Authorization: Bearer ${TOKEN}" \
|
||
"https://${CR}/v2/${ORG}/${REPO}/manifests/${DIGEST}"
|
||
echo " Deleted: ${TAG}"
|
||
fi
|
||
done
|
||
done
|
||
|
||
# ==================== CI/CD 错误上报 ====================
|
||
|
||
- name: Report failure to Log Center
|
||
if: failure()
|
||
run: |
|
||
# 收集各阶段日志（取最后 50 行）
|
||
BUILD_API_LOG=""
|
||
BUILD_WEB_LOG=""
|
||
BUILD_MONITOR_LOG=""
|
||
DEPLOY_LOG=""
|
||
FAILED_STEP="unknown"
|
||
PROJECT_ID="log_center_api"
|
||
|
||
if [[ "${{ steps.build-api.outcome }}" == "failure" ]]; then
|
||
FAILED_STEP="build-api"
|
||
PROJECT_ID="log_center_api"
|
||
if [ -f /tmp/build-api.log ]; then
|
||
BUILD_API_LOG=$(tail -50 /tmp/build-api.log | sed 's/"/\\"/g' | sed ':a;N;$!ba;s/\n/\\n/g')
|
||
fi
|
||
elif [[ "${{ steps.build-web.outcome }}" == "failure" ]]; then
|
||
FAILED_STEP="build-web"
|
||
PROJECT_ID="log_center_web"
|
||
if [ -f /tmp/build-web.log ]; then
|
||
BUILD_WEB_LOG=$(tail -50 /tmp/build-web.log | sed 's/"/\\"/g' | sed ':a;N;$!ba;s/\n/\\n/g')
|
||
fi
|
||
elif [[ "${{ steps.build-monitor.outcome }}" == "failure" ]]; then
|
||
FAILED_STEP="build-monitor"
|
||
PROJECT_ID="log_center_api"
|
||
if [ -f /tmp/build-monitor.log ]; then
|
||
BUILD_MONITOR_LOG=$(tail -50 /tmp/build-monitor.log | sed 's/"/\\"/g' | sed ':a;N;$!ba;s/\n/\\n/g')
|
||
fi
|
||
elif [[ "${{ steps.deploy.outcome }}" == "failure" ]]; then
|
||
FAILED_STEP="deploy"
|
||
PROJECT_ID="log_center_api"
|
||
if [ -f /tmp/deploy.log ]; then
|
||
DEPLOY_LOG=$(tail -50 /tmp/deploy.log | sed 's/"/\\"/g' | sed ':a;N;$!ba;s/\n/\\n/g')
|
||
fi
|
||
fi
|
||
|
||
# 合并错误日志
|
||
ERROR_LOG="${BUILD_API_LOG}${BUILD_WEB_LOG}${BUILD_MONITOR_LOG}${DEPLOY_LOG}"
|
||
if [ -z "$ERROR_LOG" ]; then
|
||
ERROR_LOG="No captured output. Check Gitea Actions UI for details."
|
||
fi
|
||
|
||
# 判断 source
|
||
if [[ "$FAILED_STEP" == "deploy" ]]; then
|
||
SOURCE="deployment"
|
||
ERROR_TYPE="DeployError"
|
||
else
|
||
SOURCE="cicd"
|
||
ERROR_TYPE="DockerBuildError"
|
||
fi
|
||
|
||
curl -s -X POST "https://${{ env.DOMAIN_API }}/api/v1/logs/report" \
|
||
-H "Content-Type: application/json" \
|
||
-d "{
|
||
\"project_id\": \"${PROJECT_ID}\",
|
||
\"environment\": \"${{ env.DEPLOY_ENV }}\",
|
||
\"level\": \"ERROR\",
|
||
\"source\": \"${SOURCE}\",
|
||
\"commit_hash\": \"${{ github.sha }}\",
|
||
\"repo_url\": \"https://gitea.airlabs.art/zyc/log-center.git\",
|
||
\"error\": {
|
||
\"type\": \"${ERROR_TYPE}\",
|
||
\"message\": \"[${FAILED_STEP}] Build and Deploy Log Center failed on branch ${{ github.ref_name }}\",
|
||
\"stack_trace\": [\"${ERROR_LOG}\"]
|
||
},
|
||
\"context\": {
|
||
\"job_name\": \"build-and-deploy\",
|
||
\"step_name\": \"${FAILED_STEP}\",
|
||
\"workflow\": \"${{ github.workflow }}\",
|
||
\"run_id\": \"${{ github.run_number }}\",
|
||
\"branch\": \"${{ github.ref_name }}\",
|
||
\"actor\": \"${{ github.actor }}\",
|
||
\"commit\": \"${{ github.sha }}\",
|
||
\"run_url\": \"https://gitea.airlabs.art/${{ github.repository }}/actions/runs/${{ github.run_number }}\"
|
||
}
|
||
}" || true
|