name: Build and Deploy LTY on: push: branches: - main - master - dev jobs: build-and-deploy: runs-on: ubuntu-latest steps: - name: Checkout run: | git clone --depth=1 --branch=${{ github.ref_name }} https://gitea.airlabs.art/${{ github.repository }}.git . - name: Set environment by branch run: | SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7) BUILD_DATE=$(date +%Y%m%d) if [[ "${{ github.ref_name }}" == "main" || "${{ github.ref_name }}" == "master" ]]; then echo "IMAGE_TAG=prod-${BUILD_DATE}-${SHORT_SHA}" >> $GITHUB_ENV echo "CR_SERVER_ACTIVE=gitea-prod-cn-shanghai.cr.volces.com" >> $GITHUB_ENV echo "CR_USERNAME_ACTIVE=seaislee@76339115" >> $GITHUB_ENV echo "CR_PASSWORD_ACTIVE=${{ secrets.CR_PROD_PASSWORD }}" >> $GITHUB_ENV echo "CR_ORG=prod" >> $GITHUB_ENV echo "DEPLOY_ENV=production" >> $GITHUB_ENV echo "DOMAIN_API=qy-lty.airlabs.art" >> $GITHUB_ENV echo "DOMAIN_ADMIN=qy-lty-admin.airlabs.art" >> $GITHUB_ENV echo "DB_HOST=pgm-7xv4811oj11j86htzo.pg.rds.aliyuncs.com" >> $GITHUB_ENV echo "REDIS_LOCATION=redis://r-7xvat0vez5clwbzk5vpd.redis.rds.aliyuncs.com:6379/0" >> $GITHUB_ENV elif [[ "${{ github.ref_name }}" == "dev" ]]; then echo "IMAGE_TAG=dev-${BUILD_DATE}-${SHORT_SHA}" >> $GITHUB_ENV echo "CR_SERVER_ACTIVE=${{ secrets.CR_SERVER }}" >> $GITHUB_ENV echo "CR_USERNAME_ACTIVE=${{ secrets.CR_USERNAME }}" >> $GITHUB_ENV echo "CR_PASSWORD_ACTIVE=${{ secrets.CR_PASSWORD }}" >> $GITHUB_ENV echo "CR_ORG=dev" >> $GITHUB_ENV echo "DEPLOY_ENV=development" >> $GITHUB_ENV echo "DOMAIN_API=qy-lty.test.airlabs.art" >> $GITHUB_ENV echo "DOMAIN_ADMIN=qy-lty-admin.test.airlabs.art" >> $GITHUB_ENV echo "DB_HOST=pgm-7xv4811oj11j86htzo.pg.rds.aliyuncs.com" >> $GITHUB_ENV echo "REDIS_LOCATION=redis://r-7xvat0vez5clwbzk5vpd.redis.rds.aliyuncs.com:6379/0" >> $GITHUB_ENV fi - name: Login to Container Registry run: | echo "${{ env.CR_PASSWORD_ACTIVE }}" | docker login --username "${{ env.CR_USERNAME_ACTIVE }}" --password-stdin ${{ env.CR_SERVER_ACTIVE }} - name: Ensure CR Repositories Exist run: | REGISTRY_NAME=$(echo "${{ env.CR_SERVER_ACTIVE }}" | sed 's/.cr.volces.com//') python3 << PYEOF import hashlib, hmac, json, os, sys from datetime import datetime, timezone from urllib.request import Request, urlopen from urllib.parse import urlencode def create_repo(ak, sk, registry, ns, name, region='cn-shanghai'): service, host = 'cr', 'open.volcengineapi.com' t = datetime.now(timezone.utc) ds, amz = t.strftime('%Y%m%d'), t.strftime('%Y%m%dT%H%M%SZ') body = json.dumps({'Registry': registry, 'Namespace': ns, 'Name': name}) qs = urlencode(sorted({'Action': 'CreateRepository', 'Version': '2022-05-12'}.items())) ph = hashlib.sha256(body.encode()).hexdigest() sh = 'content-type;host;x-content-sha256;x-date' cr = 'POST\n/\n%s\ncontent-type:application/json\nhost:%s\nx-content-sha256:%s\nx-date:%s\n\n%s\n%s' % (qs, host, ph, amz, sh, ph) cs = '%s/%s/%s/request' % (ds, region, service) sts = 'HMAC-SHA256\n%s\n%s\n%s' % (amz, cs, hashlib.sha256(cr.encode()).hexdigest()) k = sk.encode() for p in [ds, region, service, 'request']: k = hmac.new(k, p.encode(), hashlib.sha256).digest() sig = hmac.new(k, sts.encode(), hashlib.sha256).hexdigest() auth = 'HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s' % (ak, cs, sh, sig) req = Request('https://%s?%s' % (host, qs), data=body.encode(), headers={ 'Content-Type': 'application/json', 'Host': host, 'X-Date': amz, 'X-Content-Sha256': ph, 'Authorization': auth }, method='POST') try: urlopen(req) print('Created %s/%s' % (ns, name)) except Exception as e: err = e.read().decode() if hasattr(e, 'read') else str(e) if 'Exist' in err: print('%s/%s already exists, skip' % (ns, name)) else: print('Warning: %s/%s: %s' % (ns, name, err)) ak = '${{ secrets.VOLC_ACCESS_KEY }}' sk = '${{ secrets.VOLC_SECRET_KEY }}' registry = '${REGISTRY_NAME}' ns = '${{ env.CR_ORG }}' for repo in ['lty-backend', 'lty-admin']: create_repo(ak, sk, registry, ns, repo) PYEOF - name: Build and Push Backend id: build_backend run: | set -o pipefail for attempt in 1 2 3; do echo "Build backend attempt $attempt/3..." DOCKER_BUILDKIT=0 docker build \ --tag ${{ env.CR_SERVER_ACTIVE }}/${{ env.CR_ORG }}/lty-backend:${{ env.IMAGE_TAG }} \ --tag ${{ env.CR_SERVER_ACTIVE }}/${{ env.CR_ORG }}/lty-backend:latest \ ./qy_lty 2>&1 | tee /tmp/build.log && break echo "Attempt $attempt failed, retrying in 10s..." && sleep 10 done for attempt in 1 2 3; do docker push ${{ env.CR_SERVER_ACTIVE }}/${{ env.CR_ORG }}/lty-backend:${{ env.IMAGE_TAG }} && \ docker push ${{ env.CR_SERVER_ACTIVE }}/${{ env.CR_ORG }}/lty-backend:latest && break echo "Push attempt $attempt failed, retrying in 10s..." && sleep 10 done - name: Build and Push Admin Frontend id: build_admin run: | set -o pipefail for attempt in 1 2 3; do echo "Build admin attempt $attempt/3..." DOCKER_BUILDKIT=0 docker build \ --tag ${{ env.CR_SERVER_ACTIVE }}/${{ env.CR_ORG }}/lty-admin:${{ env.IMAGE_TAG }} \ --tag ${{ env.CR_SERVER_ACTIVE }}/${{ env.CR_ORG }}/lty-admin:latest \ ./qy-lty-admin 2>&1 | tee -a /tmp/build.log && break echo "Attempt $attempt failed, retrying in 10s..." && sleep 10 done for attempt in 1 2 3; do docker push ${{ env.CR_SERVER_ACTIVE }}/${{ env.CR_ORG }}/lty-admin:${{ env.IMAGE_TAG }} && \ docker push ${{ env.CR_SERVER_ACTIVE }}/${{ env.CR_ORG }}/lty-admin:latest && break echo "Push attempt $attempt failed, retrying in 10s..." && sleep 10 done - name: Setup Kubectl run: | if ! command -v kubectl &>/dev/null; then for attempt in 1 2 3; do curl -LO "https://files.m.daocloud.io/dl.k8s.io/release/v1.28.0/bin/linux/amd64/kubectl" && break echo "Download attempt $attempt failed, retrying in 5s..." && sleep 5 done chmod +x kubectl && mv kubectl /usr/local/bin/ fi kubectl version --client - name: Set kubeconfig run: | mkdir -p $HOME/.kube if [[ "${{ github.ref_name }}" == "main" || "${{ github.ref_name }}" == "master" ]]; then echo "${{ secrets.KUBE_CONFIG }}" > $HOME/.kube/config elif [[ "${{ github.ref_name }}" == "dev" ]]; then echo "${{ secrets.VOLCANO_TEST_KUBE_CONFIG }}" > $HOME/.kube/config fi chmod 600 $HOME/.kube/config - name: Deploy to K3s id: deploy run: | echo "Environment: ${{ env.DEPLOY_ENV }}" CR_IMAGE="${{ env.CR_SERVER_ACTIVE }}/${{ env.CR_ORG }}" # Replace image placeholders sed -i "s|\${CI_REGISTRY_IMAGE}/lty-backend:latest|${CR_IMAGE}/lty-backend:${{ env.IMAGE_TAG }}|g" k8s/backend-deployment-prod.yaml sed -i "s|\${CI_REGISTRY_IMAGE}/lty-admin:latest|${CR_IMAGE}/lty-admin:${{ env.IMAGE_TAG }}|g" k8s/admin-deployment-prod.yaml # Replace domain placeholders by environment sed -i "s|qy-lty.airlabs.art|${{ env.DOMAIN_API }}|g" k8s/ingress.yaml sed -i "s|qy-lty-admin.airlabs.art|${{ env.DOMAIN_ADMIN }}|g" k8s/ingress.yaml sed -i "s|https://qy-lty.airlabs.art|https://${{ env.DOMAIN_API }}|g" k8s/admin-deployment-prod.yaml # Replace DB host by environment sed -i "s|pgm-7xv4811oj11j86htzo.pg.rds.aliyuncs.com|${{ env.DB_HOST }}|g" k8s/backend-deployment-prod.yaml # Replace Redis by environment sed -i "s|redis://r-7xvat0vez5clwbzk5vpd.redis.rds.aliyuncs.com:6379/0|${{ env.REDIS_LOCATION }}|g" k8s/backend-deployment-prod.yaml # All kubectl operations with retry for attempt in 1 2 3; do echo "Deploy attempt $attempt/3..." { # Create/update image pull secret kubectl create secret docker-registry cr-pull-secret \ --docker-server="${{ env.CR_SERVER_ACTIVE }}" \ --docker-username="${{ env.CR_USERNAME_ACTIVE }}" \ --docker-password="${{ env.CR_PASSWORD_ACTIVE }}" \ --dry-run=client -o yaml | kubectl apply -f - # Apply manifests kubectl apply -f k8s/backend-deployment-prod.yaml kubectl apply -f k8s/admin-deployment-prod.yaml kubectl apply -f k8s/ingress.yaml # Preserve real client IP kubectl patch svc traefik -n kube-system -p '{"spec":{"externalTrafficPolicy":"Local"}}' 2>/dev/null || true kubectl rollout restart deployment/lty-backend kubectl rollout restart deployment/lty-admin } 2>&1 | tee /tmp/deploy.log && break echo "Attempt $attempt failed, retrying in 10s..." sleep 10 done # ===== Log Center: failure reporting ===== - name: Report failure to Log Center if: failure() run: | BUILD_LOG="" DEPLOY_LOG="" FAILED_STEP="unknown" if [[ "${{ steps.build_backend.outcome }}" == "failure" || "${{ steps.build_admin.outcome }}" == "failure" ]]; then FAILED_STEP="build" if [ -f /tmp/build.log ]; then BUILD_LOG=$(tail -50 /tmp/build.log | sed 's/"/\\"/g' | sed ':a;N;$!ba;s/\n/\\n/g') fi elif [[ "${{ steps.deploy.outcome }}" == "failure" ]]; then FAILED_STEP="deploy" if [ -f /tmp/deploy.log ]; then DEPLOY_LOG=$(tail -50 /tmp/deploy.log | sed 's/"/\\"/g' | sed ':a;N;$!ba;s/\n/\\n/g') fi fi ERROR_LOG="${BUILD_LOG}${DEPLOY_LOG}" if [ -z "$ERROR_LOG" ]; then ERROR_LOG="No captured output. Check Gitea Actions UI for details." fi if [[ "$FAILED_STEP" == "deploy" ]]; then SOURCE="deployment" ERROR_TYPE="DeployError" else SOURCE="cicd" ERROR_TYPE="DockerBuildError" fi curl -s -X POST "https://qiyuan-log-center-api.airlabs.art/api/v1/logs/report" \ -H "Content-Type: application/json" \ -d "{ \"project_id\": \"lty\", \"environment\": \"${{ env.DEPLOY_ENV }}\", \"level\": \"ERROR\", \"source\": \"${SOURCE}\", \"commit_hash\": \"${{ github.sha }}\", \"repo_url\": \"https://gitea.airlabs.art/zyc/lty.git\", \"error\": { \"type\": \"${ERROR_TYPE}\", \"message\": \"[${FAILED_STEP}] Build and Deploy failed on branch ${{ github.ref_name }}\", \"stack_trace\": [\"${ERROR_LOG}\"] }, \"context\": { \"job_name\": \"build-and-deploy\", \"step_name\": \"${FAILED_STEP}\", \"workflow\": \"${{ github.workflow }}\", \"run_id\": \"${{ github.run_number }}\", \"branch\": \"${{ github.ref_name }}\", \"actor\": \"${{ github.actor }}\", \"commit\": \"${{ github.sha }}\", \"run_url\": \"https://gitea.airlabs.art/${{ github.repository }}/actions/runs/${{ github.run_number }}\" } }" || true # ===== Cleanup: remove unused Docker resources ===== - name: Docker Cleanup if: always() run: | docker container prune -f docker image prune -a -f docker builder prune -a -f echo "Disk usage after cleanup:" df -h / | tail -1