Compare commits
19 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
58ee345895 | ||
|
|
749cddf561 | ||
|
|
a2c6e54346 | ||
|
|
31d6277531 | ||
|
|
72703b84bb | ||
|
|
9606d4a94b | ||
|
|
8920dad45c | ||
|
|
0fdc62b99e | ||
|
|
832bfdd80d | ||
|
|
eb75fa126c | ||
|
|
836c8e9c18 | ||
|
|
e9b9a4fd2f | ||
|
|
b9a4dd62c4 | ||
|
|
34938a1aa0 | ||
|
|
7e44d5b786 | ||
|
|
0d9acd5bbc | ||
|
|
b2a2b1079f | ||
|
|
eeb27514bb | ||
|
|
cbcf7737d0 |
@ -3,88 +3,112 @@ name: Build and Deploy Log Center
|
|||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- main
|
|
||||||
- master
|
- master
|
||||||
|
- dev
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build-and-deploy:
|
build-and-deploy:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v3
|
run: |
|
||||||
|
git clone --depth=1 --branch=${{ github.ref_name }} https://gitea.airlabs.art/${{ github.repository }}.git .
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set environment by branch
|
||||||
uses: docker/setup-buildx-action@v2
|
run: |
|
||||||
with:
|
# 版本标签:环境-日期-commit
|
||||||
config-inline: |
|
SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
|
||||||
[registry."docker.io"]
|
BUILD_DATE=$(date +%Y%m%d)
|
||||||
mirrors = ["https://docker.m.daocloud.io", "https://docker.1panel.live", "https://hub.rat.dev"]
|
|
||||||
|
|
||||||
- name: Login to Huawei Cloud SWR
|
if [[ "${{ github.ref_name }}" == "master" ]]; then
|
||||||
uses: docker/login-action@v2
|
echo "IMAGE_TAG=prod-${BUILD_DATE}-${SHORT_SHA}" >> $GITHUB_ENV
|
||||||
with:
|
echo "CR_ORG=prod" >> $GITHUB_ENV
|
||||||
registry: ${{ secrets.SWR_SERVER }}
|
echo "DEPLOY_ENV=production" >> $GITHUB_ENV
|
||||||
username: ${{ secrets.SWR_USERNAME }}
|
echo "DOMAIN_API=qiyuan-log-center-api.airlabs.art" >> $GITHUB_ENV
|
||||||
password: ${{ secrets.SWR_PASSWORD }}
|
echo "DOMAIN_WEB=qiyuan-log-center-web.airlabs.art" >> $GITHUB_ENV
|
||||||
|
elif [[ "${{ github.ref_name }}" == "dev" ]]; then
|
||||||
|
echo "IMAGE_TAG=dev-${BUILD_DATE}-${SHORT_SHA}" >> $GITHUB_ENV
|
||||||
|
echo "CR_ORG=dev" >> $GITHUB_ENV
|
||||||
|
echo "DEPLOY_ENV=development" >> $GITHUB_ENV
|
||||||
|
echo "DOMAIN_API=qiyuan-log-center-api.test.airlabs.art" >> $GITHUB_ENV
|
||||||
|
echo "DOMAIN_WEB=qiyuan-log-center-web.test.airlabs.art" >> $GITHUB_ENV
|
||||||
|
fi
|
||||||
|
|
||||||
|
- name: Login to Volcano Engine CR
|
||||||
|
run: |
|
||||||
|
echo "${{ secrets.CR_PASSWORD }}" | docker login --username "${{ secrets.CR_USERNAME }}" --password-stdin ${{ secrets.CR_SERVER }}
|
||||||
|
|
||||||
# Build API Image
|
# Build API Image
|
||||||
- name: Build and Push API
|
- name: Build and Push API
|
||||||
id: build-api
|
id: build-api
|
||||||
run: |
|
run: |
|
||||||
set -o pipefail
|
set -o pipefail
|
||||||
docker buildx build \
|
DOCKER_BUILDKIT=0 docker build \
|
||||||
--push \
|
--tag ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/log-center-api:${{ env.IMAGE_TAG }} \
|
||||||
--provenance=false \
|
--tag ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/log-center-api:latest \
|
||||||
--tag ${{ secrets.SWR_SERVER }}/${{ secrets.SWR_ORG }}/log-center-api:latest \
|
|
||||||
-f ./Dockerfile \
|
-f ./Dockerfile \
|
||||||
. 2>&1 | tee /tmp/build-api.log
|
. 2>&1 | tee /tmp/build-api.log
|
||||||
|
docker push ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/log-center-api:${{ env.IMAGE_TAG }} 2>&1 | tee -a /tmp/build-api.log
|
||||||
|
docker push ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/log-center-api:latest 2>&1 | tee -a /tmp/build-api.log
|
||||||
|
|
||||||
# Build Web Image
|
# Build Web Image
|
||||||
- name: Build and Push Web
|
- name: Build and Push Web
|
||||||
id: build-web
|
id: build-web
|
||||||
run: |
|
run: |
|
||||||
set -o pipefail
|
set -o pipefail
|
||||||
docker buildx build \
|
DOCKER_BUILDKIT=0 docker build \
|
||||||
--push \
|
--build-arg VITE_API_BASE_URL=https://${{ env.DOMAIN_API }} \
|
||||||
--provenance=false \
|
--tag ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/log-center-web:${{ env.IMAGE_TAG }} \
|
||||||
--build-arg VITE_API_BASE_URL=https://qiyuan-log-center-api.airlabs.art \
|
--tag ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/log-center-web:latest \
|
||||||
--tag ${{ secrets.SWR_SERVER }}/${{ secrets.SWR_ORG }}/log-center-web:latest \
|
|
||||||
-f ./web/Dockerfile \
|
-f ./web/Dockerfile \
|
||||||
./web 2>&1 | tee /tmp/build-web.log
|
./web 2>&1 | tee /tmp/build-web.log
|
||||||
|
docker push ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/log-center-web:${{ env.IMAGE_TAG }} 2>&1 | tee -a /tmp/build-web.log
|
||||||
|
docker push ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/log-center-web:latest 2>&1 | tee -a /tmp/build-web.log
|
||||||
|
|
||||||
# Build K8s Monitor Image
|
# Build K8s Monitor Image
|
||||||
- name: Build and Push K8s Monitor
|
- name: Build and Push K8s Monitor
|
||||||
id: build-monitor
|
id: build-monitor
|
||||||
run: |
|
run: |
|
||||||
set -o pipefail
|
set -o pipefail
|
||||||
docker buildx build \
|
DOCKER_BUILDKIT=0 docker build \
|
||||||
--push \
|
--tag ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/k8s-pod-monitor:${{ env.IMAGE_TAG }} \
|
||||||
--provenance=false \
|
--tag ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/k8s-pod-monitor:latest \
|
||||||
--tag ${{ secrets.SWR_SERVER }}/${{ secrets.SWR_ORG }}/k8s-pod-monitor:latest \
|
|
||||||
-f ./k8s-monitor/Dockerfile \
|
-f ./k8s-monitor/Dockerfile \
|
||||||
./k8s-monitor 2>&1 | tee /tmp/build-monitor.log
|
./k8s-monitor 2>&1 | tee /tmp/build-monitor.log
|
||||||
|
docker push ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/k8s-pod-monitor:${{ env.IMAGE_TAG }} 2>&1 | tee -a /tmp/build-monitor.log
|
||||||
|
docker push ${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/k8s-pod-monitor:latest 2>&1 | tee -a /tmp/build-monitor.log
|
||||||
|
|
||||||
- name: Setup Kubectl
|
- name: Setup Kubectl
|
||||||
|
run: kubectl version --client
|
||||||
|
|
||||||
|
- name: Set kubeconfig
|
||||||
run: |
|
run: |
|
||||||
curl -LO "https://files.m.daocloud.io/dl.k8s.io/release/v1.28.2/bin/linux/amd64/kubectl"
|
mkdir -p $HOME/.kube
|
||||||
chmod +x kubectl
|
if [[ "${{ github.ref_name }}" == "master" ]]; then
|
||||||
mv kubectl /usr/local/bin/
|
echo "${{ secrets.VOLCANO_PROD_KUBE_CONFIG }}" > $HOME/.kube/config
|
||||||
|
elif [[ "${{ github.ref_name }}" == "dev" ]]; then
|
||||||
|
echo "${{ secrets.VOLCANO_TEST_KUBE_CONFIG }}" > $HOME/.kube/config
|
||||||
|
fi
|
||||||
|
chmod 600 $HOME/.kube/config
|
||||||
|
|
||||||
- name: Deploy to K3s
|
- name: Deploy to K3s
|
||||||
uses: Azure/k8s-set-context@v3
|
|
||||||
with:
|
|
||||||
method: kubeconfig
|
|
||||||
kubeconfig: ${{ secrets.KUBE_CONFIG }}
|
|
||||||
|
|
||||||
- name: Update K8s Manifests
|
|
||||||
id: deploy
|
id: deploy
|
||||||
run: |
|
run: |
|
||||||
echo "Environment: Production"
|
echo "Environment: ${{ env.DEPLOY_ENV }}"
|
||||||
|
echo "API Domain: ${{ env.DOMAIN_API }}"
|
||||||
|
echo "Web Domain: ${{ env.DOMAIN_WEB }}"
|
||||||
|
|
||||||
# Replace image placeholders
|
# Replace image placeholders with versioned tag
|
||||||
sed -i "s|\${CI_REGISTRY_IMAGE}/log-center-api:latest|${{ secrets.SWR_SERVER }}/${{ secrets.SWR_ORG }}/log-center-api:latest|g" k8s/api-deployment-prod.yaml
|
sed -i "s|\${CI_REGISTRY_IMAGE}/log-center-api:latest|${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/log-center-api:${{ env.IMAGE_TAG }}|g" k8s/api-deployment-prod.yaml
|
||||||
sed -i "s|\${CI_REGISTRY_IMAGE}/log-center-web:latest|${{ secrets.SWR_SERVER }}/${{ secrets.SWR_ORG }}/log-center-web:latest|g" k8s/web-deployment-prod.yaml
|
sed -i "s|\${CI_REGISTRY_IMAGE}/log-center-web:latest|${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/log-center-web:${{ env.IMAGE_TAG }}|g" k8s/web-deployment-prod.yaml
|
||||||
sed -i "s|\${CI_REGISTRY_IMAGE}/k8s-pod-monitor:latest|${{ secrets.SWR_SERVER }}/${{ secrets.SWR_ORG }}/k8s-pod-monitor:latest|g" k8s/monitor-cronjob.yaml
|
sed -i "s|\${CI_REGISTRY_IMAGE}/k8s-pod-monitor:latest|${{ secrets.CR_SERVER }}/${{ env.CR_ORG }}/k8s-pod-monitor:${{ env.IMAGE_TAG }}|g" k8s/monitor-cronjob.yaml
|
||||||
|
|
||||||
|
# Replace domain placeholders in ingress
|
||||||
|
sed -i "s|qiyuan-log-center-api.airlabs.art|${{ env.DOMAIN_API }}|g" k8s/ingress.yaml
|
||||||
|
sed -i "s|qiyuan-log-center-web.airlabs.art|${{ env.DOMAIN_WEB }}|g" k8s/ingress.yaml
|
||||||
|
|
||||||
|
# Replace LOG_CENTER_URL in monitor
|
||||||
|
sed -i "s|https://qiyuan-log-center-api.airlabs.art|https://${{ env.DOMAIN_API }}|g" k8s/monitor-cronjob.yaml
|
||||||
|
|
||||||
# Apply configurations and capture output
|
# Apply configurations and capture output
|
||||||
set -o pipefail
|
set -o pipefail
|
||||||
@ -100,6 +124,7 @@ jobs:
|
|||||||
} 2>&1 | tee /tmp/deploy.log
|
} 2>&1 | tee /tmp/deploy.log
|
||||||
|
|
||||||
# ==================== CI/CD 错误上报 ====================
|
# ==================== CI/CD 错误上报 ====================
|
||||||
|
# 注:火山引擎 CR 小微版不支持 API 删除镜像,需在控制台手动清理旧版本
|
||||||
|
|
||||||
- name: Report failure to Log Center
|
- name: Report failure to Log Center
|
||||||
if: failure()
|
if: failure()
|
||||||
@ -153,11 +178,11 @@ jobs:
|
|||||||
ERROR_TYPE="DockerBuildError"
|
ERROR_TYPE="DockerBuildError"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
curl -s -X POST "https://qiyuan-log-center-api.airlabs.art/api/v1/logs/report" \
|
curl -s -X POST "https://${{ env.DOMAIN_API }}/api/v1/logs/report" \
|
||||||
-H "Content-Type: application/json" \
|
-H "Content-Type: application/json" \
|
||||||
-d "{
|
-d "{
|
||||||
\"project_id\": \"${PROJECT_ID}\",
|
\"project_id\": \"${PROJECT_ID}\",
|
||||||
\"environment\": \"${{ github.ref_name }}\",
|
\"environment\": \"${{ env.DEPLOY_ENV }}\",
|
||||||
\"level\": \"ERROR\",
|
\"level\": \"ERROR\",
|
||||||
\"source\": \"${SOURCE}\",
|
\"source\": \"${SOURCE}\",
|
||||||
\"commit_hash\": \"${{ github.sha }}\",
|
\"commit_hash\": \"${{ github.sha }}\",
|
||||||
|
|||||||
@ -5,7 +5,7 @@ WORKDIR /app
|
|||||||
|
|
||||||
# Install dependencies
|
# Install dependencies
|
||||||
COPY requirements.txt ./
|
COPY requirements.txt ./
|
||||||
RUN pip install --no-cache-dir -r requirements.txt
|
RUN pip install --no-cache-dir -i https://mirrors.aliyun.com/pypi/simple/ -r requirements.txt
|
||||||
|
|
||||||
# Production stage
|
# Production stage
|
||||||
FROM python:3.12-slim AS production-stage
|
FROM python:3.12-slim AS production-stage
|
||||||
|
|||||||
@ -11,10 +11,10 @@ load_dotenv()
|
|||||||
DB_USER = os.getenv("DB_USER")
|
DB_USER = os.getenv("DB_USER")
|
||||||
DB_PASSWORD = os.getenv("DB_PASSWORD")
|
DB_PASSWORD = os.getenv("DB_PASSWORD")
|
||||||
DB_HOST = os.getenv("DB_HOST")
|
DB_HOST = os.getenv("DB_HOST")
|
||||||
DB_PORT = os.getenv("DB_PORT", "5432")
|
DB_PORT = os.getenv("DB_PORT", "3306")
|
||||||
DB_NAME = os.getenv("DB_NAME")
|
DB_NAME = os.getenv("DB_NAME")
|
||||||
|
|
||||||
DATABASE_URL = f"postgresql+asyncpg://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
|
DATABASE_URL = f"mysql+aiomysql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}?charset=utf8mb4"
|
||||||
|
|
||||||
_engine = None
|
_engine = None
|
||||||
|
|
||||||
@ -57,9 +57,9 @@ async def init_db():
|
|||||||
"ALTER TABLE repairtask ADD COLUMN IF NOT EXISTS failure_reason TEXT",
|
"ALTER TABLE repairtask ADD COLUMN IF NOT EXISTS failure_reason TEXT",
|
||||||
# Log source support
|
# Log source support
|
||||||
"ALTER TABLE errorlog ADD COLUMN IF NOT EXISTS source VARCHAR(20) DEFAULT 'runtime'",
|
"ALTER TABLE errorlog ADD COLUMN IF NOT EXISTS source VARCHAR(20) DEFAULT 'runtime'",
|
||||||
"ALTER TABLE errorlog ALTER COLUMN file_path DROP NOT NULL",
|
"ALTER TABLE errorlog MODIFY COLUMN file_path VARCHAR(255) NULL",
|
||||||
"ALTER TABLE errorlog ALTER COLUMN line_number DROP NOT NULL",
|
"ALTER TABLE errorlog MODIFY COLUMN line_number INTEGER NULL",
|
||||||
"CREATE INDEX IF NOT EXISTS ix_errorlog_source ON errorlog (source)",
|
"CREATE INDEX ix_errorlog_source ON errorlog (source)",
|
||||||
# ErrorLog failure_reason
|
# ErrorLog failure_reason
|
||||||
"ALTER TABLE errorlog ADD COLUMN IF NOT EXISTS failure_reason TEXT",
|
"ALTER TABLE errorlog ADD COLUMN IF NOT EXISTS failure_reason TEXT",
|
||||||
# Bug severity (1-10 AI评估等级)
|
# Bug severity (1-10 AI评估等级)
|
||||||
|
|||||||
29
app/main.py
29
app/main.py
@ -3,6 +3,7 @@ from fastapi.middleware.cors import CORSMiddleware
|
|||||||
from fastapi.responses import JSONResponse
|
from fastapi.responses import JSONResponse
|
||||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||||
from sqlmodel import select, func, text
|
from sqlmodel import select, func, text
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
from .database import init_db, get_session, get_engine
|
from .database import init_db, get_session, get_engine
|
||||||
from .models import ErrorLog, ErrorLogCreate, LogStatus, TaskStatusUpdate, RepairTask, RepairTaskCreate, Project, ProjectUpdate
|
from .models import ErrorLog, ErrorLogCreate, LogStatus, TaskStatusUpdate, RepairTask, RepairTaskCreate, Project, ProjectUpdate
|
||||||
from .gitea_client import GiteaClient
|
from .gitea_client import GiteaClient
|
||||||
@ -163,9 +164,29 @@ async def report_log(log_data: ErrorLogCreate, session: AsyncSession = Depends(g
|
|||||||
)
|
)
|
||||||
|
|
||||||
session.add(new_log)
|
session.add(new_log)
|
||||||
await session.commit()
|
try:
|
||||||
await session.refresh(new_log)
|
await session.commit()
|
||||||
|
except IntegrityError:
|
||||||
|
await session.rollback()
|
||||||
|
# Race condition: another request inserted the same fingerprint concurrently.
|
||||||
|
# Re-query and update the existing record instead.
|
||||||
|
dup_stmt = select(ErrorLog).where(ErrorLog.fingerprint == fingerprint)
|
||||||
|
dup_results = await session.exec(dup_stmt)
|
||||||
|
dup_log = dup_results.first()
|
||||||
|
if dup_log:
|
||||||
|
dup_log.error_message = log_data.error.get("message", dup_log.error_message)
|
||||||
|
dup_log.stack_trace = log_data.error.get("stack_trace", dup_log.stack_trace)
|
||||||
|
dup_log.context = log_data.context or dup_log.context
|
||||||
|
dup_log.timestamp = log_data.timestamp or datetime.utcnow()
|
||||||
|
if log_data.commit_hash:
|
||||||
|
dup_log.commit_hash = log_data.commit_hash
|
||||||
|
session.add(dup_log)
|
||||||
|
await session.commit()
|
||||||
|
await session.refresh(dup_log)
|
||||||
|
return {"message": "Log deduplicated (content updated)", "id": dup_log.id, "status": dup_log.status}
|
||||||
|
raise
|
||||||
|
|
||||||
|
await session.refresh(new_log)
|
||||||
return {"message": "Log reported", "id": new_log.id}
|
return {"message": "Log reported", "id": new_log.id}
|
||||||
|
|
||||||
# ==================== Agent Tasks ====================
|
# ==================== Agent Tasks ====================
|
||||||
@ -307,9 +328,9 @@ async def get_repair_reports(
|
|||||||
if project_id:
|
if project_id:
|
||||||
query = query.where(RepairTask.project_id == project_id)
|
query = query.where(RepairTask.project_id == project_id)
|
||||||
if error_log_id:
|
if error_log_id:
|
||||||
# PostgreSQL JSONB contains: error_log_ids @> '[28]'
|
# MySQL JSON contains
|
||||||
query = query.where(
|
query = query.where(
|
||||||
text(f"error_log_ids @> '{json.dumps([error_log_id])}'::jsonb")
|
text(f"JSON_CONTAINS(error_log_ids, '{json.dumps([error_log_id])}')")
|
||||||
)
|
)
|
||||||
|
|
||||||
offset = (page - 1) * page_size
|
offset = (page - 1) * page_size
|
||||||
|
|||||||
@ -21,17 +21,17 @@ spec:
|
|||||||
ports:
|
ports:
|
||||||
- containerPort: 8002
|
- containerPort: 8002
|
||||||
env:
|
env:
|
||||||
# PostgreSQL Database
|
# MySQL Database (Volcano Engine RDS)
|
||||||
- name: DB_HOST
|
- name: DB_HOST
|
||||||
value: "pgm-7xv4811oj11j86htzo.pg.rds.aliyuncs.com"
|
value: "192.168.0.131"
|
||||||
- name: DB_PORT
|
- name: DB_PORT
|
||||||
value: "5432"
|
value: "3306"
|
||||||
- name: DB_NAME
|
- name: DB_NAME
|
||||||
value: "log_center"
|
value: "log_center"
|
||||||
- name: DB_USER
|
- name: DB_USER
|
||||||
value: "log_center"
|
value: "zyc"
|
||||||
- name: DB_PASSWORD
|
- name: DB_PASSWORD
|
||||||
value: "JogNQdtrd3WY8CBCAiYfYEGx"
|
value: "Zyc188208"
|
||||||
# Gitea Token(URL 从 PR URL 自动解析)
|
# Gitea Token(URL 从 PR URL 自动解析)
|
||||||
- name: GITEA_TOKEN
|
- name: GITEA_TOKEN
|
||||||
value: "443f7f2f556b4832f90e46df9af3e21ccb06b8a3"
|
value: "443f7f2f556b4832f90e46df9af3e21ccb06b8a3"
|
||||||
|
|||||||
@ -1,7 +1,7 @@
|
|||||||
fastapi
|
fastapi
|
||||||
uvicorn[standard]
|
uvicorn[standard]
|
||||||
sqlmodel
|
sqlmodel
|
||||||
psycopg2-binary
|
aiomysql
|
||||||
asyncpg
|
pymysql
|
||||||
python-dotenv
|
python-dotenv
|
||||||
httpx
|
httpx
|
||||||
|
|||||||
BIN
test_bug71.db
Normal file
BIN
test_bug71.db
Normal file
Binary file not shown.
@ -5,7 +5,7 @@ WORKDIR /app
|
|||||||
|
|
||||||
COPY package*.json ./
|
COPY package*.json ./
|
||||||
|
|
||||||
RUN npm install
|
RUN npm config set registry https://registry.npmmirror.com && npm install
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user