[ci]: Introduce a regular inspection mechanism for abnormal CI reports (#10852)

5 files changed, 574 insertions, 0 deletions

diff --git a/tools/ci/scheduled-ci-trigger/generate_report.py b/tools/ci/scheduled-ci-trigger/generate_report.py
new file mode 100644
index 0000000000..aff419cd1b
--- /dev/null
+++ b/tools/ci/scheduled-ci-trigger/generate_report.py
@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+import json
+import os
+from datetime import datetime, timedelta
+from typing import List, Dict, Any
+
+def load_monitoring_results() -> List[Dict[str, Any]]:
+    """加载 monitoring_results.json"""
+    if not os.path.exists("monitoring_results.json"):
+        print("No monitoring results found")
+        return []
+    try:
+        with open("monitoring_results.json", "r", encoding="utf-8") as f:
+            return json.load(f)
+    except (json.JSONDecodeError, OSError) as e:
+        print(f"Error loading monitoring_results.json: {e}")
+        return []
+
+def get_beijing_time() -> datetime:
+    return datetime.utcnow() + timedelta(hours=8)
+
+def format_time(dt: datetime) -> str:
+    return dt.strftime("%Y-%m-%d %H:%M")
+
+def classify_error(step_name: str, job_name: str) -> str:
+    """错误类型分类"""
+    step_lower = step_name.lower()
+    if any(x in step_lower for x in ["test", "suite", "pytest", "unittest"]):
+        return "TEST_FAILURE"
+    if "lint" in step_lower or "flake8" in step_lower:
+        return "LINT_ERROR"
+    if "build" in step_lower or "compile" in step_lower:
+        return "BUILD_ERROR"
+    if "deploy" in step_lower or "upload" in step_lower or "publish" in step_lower:
+        return "DEPLOY_ERROR"
+    if "check" in step_lower or "validate" in step_lower or "verify" in step_lower:
+        return "VALIDATION_ERROR"
+    if "generate" in step_lower or "render" in step_lower:
+        return "GENERATION_ERROR"
+    return "UNKNOWN"
+
+def generate_report():
+    """生成符合最新样式的故障聚合报告"""
+    results = load_monitoring_results()
+    if not results:
+        return
+
+    failed_workflows = [r for r in results if r.get('conclusion') == 'failure']
+    if not failed_workflows:
+        print("No failed workflows to report")
+        return
+
+    now = get_beijing_time()
+    date_str = now.strftime("%Y%m%d")
+
+    # 时间范围
+    created_times = [
+        datetime.fromisoformat(r["created_at"].replace("Z", "+00:00")) + timedelta(hours=8)
+        for r in failed_workflows
+    ]
+    updated_times = [
+        datetime.fromisoformat(r["updated_at"].replace("Z", "+00:00")) + timedelta(hours=8)
+        for r in failed_workflows
+    ]
+    start_time = min(created_times)
+    end_time = max(updated_times)
+
+    total = len(results)
+    failed_count = len(failed_workflows)
+    success_rate = 0.0 if total == 0 else round((total - failed_count) / total * 100, 1)
+
+    # === 第一行：用于 JS 提取标题（必须）===
+    report = f"# {date_str}_ci_integration-failed-report\n\n"
+
+    # === 第二行：用户看到的主标题（H1）===
+    report += f"# 🚨 {date_str} GitHub Actions 故障聚合报告 | Incident Aggregate Report\n\n"
+
+    # === 执行概览 ===
+    report += f"## 执行概览 | Executive Summary\n"
+    report += f"- **监控时间范围 | Monitoring Period**: {format_time(start_time)}–{format_time(end_time)} (UTC+8)\n"
+    report += f"- **检测到失败运行 | Failed Runs Detected**: {failed_count}个\n"
+    report += f"- **成功率 | Success Rate**: {success_rate}% \n\n"
+
+    # === 故障详情 ===
+    report += f"## 🔍 故障详情 | Failure Details\n\n"
+
+    for wf in failed_workflows:
+        run_id = wf.get("run_id", "N/A")
+        name = wf["name"]
+        html_url = wf.get("html_url", "#")
+        details = wf.get("failure_details", [])
+
+        report += f"**📌 Run-{run_id}** | [{name}]({html_url})\n"
+
+        if not details:
+            report += "└─ 无失败作业详情 | No details of failed jobs\n\n"
+            continue
+
+        failed_jobs = [j for j in details if j.get("steps")]
+        for i, job in enumerate(failed_jobs):
+            job_name = job["name"]
+            steps = job["steps"]
+            job_prefix = "└─" if i == len(failed_jobs) - 1 else "├─"
+            report += f"{job_prefix} **失败作业 | Failed Job**: {job_name}\n"
+
+            for j, step in enumerate(steps):
+                step_name = step["name"]
+                step_num = step["number"]
+                error_type = classify_error(step_name, job_name)
+                step_prefix = "   └─" if j == len(steps) - 1 else "   ├─"
+                report += f"{step_prefix} **失败步骤 | Failed Step**: {step_name} (Step {step_num})\n"
+                indent = "      " if j == len(steps) - 1 else "   │   "
+                report += f"{indent}**错误类型 | Error Type**: `{error_type}`\n"
+        report += "\n"
+
+    # === Team Collaboration & Support ===
+    report += f"## 👥 团队协作与支持 | Team Collaboration & Support\n\n"
+    report += f"请求维护支持：本报告需要RT-Thread官方团队的专业经验进行审核与指导。 \n"
+    report += f"Call for Maintenance Support: This report requires the expertise of the RT-Thread official team for review and guidance.\n\n"
+    report += f"提审负责人：@Rbb666 @kurisaW\n"
+    report += f"Requested Reviewers from RT-Thread: @Rbb666 @kurisaW\n\n"
+    report += f"烦请尽快关注此事，万分感谢。  \n"
+    report += f"Your prompt attention to this matter is greatly appreciated.\n"
+
+    # 保存
+    try:
+        with open("failure_details.md", "w", encoding="utf-8") as f:
+            f.write(report.rstrip() + "\n")
+        print("Report generated: failure_details.md")
+        print(f"Report size: {os.path.getsize('failure_details.md')} bytes")
+    except Exception as e:
+        print(f"Error writing report: {e}")
+
+if __name__ == "__main__":
+    generate_report()
\ No newline at end of file
diff --git a/tools/ci/scheduled-ci-trigger/monitor_workflows.py b/tools/ci/scheduled-ci-trigger/monitor_workflows.py
new file mode 100644
index 0000000000..5a72a17efa
--- /dev/null
+++ b/tools/ci/scheduled-ci-trigger/monitor_workflows.py
@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+import os
+import json
+import requests
+import time
+import sys
+from datetime import datetime, timezone
+
+def monitor_workflows(github_token, repo, workflow_names, start_time):
+    """监控工作流运行"""
+    headers = {
+        "Authorization": f"token {github_token}",
+        "Accept": "application/vnd.github.v3+json"
+    }
+    
+    monitoring_results = []
+    
+    for workflow_name in workflow_names:
+        print(f"\n=== Monitoring {workflow_name} ===")
+        
+        try:
+            workflow_id = get_workflow_id(github_token, repo, workflow_name)
+            if not workflow_id:
+                monitoring_results.append({
+                    "name": workflow_name,
+                    "status": "error",
+                    "conclusion": "error", 
+                    "error": "Workflow not found"
+                })
+                continue
+            
+            # 查找开始时间后的运行
+            runs = get_recent_runs(github_token, repo, workflow_id, start_time)
+            
+            if not runs:
+                print(f"No runs found for {workflow_name} after {start_time}")
+                # 尝试查找任何正在运行的工作流
+                all_runs = get_all_runs(github_token, repo, workflow_id, 10)
+                if all_runs:
+                    latest_run = all_runs[0]
+                    print(f"Using latest run instead: {latest_run['id']} created at {latest_run['created_at']}")
+                    result = monitor_single_run(github_token, repo, latest_run["id"], workflow_name)
+                    monitoring_results.append(result)
+                else:
+                    monitoring_results.append({
+                        "name": workflow_name,
+                        "status": "not_found",
+                        "conclusion": "not_found",
+                        "error": f"No runs found after {start_time}"
+                    })
+            else:
+                # 监控找到的运行
+                run_to_monitor = runs[0]  # 取最新的一个
+                print(f"Monitoring run: {run_to_monitor['id']}")
+                result = monitor_single_run(github_token, repo, run_to_monitor["id"], workflow_name)
+                monitoring_results.append(result)
+                
+        except Exception as e:
+            print(f"Error monitoring {workflow_name}: {str(e)}")
+            monitoring_results.append({
+                "name": workflow_name,
+                "status": "error",
+                "conclusion": "error",
+                "error": str(e)
+            })
+    
+    return monitoring_results
+
+def get_all_runs(github_token, repo, workflow_id, per_page=10):
+    """获取所有运行"""
+    headers = {
+        "Authorization": f"token {github_token}",
+        "Accept": "application/vnd.github.v3+json"
+    }
+    
+    url = f"https://api.github.com/repos/{repo}/actions/workflows/{workflow_id}/runs"
+    params = {"per_page": per_page}
+    
+    response = requests.get(url, headers=headers, params=params)
+    if response.status_code == 200:
+        return response.json()["workflow_runs"]
+    return []
+
+def get_recent_runs(github_token, repo, workflow_id, start_time):
+    """获取开始时间后的运行"""
+    all_runs = get_all_runs(github_token, repo, workflow_id, 10)
+    start_time_dt = datetime.fromisoformat(start_time.replace('Z', '+00:00'))
+    
+    recent_runs = []
+    for run in all_runs:
+        run_time = datetime.fromisoformat(run["created_at"].replace('Z', '+00:00'))
+        if run_time >= start_time_dt:
+            recent_runs.append(run)
+    
+    return recent_runs
+
+def monitor_single_run(github_token, repo, run_id, workflow_name):
+    """监控单个运行"""
+    headers = {
+        "Authorization": f"token {github_token}",
+        "Accept": "application/vnd.github.v3+json"
+    }
+    
+    max_wait_time = 1800  # 30分钟
+    check_interval = 30
+    start_time = time.time()
+    
+    print(f"Monitoring {workflow_name} (run {run_id})")
+    
+    while time.time() - start_time < max_wait_time:
+        url = f"https://api.github.com/repos/{repo}/actions/runs/{run_id}"
+        response = requests.get(url, headers=headers)
+        
+        if response.status_code != 200:
+            print(f"Error getting run status: {response.status_code}")
+            time.sleep(check_interval)
+            continue
+        
+        run_data = response.json()
+        status = run_data["status"]
+        conclusion = run_data.get("conclusion")
+        
+        print(f"  {workflow_name}: status={status}, conclusion={conclusion}")
+        
+        if status == "completed":
+            result = {
+                "name": workflow_name,
+                "run_id": run_id,
+                "status": status,
+                "conclusion": conclusion,
+                "html_url": run_data["html_url"],
+                "created_at": run_data["created_at"],
+                "updated_at": run_data["updated_at"]
+            }
+            
+            if conclusion == "failure":
+                result["failure_details"] = get_failure_logs(github_token, repo, run_id)
+            
+            return result
+        
+        time.sleep(check_interval)
+    
+    # 超时
+    return {
+        "name": workflow_name,
+        "run_id": run_id,
+        "status": "timed_out",
+        "conclusion": "timed_out",
+        "html_url": f"https://github.com/{repo}/actions/runs/{run_id}",
+        "error": "Monitoring timed out after 30 minutes"
+    }
+
+def get_failure_logs(github_token, repo, run_id):
+    """获取失败日志"""
+    headers = {
+        "Authorization": f"token {github_token}",
+        "Accept": "application/vnd.github.v3+json"
+    }
+    
+    try:
+        jobs_url = f"https://api.github.com/repos/{repo}/actions/runs/{run_id}/jobs"
+        jobs_response = requests.get(jobs_url, headers=headers)
+        
+        failure_details = []
+        
+        if jobs_response.status_code == 200:
+            jobs_data = jobs_response.json()["jobs"]
+            for job in jobs_data:
+                if job["conclusion"] == "failure":
+                    job_info = {
+                        "name": job["name"],
+                        "steps": []
+                    }
+                    
+                    for step in job["steps"]:
+                        if step["conclusion"] == "failure":
+                            job_info["steps"].append({
+                                "name": step["name"],
+                                "number": step["number"]
+                            })
+                    
+                    failure_details.append(job_info)
+        
+        return failure_details
+    except Exception as e:
+        print(f"Error getting failure logs: {e}")
+        return []
+
+def get_workflow_id(github_token, repo, workflow_name):
+    """获取工作流ID"""
+    headers = {
+        "Authorization": f"token {github_token}",
+        "Accept": "application/vnd.github.v3+json"
+    }
+    
+    url = f"https://api.github.com/repos/{repo}/actions/workflows"
+    response = requests.get(url, headers=headers)
+    
+    if response.status_code == 200:
+        workflows = response.json()["workflows"]
+        for workflow in workflows:
+            if workflow["name"] == workflow_name:
+                return workflow["id"]
+    return None
+
+def main():
+    github_token = os.getenv("GITHUB_TOKEN")
+    repo = os.getenv("GITHUB_REPOSITORY")
+    workflows_json = os.getenv("TARGET_WORKFLOWS")
+    start_time = sys.argv[1] if len(sys.argv) > 1 else datetime.now(timezone.utc).isoformat()
+    
+    if not all([github_token, repo, workflows_json]):
+        raise ValueError("Missing required environment variables")
+    
+    workflows = json.loads(workflows_json)
+    results = monitor_workflows(github_token, repo, workflows, start_time)
+    
+    with open("monitoring_results.json", "w") as f:
+        json.dump(results, f, indent=2)
+    
+    print(f"\n=== Monitoring Summary ===")
+    for result in results:
+        status_icon = "✅" if result.get("conclusion") == "success" else "❌" if result.get("conclusion") == "failure" else "⚠️"
+        print(f"{status_icon} {result['name']}: {result.get('conclusion', 'unknown')}")
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/tools/ci/scheduled-ci-trigger/requirements.txt b/tools/ci/scheduled-ci-trigger/requirements.txt
new file mode 100644
index 0000000000..9e4b84c454
--- /dev/null
+++ b/tools/ci/scheduled-ci-trigger/requirements.txt
@@ -0,0 +1 @@
+requests>=2.25.1
\ No newline at end of file
diff --git a/tools/ci/scheduled-ci-trigger/trigger_workflows_direct.py b/tools/ci/scheduled-ci-trigger/trigger_workflows_direct.py
new file mode 100644
index 0000000000..9d91e74359
--- /dev/null
+++ b/tools/ci/scheduled-ci-trigger/trigger_workflows_direct.py
@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+import os
+import json
+import requests
+import time
+from datetime import datetime, timezone
+
+def trigger_workflow_directly(workflow_name, github_token, repo):
+    """直接触发工作流"""
+    headers = {
+        "Authorization": f"token {github_token}",
+        "Accept": "application/vnd.github.v3+json"
+    }
+    
+    # 首先获取工作流ID
+    workflow_id = get_workflow_id(github_token, repo, workflow_name)
+    if not workflow_id:
+        print(f"✗ Workflow '{workflow_name}' not found")
+        return False
+    
+    # 使用 workflow_dispatch API 直接触发
+    dispatch_url = f"https://api.github.com/repos/{repo}/actions/workflows/{workflow_id}/dispatches"  # 🔧 修复：添加这行
+    
+    # 根据工作流实际定义的输入参数进行调整
+    dispatch_data = {
+        "ref": "master",
+        "inputs": {
+            "trigger_type": "scheduled"  # 使用工作流实际定义的输入参数
+        }
+    }
+    
+    try:
+        print(f"Triggering workflow: {workflow_name} (ID: {workflow_id})")
+        response = requests.post(dispatch_url, headers=headers, json=dispatch_data)  # 🔧 修复：现在 dispatch_url 已定义
+        
+        if response.status_code == 204:
+            print(f"✓ Successfully triggered workflow: {workflow_name}")
+            return True
+        else:
+            print(f"✗ Failed to trigger {workflow_name}: {response.status_code}")
+            print(f"Response: {response.text}")
+            return False
+            
+    except Exception as e:
+        print(f"✗ Error triggering {workflow_name}: {str(e)}")
+        return False
+
+def get_workflow_id(github_token, repo, workflow_name):
+    """获取工作流ID"""
+    headers = {
+        "Authorization": f"token {github_token}",
+        "Accept": "application/vnd.github.v3+json"
+    }
+    
+    url = f"https://api.github.com/repos/{repo}/actions/workflows"
+    response = requests.get(url, headers=headers)
+    
+    if response.status_code == 200:
+        workflows = response.json()["workflows"]
+        for workflow in workflows:
+            if workflow["name"] == workflow_name:
+                return workflow["id"]
+        print(f"Available workflows: {[w['name'] for w in workflows]}")
+    else:
+        print(f"Failed to get workflows: {response.status_code}")
+    
+    return None
+
+def main():
+    github_token = os.getenv("GITHUB_TOKEN")
+    repo = os.getenv("GITHUB_REPOSITORY")
+    workflows_json = os.getenv("TARGET_WORKFLOWS")
+    
+    if not all([github_token, repo, workflows_json]):
+        raise ValueError("Missing required environment variables")
+    
+    try:
+        workflows = json.loads(workflows_json)
+    except json.JSONDecodeError:
+        raise ValueError("Invalid TARGET_WORKFLOWS JSON format")
+    
+    print(f"Directly triggering {len(workflows)} workflows...")
+    
+    success_count = 0
+    for i, workflow in enumerate(workflows):
+        success = trigger_workflow_directly(workflow, github_token, repo)
+        if success:
+            success_count += 1
+        
+        # 在触发之间等待
+        if i < len(workflows) - 1:
+            print("Waiting 10 seconds before next trigger...")
+            time.sleep(10)
+    
+    print(f"Triggering completed: {success_count}/{len(workflows)} successful")
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/tools/ci/scheduled-ci-trigger/wait_for_workflows.py b/tools/ci/scheduled-ci-trigger/wait_for_workflows.py
new file mode 100644
index 0000000000..f840a7166b
--- /dev/null
+++ b/tools/ci/scheduled-ci-trigger/wait_for_workflows.py
@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+import os
+import json
+import requests
+import time
+import sys
+from datetime import datetime, timezone
+
+def wait_for_workflows_to_appear(github_token, repo, workflow_names, start_time, max_wait=300):
+    """等待工作流出现在API中"""
+    headers = {
+        "Authorization": f"token {github_token}",
+        "Accept": "application/vnd.github.v3+json"
+    }
+    
+    print(f"Waiting for {len(workflow_names)} workflows to appear...")
+    print(f"Start time: {start_time}")
+    print(f"Max wait time: {max_wait} seconds")
+    
+    found_workflows = set()
+    start_timestamp = time.time()
+    
+    while time.time() - start_timestamp < max_wait:
+        all_found = True
+        
+        for workflow_name in workflow_names:
+            if workflow_name in found_workflows:
+                continue
+                
+            workflow_id = get_workflow_id(github_token, repo, workflow_name)
+            if not workflow_id:
+                print(f"Workflow {workflow_name} not found, skipping")
+                found_workflows.add(workflow_name)
+                continue
+            
+            # 检查是否有新的运行
+            runs = get_recent_runs(github_token, repo, workflow_id, start_time)
+            if runs:
+                print(f"✓ Found new run for {workflow_name}: {runs[0]['id']}")
+                found_workflows.add(workflow_name)
+            else:
+                print(f"⏳ Waiting for {workflow_name}...")
+                all_found = False
+        
+        if all_found:
+            print("✓ All workflows have started!")
+            return True
+        
+        time.sleep(10)  # 每10秒检查一次
+    
+    print("⚠️ Timeout waiting for workflows to appear")
+    print(f"Found {len(found_workflows)} out of {len(workflow_names)} workflows")
+    return False
+
+def get_workflow_id(github_token, repo, workflow_name):
+    """获取工作流ID"""
+    headers = {
+        "Authorization": f"token {github_token}",
+        "Accept": "application/vnd.github.v3+json"
+    }
+    
+    url = f"https://api.github.com/repos/{repo}/actions/workflows"
+    response = requests.get(url, headers=headers)
+    
+    if response.status_code == 200:
+        workflows = response.json()["workflows"]
+        for workflow in workflows:
+            if workflow["name"] == workflow_name:
+                return workflow["id"]
+    return None
+
+def get_recent_runs(github_token, repo, workflow_id, start_time):
+    """获取开始时间后的运行"""
+    headers = {
+        "Authorization": f"token {github_token}",
+        "Accept": "application/vnd.github.v3+json"
+    }
+    
+    url = f"https://api.github.com/repos/{repo}/actions/workflows/{workflow_id}/runs"
+    params = {"per_page": 5}
+    
+    response = requests.get(url, headers=headers, params=params)
+    if response.status_code != 200:
+        return []
+    
+    runs = response.json()["workflow_runs"]
+    start_time_dt = datetime.fromisoformat(start_time.replace('Z', '+00:00'))
+    
+    recent_runs = []
+    for run in runs:
+        run_time = datetime.fromisoformat(run["created_at"].replace('Z', '+00:00'))
+        if run_time >= start_time_dt:
+            recent_runs.append(run)
+    
+    return recent_runs
+
+def main():
+    github_token = os.getenv("GITHUB_TOKEN")
+    repo = os.getenv("GITHUB_REPOSITORY")
+    workflows_json = os.getenv("TARGET_WORKFLOWS")
+    start_time = sys.argv[1] if len(sys.argv) > 1 else datetime.now(timezone.utc).isoformat()
+    
+    if not all([github_token, repo, workflows_json]):
+        raise ValueError("Missing required environment variables")
+    
+    workflows = json.loads(workflows_json)
+    success = wait_for_workflows_to_appear(github_token, repo, workflows, start_time)
+    
+    if not success:
+        print("Proceeding anyway, some workflows may not be detected...")
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file