[ci] Fix scheduled failure reporting and monthly BSP builds (#11921)

* [ci] Fix scheduled failure reporting and monthly BSP builds

* [ci] Preserve partial monitoring results and abnormal job details

---------

Co-authored-by: Rbb666 <[email protected]>

4 files changed, 109 insertions, 74 deletions

diff --git a/tools/ci/scheduled-ci-trigger/generate_report.py b/tools/ci/scheduled-ci-trigger/generate_report.py
index aff419cd1b..151d66ac07 100644
--- a/tools/ci/scheduled-ci-trigger/generate_report.py
+++ b/tools/ci/scheduled-ci-trigger/generate_report.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 import json
 import os
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from typing import List, Dict, Any
 
 def load_monitoring_results() -> List[Dict[str, Any]]:
@@ -17,7 +17,7 @@ def load_monitoring_results() -> List[Dict[str, Any]]:
         return []
 
 def get_beijing_time() -> datetime:
-    return datetime.utcnow() + timedelta(hours=8)
+    return datetime.now(timezone(timedelta(hours=8)))
 
 def format_time(dt: datetime) -> str:
     return dt.strftime("%Y-%m-%d %H:%M")
@@ -45,7 +45,7 @@ def generate_report():
     if not results:
         return
 
-    failed_workflows = [r for r in results if r.get('conclusion') == 'failure']
+    failed_workflows = [r for r in results if r.get('conclusion') != 'success']
     if not failed_workflows:
         print("No failed workflows to report")
         return
@@ -55,15 +55,15 @@ def generate_report():
 
     # 时间范围
     created_times = [
-        datetime.fromisoformat(r["created_at"].replace("Z", "+00:00")) + timedelta(hours=8)
-        for r in failed_workflows
+        datetime.fromisoformat(r["created_at"].replace("Z", "+00:00")).astimezone(now.tzinfo)
+        for r in failed_workflows if r.get("created_at")
     ]
     updated_times = [
-        datetime.fromisoformat(r["updated_at"].replace("Z", "+00:00")) + timedelta(hours=8)
-        for r in failed_workflows
+        datetime.fromisoformat(r["updated_at"].replace("Z", "+00:00")).astimezone(now.tzinfo)
+        for r in failed_workflows if r.get("updated_at")
     ]
-    start_time = min(created_times)
-    end_time = max(updated_times)
+    start_time = min(created_times, default=now)
+    end_time = max(updated_times, default=now)
 
     total = len(results)
     failed_count = len(failed_workflows)
@@ -91,26 +91,34 @@ def generate_report():
         details = wf.get("failure_details", [])
 
         report += f"**📌 Run-{run_id}** | [{name}]({html_url})\n"
+        report += f"**运行结果 | Conclusion**: `{wf.get('conclusion', 'unknown')}`\n"
+        if wf.get("error"):
+            report += f"**错误信息 | Error**: {wf['error']}\n"
 
         if not details:
-            report += "└─ 无失败作业详情 | No details of failed jobs\n\n"
+            report += "└─ 无异常作业详情 | No details of affected jobs\n\n"
             continue
 
-        failed_jobs = [j for j in details if j.get("steps")]
-        for i, job in enumerate(failed_jobs):
+        for i, job in enumerate(details):
             job_name = job["name"]
-            steps = job["steps"]
-            job_prefix = "└─" if i == len(failed_jobs) - 1 else "├─"
-            report += f"{job_prefix} **失败作业 | Failed Job**: {job_name}\n"
+            steps = job.get("steps", [])
+            job_status = job.get("conclusion", "failure") or job.get("status") or "unknown"
+            job_prefix = "└─" if i == len(details) - 1 else "├─"
+            report += f"{job_prefix} **异常作业 | Affected Job**: {job_name} (`{job_status}`)\n"
+            if not steps:
+                report += "   └─ 无异常步骤详情 | No details of affected steps\n"
 
             for j, step in enumerate(steps):
                 step_name = step["name"]
                 step_num = step["number"]
-                error_type = classify_error(step_name, job_name)
+                step_conclusion = step.get("conclusion", "failure")
+                step_status = step_conclusion or step.get("status") or "unknown"
                 step_prefix = "   └─" if j == len(steps) - 1 else "   ├─"
-                report += f"{step_prefix} **失败步骤 | Failed Step**: {step_name} (Step {step_num})\n"
-                indent = "      " if j == len(steps) - 1 else "   │   "
-                report += f"{indent}**错误类型 | Error Type**: `{error_type}`\n"
+                report += f"{step_prefix} **异常步骤 | Affected Step**: {step_name} (Step {step_num}, `{step_status}`)\n"
+                if step_conclusion == "failure":
+                    error_type = classify_error(step_name, job_name)
+                    indent = "      " if j == len(steps) - 1 else "   │   "
+                    report += f"{indent}**错误类型 | Error Type**: `{error_type}`\n"
         report += "\n"
 
     # === Team Collaboration & Support ===
@@ -132,4 +140,4 @@ def generate_report():
         print(f"Error writing report: {e}")
 
 if __name__ == "__main__":
-    generate_report()
\ No newline at end of file
+    generate_report()
diff --git a/tools/ci/scheduled-ci-trigger/monitor_workflows.py b/tools/ci/scheduled-ci-trigger/monitor_workflows.py
index 5a72a17efa..b28b7ff5a3 100644
--- a/tools/ci/scheduled-ci-trigger/monitor_workflows.py
+++ b/tools/ci/scheduled-ci-trigger/monitor_workflows.py
@@ -6,6 +6,12 @@ import time
 import sys
 from datetime import datetime, timezone
 
+def save_monitoring_results(results):
+    # Keep the last complete snapshot if the process stops during a write.
+    with open("monitoring_results.json.tmp", "w", encoding="utf-8") as f:
+        json.dump(results, f, indent=2)
+    os.replace("monitoring_results.json.tmp", "monitoring_results.json")
+
 def monitor_workflows(github_token, repo, workflow_names, start_time):
     """监控工作流运行"""
     headers = {
@@ -13,20 +19,27 @@ def monitor_workflows(github_token, repo, workflow_names, start_time):
         "Accept": "application/vnd.github.v3+json"
     }
     
-    monitoring_results = []
+    # Include unfinished workflows so a partial snapshot cannot report success.
+    monitoring_results = [{
+        "name": name,
+        "status": "not_monitored",
+        "conclusion": "error",
+        "error": "Monitoring did not complete for this workflow"
+    } for name in workflow_names]
+    save_monitoring_results(monitoring_results)
     
-    for workflow_name in workflow_names:
+    for index, workflow_name in enumerate(workflow_names):
         print(f"\n=== Monitoring {workflow_name} ===")
         
         try:
             workflow_id = get_workflow_id(github_token, repo, workflow_name)
             if not workflow_id:
-                monitoring_results.append({
+                monitoring_results[index] = {
                     "name": workflow_name,
                     "status": "error",
                     "conclusion": "error", 
                     "error": "Workflow not found"
-                })
+                }
                 continue
             
             # 查找开始时间后的运行
@@ -34,35 +47,29 @@ def monitor_workflows(github_token, repo, workflow_names, start_time):
             
             if not runs:
                 print(f"No runs found for {workflow_name} after {start_time}")
-                # 尝试查找任何正在运行的工作流
-                all_runs = get_all_runs(github_token, repo, workflow_id, 10)
-                if all_runs:
-                    latest_run = all_runs[0]
-                    print(f"Using latest run instead: {latest_run['id']} created at {latest_run['created_at']}")
-                    result = monitor_single_run(github_token, repo, latest_run["id"], workflow_name)
-                    monitoring_results.append(result)
-                else:
-                    monitoring_results.append({
-                        "name": workflow_name,
-                        "status": "not_found",
-                        "conclusion": "not_found",
-                        "error": f"No runs found after {start_time}"
-                    })
+                monitoring_results[index] = {
+                    "name": workflow_name,
+                    "status": "not_found",
+                    "conclusion": "not_found",
+                    "error": f"No dispatched runs found after {start_time}"
+                }
             else:
                 # 监控找到的运行
-                run_to_monitor = runs[0]  # 取最新的一个
+                run_to_monitor = min(runs, key=lambda run: run["created_at"])
                 print(f"Monitoring run: {run_to_monitor['id']}")
                 result = monitor_single_run(github_token, repo, run_to_monitor["id"], workflow_name)
-                monitoring_results.append(result)
+                monitoring_results[index] = result
                 
         except Exception as e:
             print(f"Error monitoring {workflow_name}: {str(e)}")
-            monitoring_results.append({
+            monitoring_results[index] = {
                 "name": workflow_name,
                 "status": "error",
                 "conclusion": "error",
                 "error": str(e)
-            })
+            }
+        finally:
+            save_monitoring_results(monitoring_results)
     
     return monitoring_results
 
@@ -74,9 +81,13 @@ def get_all_runs(github_token, repo, workflow_id, per_page=10):
     }
     
     url = f"https://api.github.com/repos/{repo}/actions/workflows/{workflow_id}/runs"
-    params = {"per_page": per_page}
+    params = {
+        "per_page": per_page,
+        "event": "workflow_dispatch",
+        "branch": "master",
+    }
     
-    response = requests.get(url, headers=headers, params=params)
+    response = requests.get(url, headers=headers, params=params, timeout=30)
     if response.status_code == 200:
         return response.json()["workflow_runs"]
     return []
@@ -101,15 +112,15 @@ def monitor_single_run(github_token, repo, run_id, workflow_name):
         "Accept": "application/vnd.github.v3+json"
     }
     
-    max_wait_time = 1800  # 30分钟
+    max_wait_time = 7200  # Allow two hours for queued BSP builds.
     check_interval = 30
-    start_time = time.time()
+    start_time = time.monotonic()
     
     print(f"Monitoring {workflow_name} (run {run_id})")
     
-    while time.time() - start_time < max_wait_time:
+    while time.monotonic() - start_time < max_wait_time:
         url = f"https://api.github.com/repos/{repo}/actions/runs/{run_id}"
-        response = requests.get(url, headers=headers)
+        response = requests.get(url, headers=headers, timeout=30)
         
         if response.status_code != 200:
             print(f"Error getting run status: {response.status_code}")
@@ -133,7 +144,7 @@ def monitor_single_run(github_token, repo, run_id, workflow_name):
                 "updated_at": run_data["updated_at"]
             }
             
-            if conclusion == "failure":
+            if conclusion != "success":
                 result["failure_details"] = get_failure_logs(github_token, repo, run_id)
             
             return result
@@ -147,7 +158,8 @@ def monitor_single_run(github_token, repo, run_id, workflow_name):
         "status": "timed_out",
         "conclusion": "timed_out",
         "html_url": f"https://github.com/{repo}/actions/runs/{run_id}",
-        "error": "Monitoring timed out after 30 minutes"
+        "error": "Monitoring timed out after 120 minutes",
+        "failure_details": get_failure_logs(github_token, repo, run_id)
     }
 
 def get_failure_logs(github_token, repo, run_id):
@@ -159,27 +171,36 @@ def get_failure_logs(github_token, repo, run_id):
     
     try:
         jobs_url = f"https://api.github.com/repos/{repo}/actions/runs/{run_id}/jobs"
-        jobs_response = requests.get(jobs_url, headers=headers)
-        
         failure_details = []
-        
-        if jobs_response.status_code == 200:
+        page = 1
+        while True:
+            jobs_response = requests.get(
+                jobs_url, headers=headers,
+                params={"per_page": 100, "page": page}, timeout=30)
+            jobs_response.raise_for_status()
             jobs_data = jobs_response.json()["jobs"]
             for job in jobs_data:
-                if job["conclusion"] == "failure":
+                if job.get("conclusion") not in ("success", "skipped", "neutral"):
                     job_info = {
                         "name": job["name"],
+                        "status": job.get("status"),
+                        "conclusion": job.get("conclusion"),
                         "steps": []
                     }
                     
-                    for step in job["steps"]:
-                        if step["conclusion"] == "failure":
+                    for step in job.get("steps", []):
+                        if step.get("conclusion") not in ("success", "skipped", "neutral"):
                             job_info["steps"].append({
                                 "name": step["name"],
-                                "number": step["number"]
+                                "number": step["number"],
+                                "status": step.get("status"),
+                                "conclusion": step.get("conclusion")
                             })
                     
                     failure_details.append(job_info)
+            if len(jobs_data) < 100:
+                break
+            page += 1
         
         return failure_details
     except Exception as e:
@@ -194,7 +215,7 @@ def get_workflow_id(github_token, repo, workflow_name):
     }
     
     url = f"https://api.github.com/repos/{repo}/actions/workflows"
-    response = requests.get(url, headers=headers)
+    response = requests.get(url, headers=headers, params={"per_page": 100}, timeout=30)
     
     if response.status_code == 200:
         workflows = response.json()["workflows"]
@@ -215,13 +236,12 @@ def main():
     workflows = json.loads(workflows_json)
     results = monitor_workflows(github_token, repo, workflows, start_time)
     
-    with open("monitoring_results.json", "w") as f:
-        json.dump(results, f, indent=2)
-    
     print(f"\n=== Monitoring Summary ===")
     for result in results:
         status_icon = "✅" if result.get("conclusion") == "success" else "❌" if result.get("conclusion") == "failure" else "⚠️"
         print(f"{status_icon} {result['name']}: {result.get('conclusion', 'unknown')}")
 
+    return 0 if results and all(r.get("conclusion") == "success" for r in results) else 1
+
 if __name__ == "__main__":
-    main()
\ No newline at end of file
+    sys.exit(main())
diff --git a/tools/ci/scheduled-ci-trigger/trigger_workflows_direct.py b/tools/ci/scheduled-ci-trigger/trigger_workflows_direct.py
index 9d91e74359..90b727f7de 100644
--- a/tools/ci/scheduled-ci-trigger/trigger_workflows_direct.py
+++ b/tools/ci/scheduled-ci-trigger/trigger_workflows_direct.py
@@ -3,6 +3,7 @@ import os
 import json
 import requests
 import time
+import sys
 from datetime import datetime, timezone
 
 def trigger_workflow_directly(workflow_name, github_token, repo):
@@ -31,7 +32,7 @@ def trigger_workflow_directly(workflow_name, github_token, repo):
     
     try:
         print(f"Triggering workflow: {workflow_name} (ID: {workflow_id})")
-        response = requests.post(dispatch_url, headers=headers, json=dispatch_data)  # 🔧 修复：现在 dispatch_url 已定义
+        response = requests.post(dispatch_url, headers=headers, json=dispatch_data, timeout=30)
         
         if response.status_code == 204:
             print(f"✓ Successfully triggered workflow: {workflow_name}")
@@ -53,7 +54,7 @@ def get_workflow_id(github_token, repo, workflow_name):
     }
     
     url = f"https://api.github.com/repos/{repo}/actions/workflows"
-    response = requests.get(url, headers=headers)
+    response = requests.get(url, headers=headers, params={"per_page": 100}, timeout=30)
     
     if response.status_code == 200:
         workflows = response.json()["workflows"]
@@ -83,7 +84,11 @@ def main():
     
     success_count = 0
     for i, workflow in enumerate(workflows):
-        success = trigger_workflow_directly(workflow, github_token, repo)
+        try:
+            success = trigger_workflow_directly(workflow, github_token, repo)
+        except requests.RequestException as error:
+            print(f"Failed to trigger {workflow}: {error}")
+            success = False
         if success:
             success_count += 1
         
@@ -93,6 +98,7 @@ def main():
             time.sleep(10)
     
     print(f"Triggering completed: {success_count}/{len(workflows)} successful")
+    return 0 if workflows and success_count == len(workflows) else 1
 
 if __name__ == "__main__":
-    main()
\ No newline at end of file
+    sys.exit(main())
diff --git a/tools/ci/scheduled-ci-trigger/wait_for_workflows.py b/tools/ci/scheduled-ci-trigger/wait_for_workflows.py
index f840a7166b..5acda65731 100644
--- a/tools/ci/scheduled-ci-trigger/wait_for_workflows.py
+++ b/tools/ci/scheduled-ci-trigger/wait_for_workflows.py
@@ -29,8 +29,8 @@ def wait_for_workflows_to_appear(github_token, repo, workflow_names, start_time,
                 
             workflow_id = get_workflow_id(github_token, repo, workflow_name)
             if not workflow_id:
-                print(f"Workflow {workflow_name} not found, skipping")
-                found_workflows.add(workflow_name)
+                print(f"Workflow {workflow_name} not found")
+                all_found = False
                 continue
             
             # 检查是否有新的运行
@@ -60,7 +60,7 @@ def get_workflow_id(github_token, repo, workflow_name):
     }
     
     url = f"https://api.github.com/repos/{repo}/actions/workflows"
-    response = requests.get(url, headers=headers)
+    response = requests.get(url, headers=headers, params={"per_page": 100}, timeout=30)
     
     if response.status_code == 200:
         workflows = response.json()["workflows"]
@@ -77,9 +77,9 @@ def get_recent_runs(github_token, repo, workflow_id, start_time):
     }
     
     url = f"https://api.github.com/repos/{repo}/actions/workflows/{workflow_id}/runs"
-    params = {"per_page": 5}
+    params = {"per_page": 10, "event": "workflow_dispatch", "branch": "master"}
     
-    response = requests.get(url, headers=headers, params=params)
+    response = requests.get(url, headers=headers, params=params, timeout=30)
     if response.status_code != 200:
         return []
     
@@ -107,7 +107,8 @@ def main():
     success = wait_for_workflows_to_appear(github_token, repo, workflows, start_time)
     
     if not success:
-        print("Proceeding anyway, some workflows may not be detected...")
+        print("Some workflows did not start within the waiting period")
+    return 0 if success else 1
 
 if __name__ == "__main__":
-    main()
\ No newline at end of file
+    sys.exit(main())