summaryrefslogtreecommitdiff
path: root/repo-tools/ai/fetch_github_data.py
diff options
context:
space:
mode:
Diffstat (limited to 'repo-tools/ai/fetch_github_data.py')
-rw-r--r--repo-tools/ai/fetch_github_data.py96
1 files changed, 96 insertions, 0 deletions
diff --git a/repo-tools/ai/fetch_github_data.py b/repo-tools/ai/fetch_github_data.py
new file mode 100644
index 00000000..8e3650cc
--- /dev/null
+++ b/repo-tools/ai/fetch_github_data.py
@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""
+fetch_github_data.py — Fetches open issues and PRs from a GitHub repo using
+the GitHub REST API and saves them as JSON for generate_dashboard.py.
+
+Usage:
+ python fetch_github_data.py [--repo OWNER/REPO] [--output-dir DIR]
+
+Requires:
+ - GITHUB_TOKEN environment variable (for authenticated API access)
+ - OR works without auth for public repos (rate-limited to 60 req/hr)
+"""
+
+import argparse
+import json
+import os
+import sys
+import urllib.request
+import urllib.error
+from pathlib import Path
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+
+
+def fetch_paginated(url: str, token: str | None = None) -> list[dict]:
+ """Fetch all pages from a GitHub REST API endpoint."""
+ results = []
+ page = 1
+ while True:
+ sep = "&" if "?" in url else "?"
+ page_url = f"{url}{sep}page={page}&per_page=100&state=open"
+ req = urllib.request.Request(page_url)
+ req.add_header("Accept", "application/vnd.github+json")
+ req.add_header("User-Agent", "repo-tools-dashboard")
+ if token:
+ req.add_header("Authorization", f"Bearer {token}")
+
+ try:
+ with urllib.request.urlopen(req) as resp:
+ data = json.loads(resp.read().decode("utf-8"))
+ except urllib.error.HTTPError as e:
+ print(f"āŒ HTTP {e.code} fetching {page_url}: {e.reason}")
+ sys.exit(1)
+
+ if not data:
+ break
+ results.extend(data)
+ if len(data) < 100:
+ break
+ page += 1
+ print(f" Fetched page {page - 1} ({len(results)} items so far)")
+
+ return results
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Fetch GitHub issues and PRs as JSON.")
+ parser.add_argument("--repo", default="microsoft/Windows-driver-samples",
+ help="GitHub repo in OWNER/REPO format")
+ parser.add_argument("--output-dir", default=str(SCRIPT_DIR / "_data"),
+ help="Output directory for JSON files")
+ args = parser.parse_args()
+
+ token = os.environ.get("GITHUB_TOKEN")
+ if not token:
+ print("āš ļø GITHUB_TOKEN not set. Using unauthenticated access (60 req/hr limit).")
+
+ os.makedirs(args.output_dir, exist_ok=True)
+ base = f"https://api.github.com/repos/{args.repo}"
+
+ # Fetch issues (the API returns PRs mixed in; filter them out)
+ print(f"šŸ“„ Fetching open issues from {args.repo}...")
+ all_items = fetch_paginated(f"{base}/issues", token)
+ issues = [i for i in all_items if "pull_request" not in i]
+ print(f" āœ… {len(issues)} open issues")
+
+ issues_path = os.path.join(args.output_dir, "issues.json")
+ with open(issues_path, "w", encoding="utf-8") as f:
+ json.dump({"issues": issues}, f, indent=2)
+ print(f" Saved to {issues_path}")
+
+ # Fetch PRs
+ print(f"šŸ“„ Fetching open PRs from {args.repo}...")
+ prs = fetch_paginated(f"{base}/pulls", token)
+ print(f" āœ… {len(prs)} open PRs")
+
+ prs_path = os.path.join(args.output_dir, "prs.json")
+ with open(prs_path, "w", encoding="utf-8") as f:
+ json.dump(prs, f, indent=2)
+ print(f" Saved to {prs_path}")
+
+ print("\nāœ… Data fetched. Run generate_dashboard.py to build the dashboard.")
+
+
+if __name__ == "__main__":
+ main()