Repository navigation
Expand file tree
/
Copy path__main__.py
More file actions
194 lines (158 loc) · 5.67 KB
/
Copy path__main__.py
File metadata and controls
194 lines (158 loc) · 5.67 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
# Copyright (c) Microsoft. All rights reserved.
"""
Sample Validation Script
Validates all Python samples in the samples directory using a workflow that:
1. Discovers all sample files
2. Builds a nested concurrent workflow with one GitHub agent per sample
3. Runs the nested workflow
4. Generates a validation report
Usage:
uv run python -m sample_validation
uv run python -m sample_validation --subdir 03-workflows
uv run python -m sample_validation --output-dir ./reports
"""
import argparse
import asyncio
import logging
import os
import sys
import time
from pathlib import Path
# Add the samples directory to the path for imports
sys.path.insert(0, str(Path(__file__).parent.parent))
from sample_validation.models import Report
from sample_validation.report import save_report
from sample_validation.workflow import ValidationConfig, create_validation_workflow
logging.basicConfig(level=logging.INFO)
def parse_arguments() -> argparse.Namespace:
"""Parse command line arguments."""
parser = argparse.ArgumentParser(
description="Validate Python samples using a dynamic nested concurrent workflow",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
uv run python -m sample_validation # Validate all samples
uv run python -m sample_validation --subdir 03-workflows # Validate only workflows
uv run python -m sample_validation --output-dir ./reports # Save reports to custom dir
""",
)
parser.add_argument(
"--subdir",
type=str,
help="Validate samples only in the specified subdirectory (relative to samples/)",
)
parser.add_argument(
"--output-dir",
type=str,
default="./sample_validation/reports",
help="Directory to save validation reports (default: ./sample_validation/reports)",
)
parser.add_argument(
"--save-report",
action="store_true",
help="Save the validation report to files",
)
parser.add_argument(
"--max-parallel-workers",
type=int,
default=10,
help="Maximum number of samples to run in parallel per batch (default: 10)",
)
parser.add_argument(
"--report-name",
type=str,
help="Custom name for the report files (without extension). If not provided, uses timestamp.",
)
parser.add_argument(
"--exclude",
nargs="+",
type=str,
help="Subdirectory paths to exclude (relative to the search directory set by --subdir)",
)
parser.add_argument(
"--playbooks-dir",
type=str,
default="./sample_validation/playbooks",
help=(
"Directory (relative to samples/) where cached per-sample playbooks are stored and "
"reused across runs (default: ./sample_validation/playbooks)"
),
)
parser.add_argument(
"--no-cache",
action="store_true",
help="Ignore cached playbooks and always validate every sample with the agent",
)
parser.add_argument(
"--agent-timeout",
type=int,
default=120,
help=(
"Per-turn timeout in seconds for the GitHub Copilot agent while validating a sample. "
"Increase for long-running samples such as hosted agents that start a server and make "
"multiple calls (default: 120)"
),
)
return parser.parse_args()
async def main() -> int:
"""Main entry point."""
args = parse_arguments()
# Determine paths
# Script is at python/scripts/sample_validation/__main__.py
# python_root is python/, samples_dir is python/samples/
python_root = Path(__file__).parent.parent.parent
samples_dir = python_root / "samples"
print("=" * 80)
print("SAMPLE VALIDATION WORKFLOW")
print("=" * 80)
print(f"Samples directory: {samples_dir}")
print(f"Python root: {python_root}")
if os.environ.get("GITHUB_COPILOT_MODEL"):
print(
f"Using GitHub Copilot model override: {os.environ['GITHUB_COPILOT_MODEL']}"
)
# Create validation config
playbooks_dir = (samples_dir / args.playbooks_dir).resolve()
config = ValidationConfig(
samples_dir=samples_dir,
python_root=python_root,
subdir=args.subdir,
exclude=args.exclude,
max_parallel_workers=max(1, args.max_parallel_workers),
playbooks_dir=playbooks_dir,
use_cache=not args.no_cache,
agent_timeout=max(1, args.agent_timeout),
)
if config.use_cache:
print(f"Playbook cache: {playbooks_dir}")
else:
print("Playbook cache: disabled (--no-cache)")
# Create and run the workflow
workflow = create_validation_workflow(config)
print("\nStarting validation workflow...")
print("-" * 80)
# Run the workflow
run_start = time.perf_counter()
try:
events = await workflow.run("start")
finally:
run_duration = time.perf_counter() - run_start
print(f"\nWorkflow run completed in {run_duration:.2f}s")
outputs = events.get_outputs()
if not outputs:
print("\n[ERROR] Workflow did not produce any output")
return 1
report: Report = outputs[0]
# Save report if requested
if args.save_report:
output_dir = samples_dir / args.output_dir
md_path, json_path = save_report(report, output_dir, name=args.report_name)
print("\nReports saved:")
print(f" Markdown: {md_path}")
print(f" JSON: {json_path}")
# Return appropriate exit code
failed = report.failure_count + report.missing_setup_count
return 1 if failed > 0 else 0
if __name__ == "__main__":
exit_code = asyncio.run(main())
sys.exit(exit_code)