-
Notifications
You must be signed in to change notification settings - Fork 21
Expand file tree
/
Copy pathtest_cli_args.py
More file actions
156 lines (140 loc) · 5.65 KB
/
Copy pathtest_cli_args.py
File metadata and controls
156 lines (140 loc) · 5.65 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
import argparse
import pytest
# Module to be tested
from eval_protocol.cli import parse_args
class TestCliArgParsing:
# --- Tests for 'preview' command ---
def test_preview_with_remote_url_and_samples(self):
args_list = [
"preview",
"--remote-url",
"http://example.com/eval",
"--samples",
"dummy.jsonl",
]
parsed, _ = parse_args(args_list)
assert parsed.command == "preview"
assert parsed.remote_url == "http://example.com/eval"
assert parsed.samples == "dummy.jsonl"
assert parsed.metrics_folders is None # Should be None if not provided
def test_preview_with_remote_url_and_hf_dataset(self):
args_list = [
"preview",
"--remote-url",
"http://example.com/eval",
"--hf",
"dataset_name",
]
parsed, _ = parse_args(args_list)
assert parsed.command == "preview"
assert parsed.remote_url == "http://example.com/eval"
assert parsed.huggingface_dataset == "dataset_name"
def test_preview_with_remote_url_and_metrics_folders(self):
"""Metrics folders should be accepted by argparse but logic in command might ignore/warn."""
args_list = [
"preview",
"--remote-url",
"http://example.com/eval",
"--metrics-folders",
"mf=path",
"--samples",
"s.jsonl",
]
parsed, _ = parse_args(args_list)
assert parsed.command == "preview"
assert parsed.remote_url == "http://example.com/eval"
assert parsed.metrics_folders == ["mf=path"]
def test_preview_without_remote_url_requires_metrics_folders_or_command_logic_handles(
self,
):
"""Argparse allows no metrics_folders, command logic should enforce if needed."""
args_list = [
"preview",
"--samples",
"dummy.jsonl",
] # No --remote-url, no --metrics-folders
parsed, _ = parse_args(args_list)
assert parsed.command == "preview"
assert parsed.remote_url is None
assert parsed.metrics_folders is None
# The command logic in preview.py now checks:
# if not args.remote_url and not args.metrics_folders: error
def test_preview_traditional_with_metrics_folders(self):
args_list = [
"preview",
"--metrics-folders",
"mf=path",
"--samples",
"dummy.jsonl",
]
parsed, _ = parse_args(args_list)
assert parsed.command == "preview"
assert parsed.metrics_folders == ["mf=path"]
assert parsed.remote_url is None
# --- Tests for 'deploy' command ---
def test_deploy_with_remote_url(self):
args_list = [
"deploy",
"--id",
"my-eval",
"--remote-url",
"http://example.com/deploy-eval",
]
parsed, _ = parse_args(args_list)
assert parsed.command == "deploy"
assert parsed.id == "my-eval"
assert parsed.remote_url == "http://example.com/deploy-eval"
assert parsed.metrics_folders is None # Not required, should be None if not given
def test_deploy_with_remote_url_and_metrics_folders(self):
"""Metrics folders should be accepted by argparse but logic in command might ignore/warn."""
args_list = [
"deploy",
"--id",
"my-eval",
"--remote-url",
"http://example.com/eval",
"--metrics-folders",
"mf=path",
]
parsed, _ = parse_args(args_list)
assert parsed.command == "deploy"
assert parsed.id == "my-eval"
assert parsed.remote_url == "http://example.com/eval"
assert parsed.metrics_folders == ["mf=path"]
def test_deploy_traditional_without_remote_url(self):
args_list = ["deploy", "--id", "my-eval", "--metrics-folders", "mf=path"]
parsed, _ = parse_args(args_list)
assert parsed.command == "deploy"
assert parsed.id == "my-eval"
assert parsed.metrics_folders == ["mf=path"]
assert parsed.remote_url is None
def test_deploy_traditional_metrics_folders_still_optional_at_parser_level(self):
"""
--metrics-folders is required=False at parser level.
The command logic in deploy.py enforces it if --remote-url is not present.
"""
args_list = [
"deploy",
"--id",
"my-eval",
] # No --metrics-folders, no --remote-url
# This should parse fine, but deploy_command will raise error.
parsed, _ = parse_args(args_list)
assert parsed.command == "deploy"
assert parsed.id == "my-eval"
assert parsed.metrics_folders is None
assert parsed.remote_url is None
def test_deploy_id_is_required(self):
with pytest.raises(SystemExit): # argparse exits on missing required arg
parse_args(["deploy"]) # Missing --id
# General verbose flag
def test_verbose_flag(self):
# Global flags like -v or --verbose should typically come before the subcommand
parsed_verbose_short, _ = parse_args(["-v", "preview", "--samples", "s.jsonl", "--metrics-folders", "m=p"])
assert parsed_verbose_short.verbose is True
parsed_verbose_long, _ = parse_args(
["--verbose", "preview", "--samples", "s.jsonl", "--metrics-folders", "m=p"]
)
assert parsed_verbose_long.verbose is True
parsed_not_verbose, _ = parse_args(["preview", "--samples", "s.jsonl", "--metrics-folders", "m=p"])
assert parsed_not_verbose.verbose is False