-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathcontinuous_deploy.py
More file actions
350 lines (289 loc) · 12.8 KB
/
continuous_deploy.py
File metadata and controls
350 lines (289 loc) · 12.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
"""
Continuous deployment based on metrics for ModelSync
"""
import json
import subprocess
import requests
from pathlib import Path
from typing import Dict, List, Optional, Any, Callable
from datetime import datetime
from modelsync.utils.helpers import ensure_directory, write_json_file, read_json_file
class DeploymentRule:
"""Represents a deployment rule based on metrics"""
def __init__(
self,
name: str,
branch: str,
metric_name: str,
threshold: float,
operator: str, # "greater_than", "less_than", "equals"
deployment_target: str,
deployment_config: Dict[str, Any]
):
self.name = name
self.branch = branch
self.metric_name = metric_name
self.threshold = threshold
self.operator = operator
self.deployment_target = deployment_target
self.deployment_config = deployment_config
self.created_at = datetime.now().isoformat()
self.last_checked = None
self.triggered_count = 0
def check_condition(self, metrics: Dict[str, float]) -> bool:
"""Check if deployment condition is met"""
if self.metric_name not in metrics:
return False
value = metrics[self.metric_name]
if self.operator == "greater_than":
return value > self.threshold
elif self.operator == "less_than":
return value < self.threshold
elif self.operator == "equals":
return abs(value - self.threshold) < 0.001
elif self.operator == "greater_equal":
return value >= self.threshold
elif self.operator == "less_equal":
return value <= self.threshold
return False
def to_dict(self) -> Dict[str, Any]:
"""Convert rule to dictionary"""
return {
"name": self.name,
"branch": self.branch,
"metric_name": self.metric_name,
"threshold": self.threshold,
"operator": self.operator,
"deployment_target": self.deployment_target,
"deployment_config": self.deployment_config,
"created_at": self.created_at,
"last_checked": self.last_checked,
"triggered_count": self.triggered_count
}
class DeploymentManager:
"""Manages continuous deployment based on metrics"""
def __init__(self, repo_path: str = "."):
self.repo_path = Path(repo_path)
self.deploy_dir = self.repo_path / ".modelsync" / "deployment"
self.rules_file = self.deploy_dir / "rules.json"
self.deployments_file = self.deploy_dir / "deployments.json"
self.rules: List[DeploymentRule] = []
self.deployments: List[Dict[str, Any]] = []
ensure_directory(str(self.deploy_dir))
self._load_data()
def _load_data(self):
"""Load deployment data"""
rules_data = read_json_file(str(self.rules_file)) or []
self.rules = [self._rule_from_dict(rule) for rule in rules_data]
self.deployments = read_json_file(str(self.deployments_file)) or []
def _save_data(self):
"""Save deployment data"""
rules_data = [rule.to_dict() for rule in self.rules]
write_json_file(str(self.rules_file), rules_data)
write_json_file(str(self.deployments_file), self.deployments)
def _rule_from_dict(self, rule_data: Dict[str, Any]) -> DeploymentRule:
"""Create rule from dictionary"""
rule = DeploymentRule(
name=rule_data["name"],
branch=rule_data["branch"],
metric_name=rule_data["metric_name"],
threshold=rule_data["threshold"],
operator=rule_data["operator"],
deployment_target=rule_data["deployment_target"],
deployment_config=rule_data["deployment_config"]
)
rule.created_at = rule_data.get("created_at", rule.created_at)
rule.last_checked = rule_data.get("last_checked")
rule.triggered_count = rule_data.get("triggered_count", 0)
return rule
def add_deployment_rule(
self,
name: str,
branch: str,
metric_name: str,
threshold: float,
operator: str,
deployment_target: str,
deployment_config: Dict[str, Any]
) -> bool:
"""Add a new deployment rule"""
# Check if rule name already exists
if any(rule.name == name for rule in self.rules):
print(f"❌ Rule '{name}' already exists")
return False
rule = DeploymentRule(
name=name,
branch=branch,
metric_name=metric_name,
threshold=threshold,
operator=operator,
deployment_target=deployment_target,
deployment_config=deployment_config
)
self.rules.append(rule)
self._save_data()
print(f"✅ Added deployment rule: {name}")
return True
def check_deployment_rules(
self,
branch: str,
metrics: Dict[str, float],
model_id: str = None
) -> List[Dict[str, Any]]:
"""Check all deployment rules for a branch"""
triggered_rules = []
for rule in self.rules:
if rule.branch != branch:
continue
if rule.check_condition(metrics):
print(f"🚀 Deployment rule triggered: {rule.name}")
# Update rule stats
rule.last_checked = datetime.now().isoformat()
rule.triggered_count += 1
# Execute deployment
deployment_result = self._execute_deployment(rule, metrics, model_id)
triggered_rules.append({
"rule_name": rule.name,
"branch": branch,
"metrics": metrics,
"model_id": model_id,
"deployment_result": deployment_result,
"triggered_at": datetime.now().isoformat()
})
if triggered_rules:
self._save_data()
return triggered_rules
def _execute_deployment(
self,
rule: DeploymentRule,
metrics: Dict[str, float],
model_id: str = None
) -> Dict[str, Any]:
"""Execute deployment based on rule"""
deployment_id = f"{rule.name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
deployment_data = {
"id": deployment_id,
"rule_name": rule.name,
"branch": rule.branch,
"model_id": model_id,
"metrics": metrics,
"deployment_target": rule.deployment_target,
"config": rule.deployment_config,
"started_at": datetime.now().isoformat(),
"status": "running"
}
try:
if rule.deployment_target == "docker":
result = self._deploy_docker(deployment_data)
elif rule.deployment_target == "kubernetes":
result = self._deploy_kubernetes(deployment_data)
elif rule.deployment_target == "api_endpoint":
result = self._deploy_api_endpoint(deployment_data)
elif rule.deployment_target == "mlflow":
result = self._deploy_mlflow(deployment_data)
else:
result = {"status": "error", "message": f"Unknown deployment target: {rule.deployment_target}"}
deployment_data.update(result)
deployment_data["completed_at"] = datetime.now().isoformat()
except Exception as e:
deployment_data["status"] = "failed"
deployment_data["error"] = str(e)
deployment_data["failed_at"] = datetime.now().isoformat()
result = {"status": "failed", "error": str(e)}
# Save deployment record
self.deployments.append(deployment_data)
self._save_data()
return result
def _deploy_docker(self, deployment_data: Dict[str, Any]) -> Dict[str, Any]:
"""Deploy using Docker"""
config = deployment_data["config"]
# Build Docker image
build_cmd = config.get("build_command", "docker build -t {image_name} .")
image_name = config.get("image_name", "modelsync-model")
build_cmd = build_cmd.format(image_name=image_name)
try:
result = subprocess.run(build_cmd, shell=True, capture_output=True, text=True)
if result.returncode != 0:
return {"status": "failed", "error": result.stderr}
except Exception as e:
return {"status": "failed", "error": str(e)}
# Run Docker container
run_cmd = config.get("run_command", "docker run -d -p {port}:8000 {image_name}")
port = config.get("port", "8000")
run_cmd = run_cmd.format(image_name=image_name, port=port)
try:
result = subprocess.run(run_cmd, shell=True, capture_output=True, text=True)
if result.returncode != 0:
return {"status": "failed", "error": result.stderr}
container_id = result.stdout.strip()
return {"status": "success", "container_id": container_id, "port": port}
except Exception as e:
return {"status": "failed", "error": str(e)}
def _deploy_kubernetes(self, deployment_data: Dict[str, Any]) -> Dict[str, Any]:
"""Deploy using Kubernetes"""
config = deployment_data["config"]
# Apply Kubernetes manifests
manifest_path = config.get("manifest_path")
if not manifest_path:
return {"status": "error", "message": "No manifest path specified"}
try:
result = subprocess.run(f"kubectl apply -f {manifest_path}", shell=True, capture_output=True, text=True)
if result.returncode != 0:
return {"status": "failed", "error": result.stderr}
return {"status": "success", "output": result.stdout}
except Exception as e:
return {"status": "failed", "error": str(e)}
def _deploy_api_endpoint(self, deployment_data: Dict[str, Any]) -> Dict[str, Any]:
"""Deploy to API endpoint"""
config = deployment_data["config"]
endpoint = config.get("endpoint")
if not endpoint:
return {"status": "error", "message": "No endpoint specified"}
# Prepare deployment payload
payload = {
"model_id": deployment_data["model_id"],
"metrics": deployment_data["metrics"],
"config": config
}
try:
response = requests.post(endpoint, json=payload, timeout=30)
if response.status_code == 200:
return {"status": "success", "response": response.json()}
else:
return {"status": "failed", "error": f"HTTP {response.status_code}: {response.text}"}
except Exception as e:
return {"status": "failed", "error": str(e)}
def _deploy_mlflow(self, deployment_data: Dict[str, Any]) -> Dict[str, Any]:
"""Deploy using MLflow"""
config = deployment_data["config"]
# MLflow deployment commands
model_uri = config.get("model_uri")
if not model_uri:
return {"status": "error", "message": "No model URI specified"}
try:
# Register model in MLflow
register_cmd = f"mlflow models serve -m {model_uri} -p {config.get('port', '5000')}"
result = subprocess.run(register_cmd, shell=True, capture_output=True, text=True)
if result.returncode != 0:
return {"status": "failed", "error": result.stderr}
return {"status": "success", "output": result.stdout}
except Exception as e:
return {"status": "failed", "error": str(e)}
def list_deployment_rules(self) -> List[Dict[str, Any]]:
"""List all deployment rules"""
return [rule.to_dict() for rule in self.rules]
def get_deployments(self, branch: str = None) -> List[Dict[str, Any]]:
"""Get deployment history"""
if branch:
return [d for d in self.deployments if d.get("branch") == branch]
return self.deployments
def remove_deployment_rule(self, name: str) -> bool:
"""Remove a deployment rule"""
for i, rule in enumerate(self.rules):
if rule.name == name:
del self.rules[i]
self._save_data()
print(f"✅ Removed deployment rule: {name}")
return True
print(f"❌ Rule '{name}' not found")
return False