forked from zhongkaifu/TensorSharp
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathModelLifecycleService.cs
More file actions
209 lines (190 loc) · 9.42 KB
/
Copy pathModelLifecycleService.cs
File metadata and controls
209 lines (190 loc) · 9.42 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
// Copyright (c) Zhongkai Fu. All rights reserved.
// https://github.com/zhongkaifu/TensorSharp
//
// This file is part of TensorSharp.
//
// TensorSharp is licensed under the BSD-3-Clause license found in the LICENSE file in the root directory of this source tree.
//
// TensorSharp is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the BSD-3-Clause License for more details.
using System;
using System.Diagnostics;
using System.IO;
using Microsoft.Extensions.Logging;
namespace TensorSharp.Server
{
internal sealed class ModelLifecycleService : IDisposable
{
private readonly ILogger _logger;
private ModelBase _model;
private string _loadedModelPath;
private string _loadedMmProjPath;
private BackendType _backend;
public ModelLifecycleService(ILogger logger)
{
_logger = logger ?? Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance;
}
public bool IsLoaded => _model != null;
/// <summary>
/// When the operator explicitly named an MTP draft via
/// <c>--mtp-draft-model</c> (<c>TS_MTP_DRAFT_MODEL</c>) but it could not be
/// activated on the loaded target (missing file, wrong architecture, an
/// incompatible draft, or an incomplete GGUF), this holds a
/// human-readable reason. It is <c>null</c> when no draft was requested or
/// when the draft loaded successfully (<c>HasMtp</c>). The startup loader
/// promotes a non-null value to a fail-fast error so an explicit but
/// unusable draft can't silently leave speculation disabled — matching the
/// fail-fast contract the rest of startup configuration follows. Runtime
/// (Web UI) model switches read the warning log but do not fail.
/// </summary>
public string MtpDraftActivationError { get; private set; }
public string LoadedModelName => _loadedModelPath != null ? Path.GetFileName(_loadedModelPath) : null;
public string LoadedModelPath => _loadedModelPath;
public string LoadedMmProjName => _loadedMmProjPath != null ? Path.GetFileName(_loadedMmProjPath) : null;
public string LoadedMmProjPath => _loadedMmProjPath;
public string LoadedBackend => _model != null ? BackendCatalog.ToBackendValue(_backend) : null;
public string Architecture => _model?.Config?.Architecture;
public ModelBase Model => _model;
public BackendType Backend => _backend;
public bool IsModelAlreadyLoaded(string modelName)
{
return _model != null && string.Equals(LoadedModelName, modelName, StringComparison.OrdinalIgnoreCase);
}
public void LoadModel(string modelPath, string mmProjPath, string backendStr)
{
_logger.LogInformation(LogEventIds.ModelLoadStarted,
"Loading model {ModelFile} (mmproj={MmProjFile}, backend={Backend}, fullPath={ModelPath}, mmprojPath={MmProjPath})",
Path.GetFileName(modelPath), Path.GetFileName(mmProjPath ?? string.Empty),
backendStr ?? "(default)", modelPath, mmProjPath ?? "(none)");
string previousModel = LoadedModelName;
_model?.Dispose();
_model = null;
_loadedModelPath = null;
_loadedMmProjPath = null;
MtpDraftActivationError = null;
if (!string.IsNullOrEmpty(previousModel))
{
_logger.LogInformation(LogEventIds.ModelUnloaded,
"Unloaded previous model {PreviousModel}", previousModel);
}
_backend = ResolveBackend(backendStr);
var loadSw = Stopwatch.StartNew();
try
{
_model = ModelBase.Create(modelPath, _backend);
_loadedModelPath = modelPath;
if (!string.IsNullOrEmpty(mmProjPath) && File.Exists(mmProjPath))
{
LoadEncoders(mmProjPath);
_loadedMmProjPath = mmProjPath;
}
// Gemma 4 MTP: the draft head ships as a SEPARATE GGUF
// (gemma4-assistant). Load it onto the target so HasMtp turns on
// and --mtp-spec engages. (Qwen3.6 embeds its NextN block in the
// trunk and needs no separate file.) MtpDraftActivationError was
// already cleared above before the model was (re)created.
string mtpDraftPath = Environment.GetEnvironmentVariable("TS_MTP_DRAFT_MODEL");
if (!string.IsNullOrEmpty(mtpDraftPath))
{
if (_model is Gemma4Model g4)
{
if (!File.Exists(mtpDraftPath))
{
MtpDraftActivationError = $"MTP draft model file not found: {mtpDraftPath}";
_logger.LogWarning("{Error}; speculation disabled.", MtpDraftActivationError);
}
else
{
try
{
g4.LoadMtpDraftWeights(mtpDraftPath);
if (g4.HasMtp)
{
_logger.LogInformation("Loaded Gemma 4 MTP draft head {Draft} (HasMtp=True)",
Path.GetFileName(mtpDraftPath));
}
else
{
MtpDraftActivationError =
$"MTP draft '{Path.GetFileName(mtpDraftPath)}' loaded but is incomplete (required draft tensors missing).";
_logger.LogWarning("{Error}; speculation disabled.", MtpDraftActivationError);
}
}
catch (Exception mtpEx)
{
MtpDraftActivationError =
$"Failed to load MTP draft '{Path.GetFileName(mtpDraftPath)}': {mtpEx.Message}";
_logger.LogWarning(mtpEx, "Failed to load MTP draft {Path}; speculation disabled.", mtpDraftPath);
}
}
}
else
{
// A draft GGUF was named but the loaded model's architecture
// does not consume a separate draft file (e.g. Qwen3.6 embeds
// its NextN block in the trunk). Record it so the operator
// isn't left wondering why their --mtp-draft-model was ignored.
MtpDraftActivationError =
$"--mtp-draft-model was given but the loaded model architecture " +
$"'{Architecture ?? "unknown"}' does not use a separate MTP draft GGUF.";
_logger.LogWarning("{Error}; speculation disabled.", MtpDraftActivationError);
}
}
loadSw.Stop();
long modelBytes = SafeGetFileSize(modelPath);
long mmProjBytes = SafeGetFileSize(mmProjPath);
_logger.LogInformation(LogEventIds.ModelLoadCompleted,
"Loaded model {Model} (architecture={Architecture}, backend={Backend}, modelBytes={ModelBytes}, mmproj={MmProjFile}, mmprojBytes={MmProjBytes}) in {ElapsedMs:F1} ms",
LoadedModelName, Architecture ?? "(unknown)", LoadedBackend ?? "(unknown)",
modelBytes, LoadedMmProjName ?? "(none)", mmProjBytes, loadSw.Elapsed.TotalMilliseconds);
}
catch (Exception ex)
{
loadSw.Stop();
_logger.LogError(LogEventIds.ModelLoadFailed, ex,
"Failed to load model {ModelFile} on backend {Backend} after {ElapsedMs:F1} ms",
Path.GetFileName(modelPath), backendStr ?? "(default)", loadSw.Elapsed.TotalMilliseconds);
throw;
}
}
public void Dispose()
{
_model?.Dispose();
_model = null;
_loadedModelPath = null;
_loadedMmProjPath = null;
}
private void LoadEncoders(string mmProjPath)
{
_model?.MultimodalInjector.LoadProjectors(mmProjPath);
}
private static BackendType ResolveBackend(string backendStr)
{
return BackendCatalog.Canonicalize(backendStr) switch
{
"mlx" => BackendType.Mlx,
"cuda" => BackendType.Cuda,
"ggml_metal" => BackendType.GgmlMetal,
"ggml_cpu" => BackendType.GgmlCpu,
"ggml_cuda" => BackendType.GgmlCuda,
"ggml_vulkan" => BackendType.GgmlVulkan,
"cpu" => BackendType.Cpu,
_ => BackendType.GgmlCpu
};
}
private static long SafeGetFileSize(string path)
{
if (string.IsNullOrEmpty(path))
return 0;
try
{
var fi = new FileInfo(path);
return fi.Exists ? fi.Length : 0;
}
catch
{
return 0;
}
}
}
}