-
Notifications
You must be signed in to change notification settings - Fork 11
Expand file tree
/
Copy pathstage.cppm
More file actions
340 lines (301 loc) · 14.8 KB
/
Copy pathstage.cppm
File metadata and controls
340 lines (301 loc) · 14.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
// mcpp.build.stage — the staging primitive behind the ninja `stage_file` rule.
//
// Staging = publish a file the build cache owns (std BMI, std.o, a dep's
// runtime DLL) into the build directory where the compiler / loader expects
// it. This used to be a shelled-out copy per platform:
//
// POSIX: mkdir -p $(dirname $out) && cp -f $in $out
// Windows: powershell -NoProfile -Command "Copy-Item -Force '$in' -Destination '$out'"
//
// Both overwrite the destination in place, unconditionally. On Windows that is
// a live hazard (#311): mcpp writes the staged std BMI path into
// compile_commands.json so clangd can resolve `import std;`, clangd then
// memory-maps that very file, and an in-place overwrite of a file with an open
// user-mapped section fails with error 1224 — the whole build reports
// "build failed" even when the bytes were already correct.
//
// The rules this module implements:
//
// 1. Never write bytes that are already there. Equivalence is decided by
// COMPARING CONTENT, which is unconditionally correct — a destination
// identical to the source needs no write, whatever the reason. Size alone
// is only a heuristic and is available as `--verify size` for the paths
// where the caller knows better (a std BMI is fingerprint-scoped: the
// cache dir and the build dir share the fp that covers compiler identity,
// target triple, stdlib, std source hash and the dialect flags). It is
// NOT the default, because staging also carries .dll payloads whose PE
// section padding makes "same size, different bytes" ordinary.
// Skipping mirrors what the dep BMI cache has always done
// (bmi_cache.cppm: "Existing project outputs are left untouched").
// 2. When we do write, write out of place and rename — atomic for readers,
// and it survives a transient sharing violation (antivirus, indexer)
// that an in-place overwrite would lose to.
// 3. Retry a few times, then fail with a diagnostic that names the file and
// the likely holder. Never downgrade a real staging failure to a warning:
// a stale or missing BMI turns into either a confusing
// "module 'std' not found" or a silently mismatched link.
//
// Skipping deliberately does not touch the destination's timestamps AT ALL —
// not even to align them with the source. Measured on the ninja side: any mtime
// bump (to "now" or to the source's mtime) counts as "the command changed this
// output", so `restat = 1` no longer suppresses the downstream rebuild and
// every importer of the staged BMI recompiles for nothing. Leaving the mtime
// alone keeps the edge dirty — ninja re-runs this ~no-op on each build until
// the content actually changes — which is the cheap side of the trade
// (one process vs. recompiling the module graph).
export module mcpp.build.stage;
import std;
export namespace mcpp::build::stage {
// How hard to look before declaring the destination already-staged.
enum class Verify {
Content, // default: byte-for-byte compare — always correct
Size, // size match only (MCPP_STAGE_VERIFY=size / --verify size)
};
struct StageOptions {
Verify verify = Verify::Content;
int retries = 3; // attempts after the first
std::chrono::milliseconds backoff{100}; // ×3 per retry: 100/300/900ms
};
struct StageOutcome {
bool copied = false; // false = destination was already equivalent
};
struct StageError {
std::string message; // multi-line, already carries the `hint:` block
};
// Publish `src` at `dst`. See the module comment for the exact semantics.
std::expected<StageOutcome, StageError> stage_file(const std::filesystem::path& src,
const std::filesystem::path& dst,
const StageOptions& opts = {});
// Byte-for-byte comparison (exported for tests). False when either file is
// unreadable or the sizes differ.
bool same_content(const std::filesystem::path& a, const std::filesystem::path& b);
// Are two BMIs equivalent for the purpose of "did this module's interface
// change?" — i.e. identical except for the wall clock GCC stamps into them.
//
// WHY THIS EXISTS. The `cxx_module` rule keeps the previous BMI, recompiles,
// and restores the old file when the new one has the same content, so ninja's
// restat sees an unchanged output and does NOT rebuild the importers. That
// mechanism was designed in 2026-05-12 and has NEVER ONCE FIRED, because GCC
// writes
//
// buildtime: 2026/08/12 02:25:01 UTC
// localtime: 2026/08/12 02:25:01 UTC
//
// INTO THE BMI CONTENT. Two compilations of identical source a second apart
// differ by exactly four bytes, so a plain `cmp` always reports "changed".
// Measured on this repository: touching a module with 46 importers and no
// content change cost 73.0 s and re-ran 180 edges — indistinguishable from a
// full rebuild.
//
// The earlier design note anticipated only that GCC would rewrite the FILE
// (mtime churn) and prescribed a content compare as the fix; it did not
// anticipate that the timestamp is part of the content, which is why the fix
// as written could not work.
//
// Deliberately NOT solved with SOURCE_DATE_EPOCH: that pins the epoch for the
// whole compilation and so changes what `__DATE__` and `__TIME__` expand to in
// USER code. Masking the two fields here changes what mcpp considers equal and
// nothing else.
//
// Conservative by construction: if the expected stamps are not found, or the
// two files disagree about where they are, this falls back to a strict
// comparison. It can report "different" for BMIs that are equivalent; it must
// never report "same" for BMIs that are not.
bool bmi_equivalent(const std::filesystem::path& a, const std::filesystem::path& b);
// Parse a --verify / MCPP_STAGE_VERIFY value. Unknown values fall back to the
// safe default (Content).
Verify parse_verify(std::string_view value);
} // namespace mcpp::build::stage
namespace mcpp::build::stage {
namespace {
std::filesystem::path temp_sibling(const std::filesystem::path& dst) {
// Same directory as the destination so the rename stays on one filesystem.
auto stamp = std::chrono::system_clock::now().time_since_epoch().count();
auto name = dst.filename().string() + std::format(".tmp.{}", stamp);
return dst.parent_path() / name;
}
// One full write attempt: temp+rename first, in-place overwrite as fallback.
// Returns an empty error_code on success; otherwise the most informative error
// (the in-place one — that's where 1224 / 32 shows up).
std::error_code write_once(const std::filesystem::path& src,
const std::filesystem::path& dst) {
auto tmp = temp_sibling(dst);
std::error_code ec;
std::filesystem::copy_file(src, tmp, std::filesystem::copy_options::overwrite_existing, ec);
if (!ec) {
std::error_code rec;
std::filesystem::rename(tmp, dst, rec);
if (!rec) return {};
// Renaming over a destination that another process holds mapped fails
// too (the destination must be deletable) — fall through to the
// in-place path, which wins when the holder allows writes but not
// deletes.
std::error_code rmec;
std::filesystem::remove(tmp, rmec);
} else {
std::error_code rmec;
std::filesystem::remove(tmp, rmec);
}
std::error_code cec;
std::filesystem::copy_file(src, dst, std::filesystem::copy_options::overwrite_existing, cec);
if (!cec) return {};
return cec;
}
std::string failure_message(const std::filesystem::path& src,
const std::filesystem::path& dst,
const std::error_code& ec) {
// ninja runs staging with cwd = the build directory, so `$out` is relative.
// Print it absolute: the reader has to go find (or unlock) this file.
std::error_code aec;
auto shown = std::filesystem::absolute(dst, aec);
if (aec) shown = dst;
return std::format(
"cannot stage file into the build directory\n"
" file: {}\n"
" from: {}\n"
" os error: {} ({})\n"
"hint: another process has this file memory-mapped, loaded or open.\n"
" The usual holder is clangd (mcpp writes staged BMI paths into\n"
" compile_commands.json so clangd can resolve `import std;`), an\n"
" editor/IDE indexer, antivirus, or — for a .dll — a still-running\n"
" program from a previous `mcpp run`. Close it or restart clangd,\n"
" then re-run the build.",
shown.string(), src.string(), ec.value(), ec.message());
}
} // namespace
bool same_content(const std::filesystem::path& a, const std::filesystem::path& b) {
std::error_code ec1, ec2;
auto sa = std::filesystem::file_size(a, ec1);
auto sb = std::filesystem::file_size(b, ec2);
if (ec1 || ec2 || sa != sb) return false;
std::ifstream fa(a, std::ios::binary);
std::ifstream fb(b, std::ios::binary);
if (!fa || !fb) return false;
constexpr std::size_t kChunk = 1u << 16;
std::vector<char> ba(kChunk), bb(kChunk);
while (fa && fb) {
fa.read(ba.data(), static_cast<std::streamsize>(kChunk));
fb.read(bb.data(), static_cast<std::streamsize>(kChunk));
auto na = fa.gcount();
auto nb = fb.gcount();
if (na != nb) return false;
if (na == 0) break;
if (std::memcmp(ba.data(), bb.data(), static_cast<std::size_t>(na)) != 0)
return false;
}
return true;
}
namespace {
// GCC writes the stamp as `<prefix>YYYY/MM/DD HH:MM:SS UTC`. Fixed width, so a
// match can be masked without re-parsing.
constexpr std::size_t kStampLen = std::string_view("2026/08/12 02:25:01 UTC").size();
bool looks_like_stamp(std::string_view v) {
if (v.size() != kStampLen) return false;
auto digit = [&](std::size_t i) { return v[i] >= '0' && v[i] <= '9'; };
return digit(0) && digit(1) && digit(2) && digit(3) && v[4] == '/'
&& digit(5) && digit(6) && v[7] == '/'
&& digit(8) && digit(9) && v[10] == ' '
&& digit(11) && digit(12) && v[13] == ':'
&& digit(14) && digit(15) && v[16] == ':'
&& digit(17) && digit(18) && v.substr(19) == " UTC";
}
// GCC writes exactly one `buildtime:` and one `localtime:` into a BMI header —
// verified across BMIs from 10 KiB to 645 KiB, always 2. Anything beyond that
// came from somewhere else (a string literal in user code that happens to look
// like a stamp), and masking it would hide a REAL difference. Finding more than
// this many makes the comparison fall back to strict equality.
constexpr std::size_t kMaxStampSpans = 2;
// Byte spans to ignore, in ascending order. Only spans whose payload actually
// looks like a timestamp are masked — a prefix that happens to appear in some
// other position is left to compare strictly.
std::vector<std::pair<std::size_t, std::size_t>> stamp_spans(std::string_view data) {
std::vector<std::pair<std::size_t, std::size_t>> spans;
for (std::string_view prefix : {"buildtime: ", "localtime: "}) {
for (std::size_t at = data.find(prefix); at != std::string_view::npos;
at = data.find(prefix, at + 1)) {
const auto start = at + prefix.size();
if (start + kStampLen > data.size()) continue;
if (!looks_like_stamp(data.substr(start, kStampLen))) continue;
spans.emplace_back(start, start + kStampLen);
}
}
std::ranges::sort(spans);
return spans;
}
std::optional<std::string> read_all(const std::filesystem::path& p) {
std::ifstream in(p, std::ios::binary);
if (!in) return std::nullopt;
return std::string((std::istreambuf_iterator<char>(in)),
std::istreambuf_iterator<char>());
}
} // namespace
bool bmi_equivalent(const std::filesystem::path& a, const std::filesystem::path& b) {
auto da = read_all(a);
auto db = read_all(b);
if (!da || !db) return false;
// The stamps are fixed width, so equivalent BMIs always have equal size. A
// size difference is a real difference, never a maskable one.
if (da->size() != db->size()) return false;
const auto sa = stamp_spans(*da);
const auto sb = stamp_spans(*db);
// Disagreement about WHERE the stamps are is itself a structural
// difference; fall back to strict equality rather than guessing.
if (sa != sb) return *da == *db;
if (sa.empty() || sa.size() > kMaxStampSpans) return *da == *db;
std::size_t cursor = 0;
for (const auto& [start, end] : sa) {
if (start > cursor
&& std::memcmp(da->data() + cursor, db->data() + cursor, start - cursor) != 0)
return false;
cursor = end;
}
return cursor >= da->size()
|| std::memcmp(da->data() + cursor, db->data() + cursor, da->size() - cursor) == 0;
}
Verify parse_verify(std::string_view value) {
return value == "size" ? Verify::Size : Verify::Content;
}
std::expected<StageOutcome, StageError> stage_file(const std::filesystem::path& src,
const std::filesystem::path& dst,
const StageOptions& opts) {
std::error_code ec;
if (!std::filesystem::exists(src, ec)) {
return std::unexpected(StageError{
std::format("staging source does not exist: {}", src.string())});
}
if (!dst.parent_path().empty()) {
std::error_code dec;
std::filesystem::create_directories(dst.parent_path(), dec);
if (dec && !std::filesystem::is_directory(dst.parent_path())) {
return std::unexpected(StageError{
std::format("cannot create '{}': {}",
dst.parent_path().string(), dec.message())});
}
}
// Already staged? Content by default — size is the caller-opt-in shortcut
// (see module comment).
std::error_code sec;
if (std::filesystem::is_regular_file(dst, sec)) {
std::error_code se1, se2;
auto sizeDst = std::filesystem::file_size(dst, se1);
auto sizeSrc = std::filesystem::file_size(src, se2);
bool equivalent = !se1 && !se2 && sizeDst == sizeSrc
&& (opts.verify == Verify::Size || same_content(src, dst));
if (equivalent) {
// Timestamps left untouched on purpose — see module comment.
return StageOutcome{.copied = false};
}
}
auto delay = opts.backoff;
std::error_code last;
for (int attempt = 0; attempt <= opts.retries; ++attempt) {
if (attempt > 0) {
std::this_thread::sleep_for(delay);
delay *= 3;
}
last = write_once(src, dst);
if (!last) return StageOutcome{.copied = true};
}
return std::unexpected(StageError{failure_message(src, dst, last)});
}
} // namespace mcpp::build::stage