Repository navigation
Expand file tree
/
Copy pathsync-content.mjs
More file actions
145 lines (135 loc) · 8.01 KB
/
Copy pathsync-content.mjs
File metadata and controls
145 lines (135 loc) · 8.01 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
import fs from 'node:fs';
import path from 'node:path';
import crypto from 'node:crypto';
import matter from 'gray-matter';
import MarkdownIt from 'markdown-it';
import mathjax from 'markdown-it-mathjax3';
import footnote from 'markdown-it-footnote';
import { loadReferenceRepairs, applyReferenceRepairs } from './lib/reference-repairs.mjs';
const root = process.cwd();
const referenceRepairs = loadReferenceRepairs(root);
const baseline = JSON.parse(fs.readFileSync('migration/baseline.json', 'utf8'));
const byId = new Map(baseline.map(post => [post.id, post]));
const pathMoves = JSON.parse(fs.readFileSync('migration/path-moves.json', 'utf8')).moves;
const bySlug = new Map(baseline.filter(post => post.published).map(post => [post.slug, post.url]));
const courses = JSON.parse(fs.readFileSync('src/data/courses.json', 'utf8'));
const byCourse = new Map(courses.map(course => [course.slug, course]));
const md = new MarkdownIt({ html: true, linkify: true, typographer: false }).use(mathjax).use(footnote);
const sha = value => crypto.createHash('sha256').update(value).digest('hex');
const escape = md.utils.escapeHtml;
const originalFence = md.renderer.rules.fence;
md.renderer.rules.fence = (tokens, index, options, env, self) => {
if (tokens[index].info.trim() === 'mermaid') return `<pre class="mermaid">${escape(tokens[index].content)}</pre>`;
return originalFence(tokens, index, options, env, self);
};
function walk(directory) {
return fs.readdirSync(directory, { withFileTypes: true }).flatMap(entry => {
const file = path.join(directory, entry.name);
return entry.isDirectory() ? walk(file) : entry.name.endsWith('.md') ? [file] : [];
});
}
function list(value, fallback = []) {
if (value == null) return fallback;
return (Array.isArray(value) ? value.flat(Infinity) : [value]).map(String);
}
function dateString(value) {
if (!value || Number.isNaN(new Date(value).valueOf())) throw new Error('New posts require a valid date or pubDate.');
return new Date(value).toISOString();
}
function text(html) {
return md.utils.unescapeAll(html.replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ')).trim();
}
function headings(html) {
const result = [];
for (const match of html.matchAll(/<h([23])\b[^>]*\bid="([^"]+)"[^>]*>([\s\S]*?)<\/h\1>/g)) {
result.push({ level: Number(match[1]), id: md.utils.unescapeAll(match[2]), text: text(match[3]) });
}
return result;
}
function renderMarkdown(content) {
content = content.replace(/\{%\s*post_link\s+(\S+)(?:\s+([\s\S]*?))?\s*%\}/g, (_, slug, label) => {
const url = bySlug.get(slug);
if (!url) throw new Error(`Unknown legacy post_link: ${slug}`);
return `[${label || slug}](${encodeURI(url)})`;
});
let html = md.render(content);
const used = new Set();
html = html.replace(/<h([1-6])>([\s\S]*?)<\/h\1>/g, (_, level, content) => {
const base = text(content).replace(/\s+/g, '-').replace(/[<>"'?#]/g, '') || 'section';
let id = base; let suffix = 1;
while (used.has(id)) id = `${base}-${suffix++}`;
used.add(id);
return `<h${level} id="${escape(id)}">${content}</h${level}>`;
});
return html;
}
const posts = [];
for (const file of walk('content/posts')) {
const physicalId = path.relative('content/posts', file).split(path.sep).join('/');
const id = pathMoves[physicalId] || physicalId;
const raw = fs.readFileSync(file, 'utf8');
const original = byId.get(id);
const { data, content } = matter(raw);
if (data.draft === true || data.published === false) continue;
const hash = sha(raw);
const bodyHash = sha(content);
const preserved = original?.published && (original.markdown_body_sha256 ? original.markdown_body_sha256 === bodyHash : original.markdown_sha256 === hash);
if (data.course) {
const course = byCourse.get(data.course);
if (!course || !course.groups.some(group => group.id === data.course_group)) throw new Error(`Invalid course or group: ${id}`);
if (course.contentDirectory && !physicalId.startsWith(`${course.contentDirectory}/`)) throw new Error(`Course article outside content directory: ${physicalId}`);
}
const title = String(data.title || original?.title || '');
if (!title) throw new Error(`Missing title: ${id}`);
if (!original && !String(data.description || '').trim()) throw new Error(`New posts require a description: ${id}`);
const date = data.pubDate ? dateString(data.pubDate) : original?.date || dateString(data.date);
const url = original?.url || '/' + String(data.permalink || data.slug || id.replace(/\.md$/, '')).replace(/^\/+|\/+$/g, '') + '/';
const html = applyReferenceRepairs(id, preserved ? fs.readFileSync(original.snapshot, 'utf8') : renderMarkdown(content), referenceRepairs);
const plain = text(html);
const fallbackDescription = (plain.startsWith(title) ? plain.slice(title.length) : plain).trim().slice(0, 170);
let series = data.series || original?.series;
if (!series && id.startsWith('ai-work/')) series = '与 AI 一起工作';
if (!series && id.startsWith('benchmarking/')) series = '基准测试方法与实践';
const inferredOrder = [path.basename(id), title].map(value => value.match(/(?:^|[||])\s*(\d+)[.、_-]/)?.[1]).find(Boolean);
const order = Number(data.series_order ?? original?.series_order ?? inferredOrder);
posts.push({
id, title, url, date,
updated: data.updated ? dateString(data.updated) : original?.updated || date,
...(data.reviewed ? { reviewed: dateString(data.reviewed) } : {}),
...(data.applicable_versions ? { applicable_versions: String(data.applicable_versions) } : {}),
...(data.verified_environment ? { verified_environment: String(data.verified_environment) } : {}),
...(data.image ? { image: String(data.image) } : {}),
description: text(String(data.description || original?.description || fallbackDescription)),
tags: list(data.tags, original?.tags),
categories: list(data.categories, original?.categories),
...(data.course ? { course: String(data.course), course_group: String(data.course_group), course_order: Number(data.course_order) } : {}),
...(data.content_type ? { content_type: String(data.content_type) } : {}),
...(series ? { series: String(series) } : {}),
...(Number.isFinite(order) ? { series_order: Number(order) } : {}),
readingMinutes: original?.readingMinutes || Math.max(1, Math.ceil(plain.length / 350)),
html, source: `content/posts/${physicalId}`, markdown_sha256: hash, markdown_body_sha256: bodyHash,
counterPath: original?.legacyUrls?.[0] || url,
renderMode: preserved ? 'preserved' : 'markdown', headings: headings(html),
});
}
const seen = new Set();
for (const post of posts) {
if (seen.has(post.url)) throw new Error(`Duplicate article URL: ${post.url}`);
seen.add(post.url);
}
posts.sort((a, b) => b.date.localeCompare(a.date) || a.title.localeCompare(b.title, 'zh-CN'));
fs.mkdirSync('src/data', { recursive: true });
fs.writeFileSync('src/data/posts.json', JSON.stringify(posts));
const aboutBaseline = JSON.parse(fs.readFileSync('migration/about.json', 'utf8'));
const aboutRaw = fs.readFileSync('content/about/index.md', 'utf8');
const aboutHtml = sha(aboutRaw) === aboutBaseline.markdown_sha256 ? fs.readFileSync('content/legacy/about.html', 'utf8') : renderMarkdown(matter(aboutRaw).content);
fs.writeFileSync('src/data/about.json', JSON.stringify({ html: aboutHtml }));
// Copy dependencies for self-hosted math styling and the static search interface.
fs.mkdirSync('public/vendor/katex', { recursive: true });
for (const entry of ['katex.min.css', 'fonts']) fs.cpSync(`node_modules/katex/dist/${entry}`, `public/vendor/katex/${entry}`, { recursive: true });
fs.writeFileSync('public/.nojekyll', '');
const analytics = JSON.parse(fs.readFileSync('src/data/analytics.json', 'utf8'));
const analyticsTemplate = fs.readFileSync('scripts/analytics-template.js', 'utf8');
fs.mkdirSync('public/js', { recursive: true });
fs.writeFileSync('public/js/analytics.js', analyticsTemplate.replace('ANALYTICS_CONFIG', JSON.stringify(analytics)));
console.log(`Content ready: ${posts.length} articles; ${posts.filter(post => post.renderMode === 'preserved').length} preserved publication snapshots; drafts excluded.`);