-
Notifications
You must be signed in to change notification settings - Fork 6
Expand file tree
/
Copy pathsubmit-indexnow.mjs
More file actions
118 lines (100 loc) · 3.49 KB
/
Copy pathsubmit-indexnow.mjs
File metadata and controls
118 lines (100 loc) · 3.49 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
import fs from "node:fs/promises";
import path from "node:path";
import { parseArgs } from "node:util";
const DEFAULT_HOST = "www.javajub.com";
const DEFAULT_ENDPOINT = "https://yandex.com/indexnow";
const DEFAULT_KEY = "e83be258c353fa1282249ebe3e69ab3595ad9667f969b329449721fdcaab3b8b";
const EXCLUDED_PATHS = new Set(["/404/", "/404.html"]);
const { values } = parseArgs({
options: {
"dry-run": { type: "boolean", default: false },
endpoint: { type: "string", default: DEFAULT_ENDPOINT },
host: { type: "string", default: process.env.INDEXNOW_HOST || DEFAULT_HOST },
key: { type: "string", default: process.env.INDEXNOW_KEY || DEFAULT_KEY },
"key-location": { type: "string" },
limit: { type: "string" },
sitemap: { type: "string" },
},
});
const host = values.host.replace(/^https?:\/\//, "").replace(/\/$/, "");
const siteUrl = `https://${host}`;
const key = values.key;
const keyLocation = values["key-location"] || `${siteUrl}/${key}.txt`;
const sitemapLocation = values.sitemap || `${siteUrl}/sitemap.xml`;
const limit = values.limit ? Number.parseInt(values.limit, 10) : undefined;
if (!/^[a-z0-9_-]{8,128}$/i.test(key)) {
throw new Error("IndexNow key must be 8-128 URL-safe characters.");
}
if (limit !== undefined && (!Number.isInteger(limit) || limit <= 0)) {
throw new Error("--limit must be a positive integer.");
}
async function readText(location) {
if (/^https?:\/\//i.test(location)) {
const response = await fetch(location, {
headers: {
"User-Agent": "JavaJub-IndexNow/1.0",
},
});
if (!response.ok) {
throw new Error(`Failed to fetch ${location}: ${response.status} ${response.statusText}`);
}
return response.text();
}
return fs.readFile(path.resolve(location), "utf8");
}
function decodeXml(value) {
return value
.replaceAll("&", "&")
.replaceAll("<", "<")
.replaceAll(">", ">")
.replaceAll(""", '"')
.replaceAll("'", "'");
}
function extractUrls(sitemapXml) {
const urls = [...sitemapXml.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)]
.map((match) => decodeXml(match[1].trim()))
.filter((url) => {
try {
const parsedUrl = new URL(url);
return parsedUrl.host === host && !EXCLUDED_PATHS.has(parsedUrl.pathname);
} catch {
return false;
}
});
return [...new Set(urls)].slice(0, limit);
}
async function submit(urlList) {
const payload = {
host,
key,
keyLocation,
urlList,
};
if (values["dry-run"]) {
console.log(`IndexNow dry-run: ${urlList.length} URL(s) would be sent to ${values.endpoint}`);
console.log(`Key location: ${keyLocation}`);
console.log(urlList.slice(0, 20).join("\n"));
if (urlList.length > 20) console.log(`...and ${urlList.length - 20} more`);
return;
}
const response = await fetch(values.endpoint, {
method: "POST",
headers: {
"Content-Type": "application/json; charset=utf-8",
"User-Agent": "JavaJub-IndexNow/1.0",
},
body: JSON.stringify(payload),
});
const body = await response.text();
if (!response.ok) {
throw new Error(`IndexNow request failed: ${response.status} ${response.statusText}\n${body}`);
}
console.log(`IndexNow submitted ${urlList.length} URL(s) to ${values.endpoint}`);
if (body.trim()) console.log(body.trim());
}
const sitemapXml = await readText(sitemapLocation);
const urls = extractUrls(sitemapXml);
if (urls.length === 0) {
throw new Error(`No ${host} URLs found in ${sitemapLocation}`);
}
await submit(urls);