forked from darkreader/darkreader
-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathparse.ts
More file actions
389 lines (336 loc) · 14.5 KB
/
Copy pathparse.ts
File metadata and controls
389 lines (336 loc) · 14.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
import {isFullyQualifiedDomain, isFullyQualifiedDomainWildcard, fullyQualifiedDomainMatchesWildcard, isURLInList, isURLMatched} from '../../utils/url';
import {parseArray} from '../../utils/text';
declare const __TEST__: boolean;
const INDEX_CACHE_CLEANUP_INTERVAL_IN_MS = 60000;
// TODO: remove cast once types are updated
declare function clearTimeout(id: ReturnType<typeof setTimeout> | null | undefined): void;
interface SitePropsMut {
url: readonly string[];
}
type SiteProps = Readonly<SitePropsMut>;
export interface SitePropsIndex<SiteFix extends SiteProps> {
offsets: Readonly<string>;
domains: Readonly<{[domain: string]: readonly number[]}>;
domainLabels: Readonly<{[domainLabel: string]: readonly number[]}>;
nonstandard: readonly number[];
cacheSiteFix: {[offsetId: number]: Readonly<SiteFix>};
cacheDomainIndex: {[domain: string]: readonly number[]};
cacheCleanupTimer: ReturnType<typeof setTimeout> | null;
}
interface ConfigIndex {
domains: Readonly<{[domain: string]: readonly number[]}>;
domainLabels: Readonly<{[domainLabel: string]: readonly number[]}>;
nonstandard: Readonly<number[] | null>;
}
export interface SiteListIndex {
urls: readonly string[];
domains: Readonly<{[domain: string]: number[]}>;
domainLabels: Readonly<{[domainLabel: string]: readonly number[]}>;
nonstandard: readonly number[];
}
export interface SitesFixesParserOptions<T> {
commands: readonly string[];
getCommandPropName: (command: string) => keyof T;
parseCommandValue: (command: string, value: string) => any;
}
export function parseSitesFixesConfig<T extends SiteProps>(text: string, options: SitesFixesParserOptions<T>): T[] {
const sites: T[] = [];
const blocks = text.replace(/\r/g, '').split(/^\s*={2,}\s*$/gm);
blocks.forEach((block) => {
const lines = block.split('\n');
const commandIndices: number[] = [];
lines.forEach((ln, i) => {
if (ln.match(/^[A-Z]+(\s[A-Z]+){0,2}$/)) {
commandIndices.push(i);
}
});
if (commandIndices.length === 0) {
return;
}
const siteFix = {
url: parseArray(lines.slice(0, commandIndices[0]).join('\n')) as readonly string[],
} as T;
commandIndices.forEach((commandIndex, i) => {
const command = lines[commandIndex].trim();
const valueText = lines.slice(commandIndex + 1, i === commandIndices.length - 1 ? lines.length : commandIndices[i + 1]).join('\n');
const prop = options.getCommandPropName(command);
if (!prop) {
return;
}
const value = options.parseCommandValue(command, valueText);
siteFix[prop] = value;
});
sites.push(siteFix);
});
return sites;
}
// URL patterns are guaranteed to not have protocol and leading '/'
export function getDomain(url: string): string {
try {
return (new URL(url)).hostname.toLowerCase();
} catch (error) {
return url.split('/')[0].toLowerCase();
}
}
/*
* Encode all offsets into a string, where each record is 7 bytes long:
* - 4 bytes for start offset
* - 3 bytes for record length (end offset - start offset)
* Both values are stored in base 36 (radix 36) notation.
* Maximum supported numbers:
* - start offset must be no more than parseInt('zzzz', 36) = 1679615
* - length must be no more than parseInt('zzz', 36) = 46655
*
* We have to encode offsets into a string to be able to save them in
* chrome.storage.local for use in non-persistent background contexts.
*/
function encodeOffsets(offsets: Array<[number, number]>): string {
return offsets.map(([offset, length]) => {
const stringOffset = offset.toString(36);
const stringLength = length.toString(36);
return '0'.repeat(4 - stringOffset.length) + stringOffset + '0'.repeat(3 - stringLength.length) + stringLength;
}).join('');
}
function decodeOffset(offsets: string, index: number): [number, number] {
const base = (4 + 3) * index;
const offset = parseInt(offsets.substring(base + 0, base + 4), 36);
const length = parseInt(offsets.substring(base + 4, base + 4 + 3), 36);
return [
offset,
offset + length,
];
}
function addLabel(set: { [label: string]: number[] }, label: string, index: number) {
if (!set[label]) {
set[label] = [index];
} else if (!(set[label].includes(index))) {
set[label].push(index);
}
}
function extractDomainLabelsFromFullyQualifiedDomainWildcard(fullyQualifiedDomainWildcard: string): string[] {
const postfixStart = fullyQualifiedDomainWildcard.lastIndexOf('*');
const postfix = fullyQualifiedDomainWildcard.substring(postfixStart + 2);
if (postfixStart < 0 || postfix.length === 0) {
return fullyQualifiedDomainWildcard.split('.');
}
const labels = [postfix];
const prefix = fullyQualifiedDomainWildcard.substring(0, postfixStart);
prefix.split('.').filter(Boolean).forEach((l) => labels.concat(l));
return labels;
}
function indexConfigURLs(urls: string[][]): {domains: { [domain: string]: number[] }; domainLabels: { [domainLabel: string]: number[] }; nonstandard: number[]} {
const domains: { [domain: string]: number[] } = {};
const domainLabels: { [domainLabel: string]: number[] } = {};
const nonstandard: number[] = [];
const domainLabelFrequencies: { [domainLabel: string]: number } = {};
const domainLabelMembers: Array<{ labels: string[]; index: number }> = [];
for (let index = 0; index < urls.length; index++) {
const block = urls[index];
const blockDomainLabels = new Set<string>();
for (const url of block) {
const domain = getDomain(url);
if (isFullyQualifiedDomain(domain)) {
addLabel(domains, domain, index);
} else if (isFullyQualifiedDomainWildcard(domain)) {
const labels = extractDomainLabelsFromFullyQualifiedDomainWildcard(domain);
domainLabelMembers.push({labels, index});
labels.forEach((l) => blockDomainLabels.add(l));
} else {
// Sitefix parser encountered non-standard URL
nonstandard.push(index);
break;
}
}
// Compute domain label frequencies, counting each label within each fix only once
for (const label of blockDomainLabels) {
if (domainLabelFrequencies[label]) {
domainLabelFrequencies[label]++;
} else {
domainLabelFrequencies[label] = 1;
}
}
}
// For each domain name, find the most specific label
for (const {labels, index} of domainLabelMembers) {
let label = labels[0];
for (const currLabel of labels) {
if (domainLabelFrequencies[currLabel] < domainLabelFrequencies[label]) {
label = currLabel;
}
}
addLabel(domainLabels, label, index);
}
return {domains, domainLabels, nonstandard};
}
function processSiteFixesConfigBlock(text: string, offsets: Array<[number, number]>, recordStart: number, recordEnd: number, urls: Array<readonly string[]>) {
// TODO: more formal definition of URLs and delimiters
const block = text.substring(recordStart, recordEnd);
const lines = block.split('\n');
const commandIndices: number[] = [];
lines.forEach((ln, i) => {
if (ln.match(/^[A-Z]+(\s[A-Z]+){0,2}$/)) {
commandIndices.push(i);
}
});
if (commandIndices.length === 0) {
return;
}
offsets.push([recordStart, recordEnd - recordStart]);
const urls_ = parseArray(lines.slice(0, commandIndices[0]).join('\n'));
urls.push(urls_);
}
function extractURLsFromSiteFixesConfig(text: string): {urls: string[][]; offsets: Array<[number, number]>} {
const urls: string[][] = [];
// Array of tuples, where first number is an offset of record start and second number is record length.
const offsets: Array<[number, number]> = [];
let recordStart = 0;
// Delimiter between two blocks
const delimiterRegex = /^\s*={2,}\s*$/gm;
let delimiter: RegExpMatchArray | null;
while ((delimiter = delimiterRegex.exec(text))) {
const nextDelimiterStart = delimiter.index!;
const nextDelimiterEnd = delimiter.index! + delimiter[0].length;
processSiteFixesConfigBlock(text, offsets, recordStart, nextDelimiterStart, urls);
recordStart = nextDelimiterEnd;
}
processSiteFixesConfigBlock(text, offsets, recordStart, text.length, urls);
return {urls, offsets};
}
export function indexSitesFixesConfig<T extends SiteProps>(text: string): SitePropsIndex<T> {
const {urls, offsets} = extractURLsFromSiteFixesConfig(text);
const {domains, domainLabels, nonstandard} = indexConfigURLs(urls);
return {offsets: encodeOffsets(offsets), domains, domainLabels, nonstandard, cacheDomainIndex: {}, cacheSiteFix: {}, cacheCleanupTimer: null};
}
function lookupConfigURLsInDomainLabels(domain: string, recordIds: number[], currRecordIds: readonly number[], getAllRecordURLs: (id: number) => readonly string[]) {
for (const recordId of currRecordIds) {
const recordURLs = getAllRecordURLs(recordId);
for (const ruleUrl of recordURLs) {
const wildcard = getDomain(ruleUrl);
if (isFullyQualifiedDomainWildcard(wildcard) && fullyQualifiedDomainMatchesWildcard(wildcard, domain)) {
recordIds.push(recordId);
} else {
// Skip this rule, since the label match must have come from a different URL
}
}
}
}
function lookupConfigURLs(domain: string, index: ConfigIndex, getAllRecordURLs: (id: number) => readonly string[]): number[] {
const labels = domain.split('.');
let recordIds: number[] = [];
// Common fix
if (index.domainLabels.hasOwnProperty('*')) {
recordIds = recordIds.concat(index.domainLabels['*']);
}
// Wildcard fixes
for (const label of labels) {
// We need to use in operator because ids are 0-based and 0 is falsy
if (index.domainLabels.hasOwnProperty(label)) {
const currRecordIds = index.domainLabels[label];
lookupConfigURLsInDomainLabels(domain, recordIds, currRecordIds, getAllRecordURLs);
}
}
for (let i = 0; i < labels.length; i++) {
const substring = labels.slice(i).join('.');
if (index.domains.hasOwnProperty(substring)) {
recordIds = recordIds.concat(index.domains[substring]);
}
if (index.domainLabels.hasOwnProperty(substring)) {
const currRecordIds = index.domainLabels[substring];
lookupConfigURLsInDomainLabels(domain, recordIds, currRecordIds, getAllRecordURLs);
}
}
// Backwards compatibility: check for nonssend over nonstandard patterns, which will be filtered out
// via regex in content script
if (index.nonstandard) {
for (const currRecordId of index.nonstandard) {
const urls = getAllRecordURLs(currRecordId);
if (urls.some((url) => isURLMatched(domain, getDomain(url)))) {
recordIds.push(currRecordId);
continue;
}
}
}
// Deduplicate array elements
recordIds = Array.from(new Set(recordIds));
return recordIds;
}
/**
* Extracts a single site fix and parses it (cached)
* @param text the fix file
* @param index site fix index
* @param options fix parsing options
* @param id numeric index of the fix
* @returns a single fix
*/
function getSiteFix<T extends SiteProps>(text: string, index: SitePropsIndex<T>, options: SitesFixesParserOptions<T>, id: number): Readonly<T> {
if (index.cacheSiteFix.hasOwnProperty(id)) {
return index.cacheSiteFix[id];
}
const [blockStart, blockEnd] = decodeOffset(index.offsets, id);
const block = text.substring(blockStart, blockEnd);
const fix = parseSitesFixesConfig<T>(block, options)[0];
index.cacheSiteFix[id] = fix;
return fix;
}
/**
* This function uses setTimeout instead of Alarms API so that background context can
* go incative (resulting in cleanup of all context variables) and then not be awoken
* by the alarm.
* @param index
*/
function scheduleCacheCleanup<T extends SiteProps>(index: SitePropsIndex<T>) {
if (__TEST__) {
return;
}
clearTimeout(index.cacheCleanupTimer);
index.cacheCleanupTimer = setTimeout(() => {
index.cacheCleanupTimer = null;
index.cacheDomainIndex = {};
index.cacheSiteFix = {};
}, INDEX_CACHE_CLEANUP_INTERVAL_IN_MS);
}
/**
* Given a URL, raw fixes, and an index, finds the applicable fixes.
* Note that dependents assume that the first returned fix is a generic fix (has URL pattern '*').
*
* This method uses two levels of caching:
* - caching the site fixes keyed by a numeric id (to avoid re-parsing the site fixes)
* - caching the numeric ids keyed by domain (to avoid re-computing lists of site fixes for the same site,
* which is useful if user has multiple tabs of the same site and toggles Dark Reader on)
*/
export function getSitesFixesFor<T extends SiteProps>(url: string, text: string, index: SitePropsIndex<T>, options: SitesFixesParserOptions<T>): Array<Readonly<T>> {
const records: T[] = [];
const domain = getDomain(url);
if (!index.cacheDomainIndex[domain]) {
index.cacheDomainIndex[domain] = lookupConfigURLs(domain, index, (recordId) => getSiteFix<T>(text, index, options, recordId).url);
}
const recordIds = index.cacheDomainIndex[domain];
for (const recordId of recordIds) {
const fix = getSiteFix<T>(text, index, options, recordId);
records.push(fix);
}
scheduleCacheCleanup(index);
return records;
}
export function indexSiteListConfig(text: string): SiteListIndex {
const urls = parseArray(text);
const urls2D = urls.map((u) => [u]);
const {domains, domainLabels, nonstandard} = indexConfigURLs(urls2D);
return {domains, domainLabels, nonstandard, urls};
}
function getSiteListFor(url: string, index: SiteListIndex): string[] {
const domain = getDomain(url);
const recordIds = lookupConfigURLs(domain, index, (recordId) => [index.urls[recordId]]);
const result: string[] = [];
for (const recordId of recordIds) {
result.push(index.urls[recordId]);
}
return result;
}
export function isURLInSiteList(url: string, index: SiteListIndex | null): boolean {
if (index === null) {
return false;
}
const urls = getSiteListFor(url, index);
return isURLInList(url, urls);
}