Repository navigation
Expand file tree
/
Copy pathstopwords.py
More file actions
148 lines (129 loc) · 2.42 KB
/
Copy pathstopwords.py
File metadata and controls
148 lines (129 loc) · 2.42 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
# -*- coding: utf-8 -*-
#
# Copyright (c) nexB Inc. and others. All rights reserved.
# ScanCode is a trademark of nexB Inc.
# SPDX-License-Identifier: Apache-2.0
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
# See https://github.com/nexB/scancode-toolkit for support or download.
# See https://aboutcode.org for more information about nexB OSS projects.
#
"""
A set of common words that are ignored from matching such as HTML tags.
"""
STOPWORDS = frozenset({
# common XML character references as "
'amp',
'apos',
'gt',
'lt',
'nbsp',
'quot',
# common html tags as <a href=https://link ...> dfsdfsdf</a>
'a',
'abbr',
'alt',
'blockquote',
'body',
'br',
'class',
'div',
'em',
'h1',
'h2',
'h3',
'h4',
'h5',
'hr',
'href',
'img',
'li',
'ol',
'p',
'pre',
'rel',
'script',
'span',
'src',
'td',
'th',
'tr',
'ul',
# comment line markers
# batch files
'rem',
# autotools
'dnl',
# doc book tags as <para>
'para',
'ulink',
# Some HTML punctuations and entities all as &emdash;
'bdquo',
'bull',
'bullet',
'colon',
'comma',
'emdash', # rare and weird
'emsp',
'ensp',
'ge',
'hairsp',
'ldquo',
'ldquor',
'le',
'lpar',
'lsaquo',
'lsquo',
'lsquor',
'mdash',
'ndash',
'numsp',
'period',
'puncsp',
'raquo',
'rdquo',
'rdquor',
'rpar',
'rsaquo',
'rsquo',
'rsquor',
'sbquo',
'semi',
'thinsp',
'tilde',
# some xml char entities
'x3c',
'x3e',
# seen in many CSS
'lists',
'side', 'nav',
'height',
'auto',
'border',
'padding',
'width',
# seen in Perl PODs
'head1',
'head2',
'head3',
# seen in RTF markup
# this may be a solution to https://github.com/nexB/scancode-toolkit/issues/1548
# 'par',
# common in C literals
'printf',
# common in shell
'echo',
})
# query breaks
# in Android generated NOTICE.html files
breaks = {
'<pre class="license-text">',
'</pre><!-- license-text -->',
'-------------------------------------------------------------------',
'==============================================================================',
}
# TODO: use me?
# translations of HTML entities to words or letters
{
'é': 'e',
'©': '(c)',
}