Skip to content

Commit 68eac2b

Browse files
freddrakefreddrake
committed
Added reasonable parsing of the DOCTYPE declaration, fixed edge cases
regarding bare ampersands in content.
1 parent 212a2e1 commit 68eac2b

1 file changed

Lines changed: 260 additions & 12 deletions

File tree

‎Lib/HTMLParser.py‎

Lines changed: 260 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,7 @@
1515

1616
interesting_normal = re.compile('[&<]')
1717
interesting_cdata = re.compile(r'<(/|\Z)')
18-
incomplete = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*'
19-
'|#([0-9]*|[xX][0-9a-fA-F]*))?')
18+
incomplete = re.compile('&[a-zA-Z#]')
2019

2120
entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]')
2221
charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]')
@@ -185,16 +184,18 @@ def goahead(self, end):
185184
k = self.parse_pi(i)
186185
elif declopen.match(rawdata, i): # <!
187186
k = self.parse_declaration(i)
188-
else:
187+
elif (i + 1) < n:
189188
self.handle_data("<")
190189
k = i + 1
190+
else:
191+
break
191192
if k < 0:
192193
if end:
193194
raise HTMLParseError("EOF in middle of construct",
194195
self.getpos())
195196
break
196197
i = self.updatepos(i, k)
197-
elif rawdata[i] == '&':
198+
elif rawdata[i:i+2] == "&#":
198199
match = charref.match(rawdata, i)
199200
if match:
200201
name = match.group()[2:-1]
@@ -204,6 +205,9 @@ def goahead(self, end):
204205
k = k - 1
205206
i = self.updatepos(i, k)
206207
continue
208+
else:
209+
break
210+
elif rawdata[i] == '&':
207211
match = entityref.match(rawdata, i)
208212
if match:
209213
name = match.group(1)
@@ -215,14 +219,21 @@ def goahead(self, end):
215219
continue
216220
match = incomplete.match(rawdata, i)
217221
if match:
222+
# match.group() will contain at least 2 chars
218223
rest = rawdata[i:]
219-
if end and rest != "&" and match.group() == rest:
224+
if end and match.group() == rest:
220225
raise HTMLParseError(
221226
"EOF in middle of entity or char ref",
222227
self.getpos())
223-
return -1 # incomplete
224-
self.handle_data("&")
225-
i = self.updatepos(i, i + 1)
228+
# incomplete
229+
break
230+
elif (i + 1) < n:
231+
# not the end of the buffer, and can't be confused
232+
# with some other construct
233+
self.handle_data("&")
234+
i = self.updatepos(i, i + 1)
235+
else:
236+
break
226237
else:
227238
assert 0, "interesting.search() lied"
228239
# end while
@@ -232,14 +243,15 @@ def goahead(self, end):
232243
self.rawdata = rawdata[i:]
233244

234245
# Internal -- parse comment, return end or -1 if not terminated
235-
def parse_comment(self, i):
246+
def parse_comment(self, i, report=1):
236247
rawdata = self.rawdata
237248
assert rawdata[i:i+4] == '<!--', 'unexpected call to parse_comment()'
238249
match = commentclose.search(rawdata, i+4)
239250
if not match:
240251
return -1
241-
j = match.start()
242-
self.handle_comment(rawdata[i+4: j])
252+
if report:
253+
j = match.start()
254+
self.handle_comment(rawdata[i+4: j])
243255
j = match.end()
244256
return j
245257

@@ -257,11 +269,17 @@ def parse_declaration(self, i):
257269
return -1
258270
# in practice, this should look like: ((name|stringlit) S*)+ '>'
259271
n = len(rawdata)
272+
decltype = None
273+
extrachars = ""
260274
while j < n:
261275
c = rawdata[j]
262276
if c == ">":
263277
# end of declaration syntax
264-
self.handle_decl(rawdata[i+2:j])
278+
data = rawdata[i+2:j]
279+
if decltype == "doctype":
280+
self.handle_decl(data)
281+
else:
282+
self.unknown_decl(data)
265283
return j + 1
266284
if c in "\"'":
267285
m = declstringlit.match(rawdata, j)
@@ -273,12 +291,242 @@ def parse_declaration(self, i):
273291
if not m:
274292
return -1 # incomplete
275293
j = m.end()
294+
if decltype is None:
295+
decltype = m.group(0).rstrip().lower()
296+
if decltype != "doctype":
297+
extrachars = "="
298+
elif c == "[" and decltype == "doctype":
299+
j = self.parse_doctype_subset(j + 1, i)
300+
if j < 0:
301+
return j
302+
elif c in extrachars:
303+
j = j + 1
304+
while j < n and rawdata[j] in string.whitespace:
305+
j = j + 1
306+
if j == n:
307+
# end of buffer while in declaration
308+
return -1
276309
else:
277310
raise HTMLParseError(
278311
"unexpected char in declaration: %s" % `rawdata[j]`,
279312
self.getpos())
313+
decltype = decltype or ''
280314
return -1 # incomplete
281315

316+
# Internal -- scan past the internal subset in a <!DOCTYPE declaration,
317+
# returning the index just past any whitespace following the trailing ']'.
318+
def parse_doctype_subset(self, i, declstartpos):
319+
rawdata = self.rawdata
320+
n = len(rawdata)
321+
j = i
322+
while j < n:
323+
c = rawdata[j]
324+
if c == "<":
325+
s = rawdata[j:j+2]
326+
if s == "<":
327+
# end of buffer; incomplete
328+
return -1
329+
if s != "<!":
330+
self.updatepos(declstartpos, j + 1)
331+
raise HTMLParseError("unexpect char in internal subset",
332+
self.getpos())
333+
if (j + 2) == n:
334+
# end of buffer; incomplete
335+
return -1
336+
if (j + 4) > n:
337+
# end of buffer; incomplete
338+
return -1
339+
if rawdata[j:j+4] == "<!--":
340+
j = self.parse_comment(j, report=0)
341+
if j < 0:
342+
return j
343+
continue
344+
name, j = self.scan_name(j + 2, declstartpos)
345+
if j == -1:
346+
return -1
347+
if name not in ("attlist", "element", "entity", "notation"):
348+
self.updatepos(declstartpos, j + 2)
349+
raise HTMLParseError(
350+
"unknown declaration %s in internal subset" % `name`,
351+
self.getpos())
352+
# handle the individual names
353+
meth = getattr(self, "parse_doctype_" + name)
354+
j = meth(j, declstartpos)
355+
if j < 0:
356+
return j
357+
elif c == "%":
358+
# parameter entity reference
359+
if (j + 1) == n:
360+
# end of buffer; incomplete
361+
return -1
362+
m = declname.match(rawdata, j + 1)
363+
s = m.group()
364+
if s == rawdata[j+1:]:
365+
return -1
366+
j = j + 1 + len(s.rstrip())
367+
if rawdata[j] == ";":
368+
j = j + 1
369+
elif c == "]":
370+
j = j + 1
371+
while j < n and rawdata[j] in string.whitespace:
372+
j = j + 1
373+
if j < n:
374+
if rawdata[j] == ">":
375+
return j
376+
self.updatepos(declstartpos, j)
377+
raise HTMLParseError(
378+
"unexpected char after internal subset",
379+
self.getpos())
380+
else:
381+
return -1
382+
elif c in string.whitespace:
383+
j = j + 1
384+
else:
385+
self.updatepos(declstartpos, j)
386+
raise HTMLParseError("unexpected char in internal subset",
387+
self.getpos())
388+
# end of buffer reached
389+
return -1
390+
391+
def parse_doctype_element(self, i, declstartpos):
392+
rawdata = self.rawdata
393+
n = len(rawdata)
394+
name, j = self.scan_name(i, declstartpos)
395+
if j == -1:
396+
return -1
397+
# style content model; just skip until '>'
398+
if '>' in rawdata[j:]:
399+
return string.find(rawdata, ">", j) + 1
400+
return -1
401+
402+
def parse_doctype_attlist(self, i, declstartpos):
403+
rawdata = self.rawdata
404+
name, j = self.scan_name(i, declstartpos)
405+
c = rawdata[j:j+1]
406+
if c == "":
407+
return -1
408+
if c == ">":
409+
return j + 1
410+
while 1:
411+
# scan a series of attribute descriptions; simplified:
412+
# name type [value] [#constraint]
413+
name, j = self.scan_name(j, declstartpos)
414+
if j < 0:
415+
return j
416+
c = rawdata[j:j+1]
417+
if c == "":
418+
return -1
419+
if c == "(":
420+
# an enumerated type; look for ')'
421+
if ")" in rawdata[j:]:
422+
j = string.find(rawdata, ")", j) + 1
423+
else:
424+
return -1
425+
while rawdata[j:j+1] in string.whitespace:
426+
j = j + 1
427+
if not rawdata[j:]:
428+
# end of buffer, incomplete
429+
return -1
430+
else:
431+
name, j = self.scan_name(j, declstartpos)
432+
c = rawdata[j:j+1]
433+
if not c:
434+
return -1
435+
if c in "'\"":
436+
m = declstringlit.match(rawdata, j)
437+
if m:
438+
j = m.end()
439+
else:
440+
return -1
441+
c = rawdata[j:j+1]
442+
if not c:
443+
return -1
444+
if c == "#":
445+
if rawdata[j:] == "#":
446+
# end of buffer
447+
return -1
448+
name, j = self.scan_name(j + 1, declstartpos)
449+
if j < 0:
450+
return j
451+
c = rawdata[j:j+1]
452+
if not c:
453+
return -1
454+
if c == '>':
455+
# all done
456+
return j + 1
457+
458+
def parse_doctype_notation(self, i, declstartpos):
459+
name, j = self.scan_name(i, declstartpos)
460+
if j < 0:
461+
return j
462+
rawdata = self.rawdata
463+
while 1:
464+
c = rawdata[j:j+1]
465+
if not c:
466+
# end of buffer; incomplete
467+
return -1
468+
if c == '>':
469+
return j + 1
470+
if c in "'\"":
471+
m = declstringlit.match(rawdata, j)
472+
if not m:
473+
return -1
474+
j = m.end()
475+
else:
476+
name, j = self.scan_name(j, declstartpos)
477+
if j < 0:
478+
return j
479+
480+
def parse_doctype_entity(self, i, declstartpos):
481+
rawdata = self.rawdata
482+
if rawdata[i:i+1] == "%":
483+
j = i + 1
484+
while 1:
485+
c = rawdata[j:j+1]
486+
if not c:
487+
return -1
488+
if c in string.whitespace:
489+
j = j + 1
490+
else:
491+
break
492+
else:
493+
j = i
494+
name, j = self.scan_name(j, declstartpos)
495+
if j < 0:
496+
return j
497+
while 1:
498+
c = self.rawdata[j:j+1]
499+
if not c:
500+
return -1
501+
if c in "'\"":
502+
m = declstringlit.match(rawdata, j)
503+
if m:
504+
j = m.end()
505+
else:
506+
return -1 # incomplete
507+
elif c == ">":
508+
return j + 1
509+
else:
510+
name, j = self.scan_name(j, declstartpos)
511+
if j < 0:
512+
return j
513+
514+
def scan_name(self, i, declstartpos):
515+
rawdata = self.rawdata
516+
n = len(rawdata)
517+
if i == n:
518+
return None, -1
519+
m = declname.match(rawdata, i)
520+
if m:
521+
s = m.group()
522+
name = s.strip()
523+
if (i + len(s)) == n:
524+
return None, -1 # end of buffer
525+
return name.lower(), m.end()
526+
else:
527+
self.updatepos(declstartpos, i)
528+
raise HTMLParseError("expected name token", self.getpos())
529+
282530
# Internal -- parse processing instr, return end or -1 if not terminated
283531
def parse_pi(self, i):
284532
rawdata = self.rawdata

0 commit comments

Comments
 (0)