|
| 1 | +module Docs |
| 2 | + class Mariadb |
| 3 | + class CleanHtmlFilter < Filter |
| 4 | + def call |
| 5 | + # Return the empty doc if the EraseInvalidPagesFilter detected this page shouldn't be scraped |
| 6 | + return doc if doc.inner_html == '' |
| 7 | + |
| 8 | + # Extract main content |
| 9 | + @doc = at_css('#content') |
| 10 | + |
| 11 | + # Remove navigation at the bottom |
| 12 | + css('.simple_section_nav').remove |
| 13 | + |
| 14 | + # Remove table of contents |
| 15 | + css('.table_of_contents').remove |
| 16 | + |
| 17 | + # Add code highlighting and remove nested tags |
| 18 | + css('pre').each do |node| |
| 19 | + node.content = node.content |
| 20 | + node['data-language'] = 'sql' |
| 21 | + end |
| 22 | + |
| 23 | + # Fix images |
| 24 | + css('img').each do |node| |
| 25 | + node['src'] = node['src'].sub('http:', 'https:') |
| 26 | + end |
| 27 | + |
| 28 | + # Remove navigation items containing only numbers |
| 29 | + css('.node_comments').each do |node| |
| 30 | + if node.content.scan(/\D/).empty? |
| 31 | + node.remove |
| 32 | + end |
| 33 | + end |
| 34 | + |
| 35 | + # Convert listings (pages like https://mariadb.com/kb/en/library/documentation/sql-statements-structure/) into tables |
| 36 | + css('ul.listing').each do |node| |
| 37 | + rows = [] |
| 38 | + |
| 39 | + node.css('li:not(.no_data)').each do |li| |
| 40 | + name = li.at_css('.media-heading').content |
| 41 | + description = li.at_css('.blurb').content |
| 42 | + url = li.at_css('a')['href'] |
| 43 | + rows << "<tr><td><a href=\"#{url}\">#{name}</a></td><td>#{description}</td></tr>" |
| 44 | + end |
| 45 | + |
| 46 | + table = "<table><thead><tr><th>Title</th><th>Description</th></tr></thead><tbody>#{rows.join('')}</tbody></table>" |
| 47 | + node.replace(table) |
| 48 | + end |
| 49 | + |
| 50 | + # Turn note titles into <strong> tags |
| 51 | + css('.product_title').each do |node| |
| 52 | + node.name = 'strong' |
| 53 | + end |
| 54 | + |
| 55 | + # Remove comments and questions |
| 56 | + css('.related_questions, #comments').remove |
| 57 | + css('h2').each do |node| |
| 58 | + if node.content == 'Comments' |
| 59 | + node.remove |
| 60 | + end |
| 61 | + end |
| 62 | + |
| 63 | + doc |
| 64 | + end |
| 65 | + end |
| 66 | + end |
| 67 | +end |
0 commit comments