Skip to content

Commit c8380bb

Browse files
committed
Merge pull request freeCodeCamp#841 from jmerle/mariadb
Add MariaDB documentation
2 parents fdb9701 + 5ed9942 commit c8380bb

10 files changed

Lines changed: 175 additions & 0 deletions

File tree

‎assets/javascripts/templates/pages/about_tmpl.coffee‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -451,6 +451,11 @@ credits = [
451451
'2006-2016 LÖVE Development Team',
452452
'GFDL',
453453
'http://www.gnu.org/copyleft/fdl.html'
454+
], [
455+
'MariaDB',
456+
'2019 MariaDB',
457+
'CC BY-SA & GFDL',
458+
'https://mariadb.com/kb/en/library/documentation/+license/'
454459
], [
455460
'Marionette.js',
456461
'2017 Muted Solutions, LLC',

‎assets/stylesheets/application.css.scss‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@
7070
'pages/liquid',
7171
'pages/love',
7272
'pages/lua',
73+
'pages/mariadb',
7374
'pages/mdn',
7475
'pages/meteor',
7576
'pages/mkdocs',
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
._mariadb {
2+
@extend %simple;
3+
4+
.graybox, .product {
5+
@extend %note;
6+
}
7+
}
Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
module Docs
2+
class Mariadb
3+
class CleanHtmlFilter < Filter
4+
def call
5+
# Return the empty doc if the EraseInvalidPagesFilter detected this page shouldn't be scraped
6+
return doc if doc.inner_html == ''
7+
8+
# Extract main content
9+
@doc = at_css('#content')
10+
11+
# Remove navigation at the bottom
12+
css('.simple_section_nav').remove
13+
14+
# Remove table of contents
15+
css('.table_of_contents').remove
16+
17+
# Add code highlighting and remove nested tags
18+
css('pre').each do |node|
19+
node.content = node.content
20+
node['data-language'] = 'sql'
21+
end
22+
23+
# Fix images
24+
css('img').each do |node|
25+
node['src'] = node['src'].sub('http:', 'https:')
26+
end
27+
28+
# Remove navigation items containing only numbers
29+
css('.node_comments').each do |node|
30+
if node.content.scan(/\D/).empty?
31+
node.remove
32+
end
33+
end
34+
35+
# Convert listings (pages like https://mariadb.com/kb/en/library/documentation/sql-statements-structure/) into tables
36+
css('ul.listing').each do |node|
37+
rows = []
38+
39+
node.css('li:not(.no_data)').each do |li|
40+
name = li.at_css('.media-heading').content
41+
description = li.at_css('.blurb').content
42+
url = li.at_css('a')['href']
43+
rows << "<tr><td><a href=\"#{url}\">#{name}</a></td><td>#{description}</td></tr>"
44+
end
45+
46+
table = "<table><thead><tr><th>Title</th><th>Description</th></tr></thead><tbody>#{rows.join('')}</tbody></table>"
47+
node.replace(table)
48+
end
49+
50+
# Turn note titles into <strong> tags
51+
css('.product_title').each do |node|
52+
node.name = 'strong'
53+
end
54+
55+
# Remove comments and questions
56+
css('.related_questions, #comments').remove
57+
css('h2').each do |node|
58+
if node.content == 'Comments'
59+
node.remove
60+
end
61+
end
62+
63+
doc
64+
end
65+
end
66+
end
67+
end
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
module Docs
2+
class Mariadb
3+
class EntriesFilter < Docs::EntriesFilter
4+
def get_name
5+
return 'Name' if doc.inner_html == ''
6+
7+
at_css('#content > h1').content.strip
8+
end
9+
10+
def get_type
11+
return 'Type' if doc.inner_html == ''
12+
13+
link = at_css('#breadcrumbs > a:nth-child(4)')
14+
link.nil? ? at_css('#breadcrumbs > a:nth-child(3)').content : link.content
15+
end
16+
17+
def entries
18+
# Don't add an entry for this page if the EraseInvalidPagesFilter detected this page shouldn't be scraped
19+
return [] if doc.inner_html == ''
20+
super
21+
end
22+
end
23+
end
24+
end
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
module Docs
2+
class Mariadb
3+
class EraseInvalidPagesFilter < Filter
4+
@@seen_urls = Hash.new
5+
6+
def call
7+
# The MariaDB documentation uses urls like mariadb.com/kb/en/*
8+
# This means there is no way to detect if a page should be scraped based on it's url
9+
# We run this filter before the internal_urls filter scrapes all internal urls
10+
# If this page should not be scraped, we erase it's contents in here so that the internal urls are not picked up
11+
# The entries filter will make sure that no entry is saved for this page
12+
13+
if at_css('a.crumb[href="https://mariadb.com/kb/en/documentation/"]').nil?
14+
doc.inner_html = ''
15+
end
16+
17+
current_page = at_css('a.crumb.node_link')
18+
unless current_page.nil?
19+
url = current_page['href']
20+
21+
# Some links lead to the same page
22+
# Only parse the page one time
23+
if @@seen_urls.has_key?(url)
24+
doc.inner_html = ''
25+
end
26+
27+
@@seen_urls[url] = true
28+
end
29+
30+
doc
31+
end
32+
end
33+
end
34+
end

‎lib/docs/scrapers/mariadb.rb‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
module Docs
2+
class Mariadb < UrlScraper
3+
self.name = 'MariaDB'
4+
self.type = 'mariadb'
5+
self.release = '10.4.8'
6+
self.base_url = 'https://mariadb.com/kb/en/'
7+
self.root_path = 'library/documentation/'
8+
self.links = {
9+
home: 'https://mariadb.com/',
10+
code: 'https://github.com/MariaDB/server'
11+
}
12+
13+
html_filters.insert_before 'internal_urls', 'mariadb/erase_invalid_pages'
14+
html_filters.push 'mariadb/entries', 'mariadb/clean_html'
15+
16+
options[:rate_limit] = 200
17+
options[:skip_patterns] = [
18+
/\+/,
19+
/\/ask\//,
20+
/-release-notes\//,
21+
/-changelog\//,
22+
/^documentation\//,
23+
/^mariadb-server-documentation\//,
24+
]
25+
26+
options[:attribution] = <<-HTML
27+
&copy; 2019 MariaDB<br>
28+
Licensed under the Creative Commons Attribution 3.0 Unported License and the GNU Free Documentation License.
29+
HTML
30+
31+
def get_latest_version(opts)
32+
doc = fetch_doc('https://mariadb.com/downloads/', opts)
33+
doc.at_css('[data-version-id="mariadb_server-versions"] option').content.split('-')[0]
34+
end
35+
end
36+
end

‎public/icons/docs/mariadb/16.png‎

936 Bytes
Loading
1.52 KB
Loading

‎public/icons/docs/mariadb/SOURCE‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
https://mariadb.org/about/logos/

0 commit comments

Comments
 (0)