More error handling

cicalese · cicalese · commit ced8d22fb61a · 2024-01-16T12:32:39.000Z
diff --git a/scripts/lib/scraper.py b/scripts/lib/scraper.py
@@ -92,10 +92,16 @@ def scrape_site(url, page_id, args, session):
 
 	general = query['general']
 	mw_version = re.sub('^MediaWiki ', '', general['generator'])
-	db_version = general['dbtype'] + ': ' + general['dbversion']
-	php_version = general['phpversion'] + '(' + general['phpsapi'] + ')'
+	if 'dbtype' in general and 'dbversion' in general:
+		db_version = general['dbtype'] + ': ' + general['dbversion']
+	else:
+		db_version = ""
+	if 'phpversion' in general and 'phpsapi' in general:
+		php_version = general['phpversion'] + '(' + general['phpsapi'] + ')'
+	else:
+		php_version = ""
 	language = general['lang']
-	if 'logo' in general:
+	if 'logo' in general and len(general['logo']) < 256:
 		logo = general['logo']
 	else:
 		logo = ''
@@ -110,7 +116,10 @@ def scrape_site(url, page_id, args, session):
 		extensions = query['extensions']
 		versions = create_version_records(extensions)
 	else:
-		versions = None
+		versions = {
+			'skins': [],
+			'extensions': []
+		}
 
 	scrape = ScrapeRecord(
 		w8y_sr_page_id=page_id,
diff --git a/scripts/scrape.py b/scripts/scrape.py
@@ -1,6 +1,6 @@
 from argparse import ArgumentParser
 from sqlalchemy.orm import Session
-from sqlalchemy import select
+from sqlalchemy import select, and_
 import sys
 import time
 sys.path.append('./lib')
@@ -10,13 +10,26 @@
 
 
 def get_args():
-	parser = ArgumentParser(prog='Create', description='creates pages in wiki corresponding to URLs in file')
+	parser = ArgumentParser(prog='Create', description='scrapes the data for rows in Wikis and stores the data in the database')
+	parser.add_argument("-n", "--new", action="store_true", help="scrape only new pages (those that have not been scraped before)")
 	parser.add_argument("-v", "--verbose", action="count", default=0, help="increase output verbosity")
 	return parser.parse_args()
 
 
-def get_wikis(session):
-	stmt = select(Wiki).where(Wiki.w8y_wi_is_defunct == False)
+def get_wikis(session, new_wikis):
+	if new_wikis:
+		stmt = select(Wiki).where(
+			and_(
+				Wiki.w8y_wi_last_sr_id.is_not(None),
+				Wiki.w8y_wi_is_defunct == False
+			)
+		)
+	else:
+		stmt = select(Wiki).where(
+			Wiki.w8y_wi_is_defunct == False
+		).order_by(
+			Wiki.w8y_wi_last_sr_id
+		)
 	return session.scalars(stmt)
 
 
@@ -27,7 +40,7 @@ def run():
 	error_count = 0
 	with Session(engine) as session:
 		try:
-			wikis = get_wikis(session)
+			wikis = get_wikis(session, args.new)
 			message = 'Starting scraping wikis.'
 			log_message(session, message)
 			if args.verbose:
diff --git a/scripts/sync_pages_to_db.py b/scripts/sync_pages_to_db.py
@@ -11,7 +11,7 @@
 
 
 def get_args():
-	parser = ArgumentParser(prog='Create', description='creates pages in wiki corresponding to URLs in file')
+	parser = ArgumentParser(prog='Create', description='creates Wiki records in the database corresponding to the pages that were created in the wiki')
 	parser.add_argument("-v", "--verbose", action="count", default=0, help="increase output verbosity")
 	return parser.parse_args()
 
