Merge branch 'main' of https://github.com/cicalese/NewWikiApiary into main

Designburo · Designburo · commit b835b6cb62bc · 2024-01-15T20:42:04.000+01:00
diff --git a/scripts/create_pages_from_file.py b/scripts/create_pages_from_file.py
@@ -1,6 +1,7 @@
 from argparse import ArgumentParser
 import requests
 import pywikibot
+from pywikibot.exceptions import InvalidTitleError, PageRelatedError
 from sqlalchemy.orm import Session
 from sqlalchemy import select
 import time
@@ -77,31 +78,38 @@ def run():
 					url = line.strip()
 					sitename = get_sitename(url, args, errors, session)
 					if sitename:
-						log_message(
-							session,
-							f'Creating/updating page for site {sitename} with URL {url}'
-						)
-						page = pywikibot.Page(site, sitename)
-						if page.exists():
-							comment = 'Updated page'
-						else:
-							comment = 'Created page'
-						page.text = f'{{{{Wiki|url={url}}}}}'
-						page.save(comment)
-						stmt = select(Wiki).where(Wiki.w8y_wi_page_id == page.pageid)
-						wiki = session.scalars(stmt).one_or_none()
-						if wiki:
-							wiki.w8y_wi_api_url = bytes(url, 'utf8')
-							wiki.w8y_wi_last_sr_id = None
-						else:
-							wiki = Wiki(
-								w8y_wi_page_id=page.pageid,
-								w8y_wi_api_url=bytes(url, 'utf8'),
-								w8y_wi_last_sr_id=None,
-								w8y_wi_is_defunct=False
+						try:
+							log_message(
+								session,
+								f'Creating/updating page for site {sitename} with URL {url}'
 							)
-							session.add(wiki)
-						session.commit()
+							page = pywikibot.Page(site, 'Wiki:' + sitename)
+							if page.exists():
+								comment = 'Updated page'
+							else:
+								comment = 'Created page'
+							page.text = f'{{{{Wiki|url={url}}}}}'
+							page.save(comment)
+							stmt = select(Wiki).where(Wiki.w8y_wi_page_id == page.pageid)
+							wiki = session.scalars(stmt).one_or_none()
+							if wiki:
+								wiki.w8y_wi_api_url = bytes(url, 'utf8')
+								wiki.w8y_wi_last_sr_id = None
+							else:
+								wiki = Wiki(
+									w8y_wi_page_id=page.pageid,
+									w8y_wi_api_url=bytes(url, 'utf8'),
+									w8y_wi_last_sr_id=None,
+									w8y_wi_is_defunct=False
+								)
+								session.add(wiki)
+							session.commit()
+						except (InvalidTitleError, PageRelatedError):
+							message = f'Error from {url} with {sitename}'
+							log_message(session, message)
+							if args.verbose:
+								print(message)
+							errors.append(url)
 		except KeyboardInterrupt:
 			pass
 		finally:
diff --git a/scripts/lib/models.py b/scripts/lib/models.py
@@ -1,15 +1,17 @@
 import os
+from sqlalchemy import engine as sa_engine
 from sqlalchemy import create_engine
 from sqlalchemy.ext.declarative import declarative_base
 
-username = os.environ['WIKIAPIARY_DB_USERNAME']
-password = os.environ['WIKIAPIARY_DB_PASSWORD']
-host = os.environ['WIKIAPIARY_DB_HOST']
-port = os.environ['WIKIAPIARY_DB_PORT']
-db_schema = os.environ['WIKIAPIARY_DB_SCHEMA']
-engine = create_engine(
-	f'mysql+pymysql://{username}:{password}@{host}:{port}/{db_schema}'
+url = sa_engine.URL.create(
+	drivername = "mysql+pymysql",
+	username = os.environ['WIKIAPIARY_DB_USERNAME'],
+	password = os.environ['WIKIAPIARY_DB_PASSWORD'],
+	host = os.environ['WIKIAPIARY_DB_HOST'],
+	port = os.environ['WIKIAPIARY_DB_PORT'],
+	database = os.environ['WIKIAPIARY_DB_SCHEMA']
 )
+engine = create_engine(url)
 
 Base = declarative_base()
 Base.metadata.reflect(engine)
