Skip to content

Commit cb67124

Browse files
coderwassananmolcoderwassananmol
authored
Merge branch 'develop' into feat/integrate-bub2-wikimedia
2 parents 9211398 + 39e708b commit cb67124

File tree

22 files changed

+774
-259
lines changed

22 files changed

+774
-259
lines changed

‎.env.example‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@ redisport = -- redis-port --
88
WIKIMEDIA_CLIENT_ID = -- oauth client id --
99
WIKIMEDIA_CLIENT_SECRET = -- oauth client secret --
1010
NEXTAUTH_URL = -- nextauth base url --
11-
EMAIL_SOURCE_URL = -- the 'from' name that is displayed in the email --
1211
EMAIL_BOT_USERNAME = -- mediawiki bot username --
1312
EMAIL_BOT_PASSWORD = -- mediawiki bot password --
13+
NEXT_PUBLIC_WIKIMEDIA_URL = -- https://meta.wikimedia.beta.wmflabs.org for local setup --
14+
NEXT_PUBLIC_COMMONS_URL = -- https://commons.wikimedia.beta.wmflabs.org for local setup --
15+
NEXT_PUBLIC_IA_USER = -- bub_wikimedia --
16+
IA_EMAIL = -- IA email --

‎README.md‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
![](https://img.shields.io/github/repo-size/coderwassananmol/BUB) ![](https://img.shields.io/github/license/coderwassananmol/BUB2?color=red)<br>
66
![](https://img.shields.io/github/issues/coderwassananmol/BUB2?color=green) ![](https://img.shields.io/github/issues-pr/coderwassananmol/BUB2?color=green) ![](https://img.shields.io/github/downloads/coderwassananmol/BUB2/total) ![](https://img.shields.io/github/last-commit/coderwassananmol/BUB2) ![](https://img.shields.io/github/contributors/coderwassananmol/BUB2)<br>
7-
A book uploader bot that transfers documents from public libraries such as Google Books, Punjab Digital Library and Trove Digital Library to [Internet Archive](https://archive.org). Built for Wikimedia Tool Labs. Check out [BUB2 on Toolforge](https://bub2.toolforge.org)!
7+
A book uploader bot that transfers documents from public libraries such as Google Books, Punjab Digital Library and Trove Digital Library to [Internet Archive](https://archive.org). Built for Wikimedia Tool Labs. Check out [BUB2 on Wikimedia Cloud](https://bub2.wmcloud.org)!
88

99
# Table of Contents
1010

@@ -72,7 +72,10 @@ Rename `.env.example` to `.env`. Then, to fill the credentials,
7272
- Enter the client application key and client application secret in the `WIKIMEDIA_CLIENT_ID` and `WIKIMEDIA_CLIENT_SECRET` respectively
7373
- Enter the `NEXTAUTH_URL` with http://localhost:5000.
7474
- Go to [wikisource Bot] (https://meta.wikimedia.beta.wmflabs.org/wiki/Special:BotPasswords) to generate your `EMAIL_BOT_USERNAME` and `EMAIL_BOT_PASSWORD`. When creating your bot, tick 'send email to users' under the Applicable Grants section. Go ahead and enter the generated credentials in the `.env` file.
75-
- Enter the `EMAIL_SOURCE_URL` which is the name to be displayed as the sender in Emails - https://meta.wikimedia.beta.wmflabs.org/w/api.php
75+
- Enter the `NEXT_PUBLIC_WIKIMEDIA_URL` which is used to authenticate with Wikimedia environment for login, send emails etc. For example - https://meta.wikimedia.beta.wmflabs.org
76+
- Enter the `NEXT_PUBLIC_COMMONS_URL` which is used to upload the files to Commons. For example - https://meta.commons.beta.wmflabs.org for local setup.
77+
- Enter the `NEXT_PUBLIC_IA_USER` which is the username of Internet Archive account.
78+
- Enter the `IA_EMAIL` which is the email of the Internet Archive account.
7679
- Go to [Trove API](https://trove.nla.gov.au/about/create-something/using-api) and follow the instructions on how to get a trove key. Fill in `trove_key` with the trove key you generated.
7780
<a id="runRedisServer"></a>
7881
### Run Redis server

‎bull/commons-queue/consumer.js‎

Lines changed: 67 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -5,40 +5,84 @@ const {
55
downloadFile,
66
uploadToCommons,
77
uploadToWikiData,
8+
convertZipToPdf
89
} = require("../../utils/helper");
10+
const JSZip = require("jszip");
911
const logger = winston.loggers.get("defaultLogger");
1012

1113
CommonsQueue.process(async (job, done) => {
12-
const downloadFileRes = await downloadFile(
13-
job.data.metadata.uri || job.data.downloadFileURL,
14-
"commonsFilePayload.pdf"
15-
);
14+
if (job.data.type === "pdlZip") {
15+
const zipBuffer = Buffer.from(job.data.downloadFileURL, "base64");
16+
const zip = await JSZip.loadAsync(zipBuffer);
17+
const convertZipToPdfRes = await convertZipToPdf(
18+
zip,
19+
"commonsFilePayload.pdf"
20+
);
21+
if (convertZipToPdfRes.status !== 200) {
22+
logger.log({
23+
level: "error",
24+
message: `convertZipToPdfRes: ${JSON.stringify(convertZipToPdfRes)}`,
25+
});
26+
process.emit("commonsJobComplete", {
27+
status: false,
28+
value: null,
29+
});
30+
return done(null, true);
31+
}
32+
const commonsResponse = await uploadToCommons(job.data.metadata);
1633

17-
if (downloadFileRes.writeFileStatus !== 200) {
18-
logger.log({
19-
level: "error",
20-
message: `downloadFile: ${downloadFileRes}`,
21-
});
34+
if (commonsResponse.fileUploadStatus !== 200) {
35+
logger.log({
36+
level: "error",
37+
message: `uploadToCommons: ${commonsResponse}`,
38+
});
39+
process.emit(`commonsJobComplete:${job.id}`, {
40+
status: false,
41+
value: null,
42+
});
43+
return done(null, true);
44+
}
2245
process.emit(`commonsJobComplete:${job.id}`, {
23-
status: false,
24-
value: null,
46+
status: true,
47+
value: commonsResponse,
2548
});
2649
return done(null, true);
27-
// return done(new Error(`downloadFile: ${downloadFileRes}`));
28-
}
29-
const commonsResponse = await uploadToCommons(job.data.metadata);
50+
} else {
51+
const url =
52+
job.data?.metadata?.uri ||
53+
job.data?.downloadFileURL?.uri ||
54+
job.data?.metadata?.pdfUrl;
55+
const downloadFileRes = await downloadFile(url, "commonsFilePayload.pdf");
3056

31-
if (commonsResponse.fileUploadStatus !== 200) {
32-
logger.log({
33-
level: "error",
34-
message: `uploadToCommons: ${commonsResponse}`,
35-
});
57+
if (downloadFileRes.writeFileStatus !== 200) {
58+
logger.log({
59+
level: "error",
60+
message: `downloadFile: ${downloadFileRes}`,
61+
});
62+
process.emit(`commonsJobComplete:${job.id}`, {
63+
status: false,
64+
value: null,
65+
});
66+
return done(null, true);
67+
}
68+
const commonsResponse = await uploadToCommons(job.data.metadata);
69+
70+
if (commonsResponse.fileUploadStatus !== 200) {
71+
logger.log({
72+
level: "error",
73+
message: `uploadToCommons: ${commonsResponse}`,
74+
});
75+
process.emit(`commonsJobComplete:${job.id}`, {
76+
status: false,
77+
value: null,
78+
});
79+
return done(null, true);
80+
}
3681
process.emit(`commonsJobComplete:${job.id}`, {
37-
status: false,
38-
value: null,
82+
status: true,
83+
value: commonsResponse,
3984
});
4085
return done(null, true);
41-
// return done(new Error(`uploadToCommons: ${commonsResponse}`));
4286
}
4387
const wikiDataResponse = await uploadToWikiData(
4488
job.data.metadata,
@@ -62,4 +106,4 @@ CommonsQueue.process(async (job, done) => {
62106
});
63107
}
64108
return done(null, true);
65-
});
109+
});

‎bull/commons-queue/producer.js‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,14 @@
11
const config = require("../../utils/bullconfig");
22
const CommonsQueue = config.getNewQueue("commons-queue");
33
require("./consumer");
4-
module.exports = async (downloadFileURL = null, metadata, callback) => {
4+
module.exports = async (
5+
type = "other",
6+
downloadFileURL = null,
7+
metadata,
8+
callback
9+
) => {
510
const job = await CommonsQueue.add({
11+
type,
612
downloadFileURL,
713
metadata,
814
callback,

‎bull/email-queue/consumer.js‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,19 +16,15 @@ function generateMessage(status, title, trueURI) {
1616
}
1717
return message;
1818
}
19-
/*
20-
Mediawiki Email API DOCS - https://www.mediawiki.org/wiki/API:Emailuser#JavaScript
21-
MWN TOOLFORGE PACKAGE DOCS -https://github.com/siddharthvp/mwn
22-
*/
2319

2420
async function mediawikiEmail(username, title, trueURI, status) {
2521
try {
2622
const bot = await Mwn.init({
27-
apiUrl: process.env.EMAIL_SOURCE_URL,
23+
apiUrl: process.env.NEXT_PUBLIC_WIKIMEDIA_URL + "/w/api.php",
2824
username: process.env.EMAIL_BOT_USERNAME,
2925
password: process.env.EMAIL_BOT_PASSWORD,
3026
// Set your user agent (required for WMF wikis, see https://meta.wikimedia.org/wiki/User-Agent_policy):
31-
userAgent: "BUB2/1.0 (https://bub2.toolforge.org)",
27+
userAgent: "BUB2/1.0 (https://bub2.wmcloud.org)",
3228
// Set default parameters to be sent to be included in every API request
3329
defaultParams: {
3430
assert: "user", // ensure we're logged in
@@ -63,7 +59,7 @@ async function mediawikiEmail(username, title, trueURI, status) {
6359
} catch (error) {
6460
logger.log({
6561
level: "error",
66-
message: `mediawikiEmail: ${error}`,
62+
message: `mediawikiEmail: ${JSON.stringify(error)}`,
6763
});
6864
return error;
6965
}
@@ -79,7 +75,7 @@ EmailQueue.process(async (job, done) => {
7975
if (emailResponse !== 200) {
8076
logger.log({
8177
level: "error",
82-
message: `EmailQueue: ${emailResponse}`,
78+
message: `EmailQueue: ${JSON.stringify(emailResponse)}`,
8379
});
8480
done(new Error(`EmailQueue: ${emailResponse}`));
8581
}

‎bull/google-books-queue/consumer.js‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,13 +113,14 @@ GoogleBooksQueue.process((job, done) => {
113113
archive: true,
114114
commons: false,
115115
});
116+
done(null, true);
116117
}
117118
if (job.data.isUploadCommons === "true") {
118119
job.progress({
119120
step: "Uploading to Wikimedia Commons",
120121
value: `(${50}%)`,
121122
});
122-
CommonsProducer(null, job.data, async (commonsResponse) => {
123+
CommonsProducer(null, null, job.data, async (commonsResponse) => {
123124
if (commonsResponse.status === true) {
124125
job.progress({
125126
step: "Upload to Wikimedia Commons",
@@ -133,7 +134,9 @@ GoogleBooksQueue.process((job, done) => {
133134
},
134135
});
135136
if (job.data.isEmailNotification === "true") {
136-
const commonsLink = `https://commons.wikimedia.org/wiki/File:${commonsResponse.value.filename}`;
137+
const commonsLink =
138+
process.env.NEXT_PUBLIC_COMMONS_URL +
139+
`/wiki/File:${commonsResponse.value.filename}`;
137140
EmailProducer(
138141
job.data.userName,
139142
title,

0 commit comments

Comments
 (0)