Skip to content

[Fixes:T355989] Integrate Google Books and Trove with WikiData #240

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
coderwassananmol merged 4 commits into from
Jun 8, 2024
Merged

[Fixes:T355989] Integrate Google Books and Trove with WikiData #240

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
414677d
add uploadToWikidata for google and trove
okerekechinweotito Jan 27, 2024
adb11c1
fix merge conflict
okerekechinweotito Jan 27, 2024
9211398
change inventory number to internet archive id
okerekechinweotito Feb 7, 2024
cb67124
Merge branch 'develop' into feat/integrate-bub2-wikimedia
coderwassananmol May 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 25 additions & 2 deletions bull/commons-queue/consumer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ const winston = require("winston");
const {
downloadFile,
uploadToCommons,
convertZipToPdf,
uploadToWikiData,
convertZipToPdf
} = require("../../utils/helper");
const JSZip = require("jszip");
const logger = winston.loggers.get("defaultLogger");
Expand Down Expand Up @@ -83,4 +84,26 @@ CommonsQueue.process(async (job, done) => {
});
return done(null, true);
}
});
const wikiDataResponse = await uploadToWikiData(
job.data.metadata,
commonsResponse.filename
);
if (wikiDataResponse !== 404) {
process.emit(`commonsJobComplete:${job.id}`, {
status: true,
value: {
commons: commonsResponse,
wikidata: wikiDataResponse,
},
});
} else {
process.emit(`commonsJobComplete:${job.id}`, {
status: true,
value: {
commons: commonsResponse,
wikidata: 404,
},
});
}
return done(null, true);
});
6 changes: 5 additions & 1 deletion bull/google-books-queue/consumer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,11 @@ GoogleBooksQueue.process((job, done) => {
step: "Upload to Wikimedia Commons",
value: `(${100}%)`,
wikiLinks: {
commons: await commonsResponse.value.filename,
commons: await commonsResponse.value.commons.filename,
wikidata:
(await commonsResponse.value.wikidata) !== 404
? await commonsResponse.value.wikidata
: 404,
},
});
if (job.data.isEmailNotification === "true") {
Expand Down
20 changes: 17 additions & 3 deletions bull/trove-queue/consumer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,12 @@ TroveQueue.process((job, done) => {
step: "Upload To IA",
value: `(${100}%)`,
});
if (
isEmailNotification !== "true" &&
job.data.details.isUploadCommons !== "true"
) {
done(null, true);
}
if (
isEmailNotification === "true" &&
job.data.details.isUploadCommons !== "true"
Expand All @@ -129,9 +135,14 @@ TroveQueue.process((job, done) => {
if (commonsResponse.status === true) {
job.progress({
step: "Upload to Wikimedia Commons",
value: `(100%)`,
value: `(${100}%)`,
wikiLinks: {
commons: await commonsResponse.value.filename,
commons: await commonsResponse.value.commons
.filename,
wikidata:
(await commonsResponse.value.wikidata) !== 404
? await commonsResponse.value.wikidata
: 404,
},
});
if (job.data.isEmailNotification === "true") {
Expand All @@ -141,7 +152,10 @@ TroveQueue.process((job, done) => {
EmailProducer(
userName,
name,
{ archiveLink: trueURI, commonsLink: commonsLink },
{
archiveLink: trueURI,
commonsLink: commonsLink,
},
{ archive: true, commons: true }
);
}
Expand Down
21 changes: 19 additions & 2 deletions components/ShowJobInformation.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,10 +144,10 @@ const ShowJobInformation = (props) => {
</CardActions>

<CardActions sx={{ marginTop: "-15px" }}>
{data.wikimedia_links !== "Not Integrated" ? (
{data.wikimedia_links.commons !== "Not Integrated" ? (
<Link
passHref
href={`${process.env.NEXT_PUBLIC_COMMONS_URL}/wiki/File:${data.wikimedia_links}`}
href={`${process.env.NEXT_PUBLIC_COMMONS_URL}/wiki/File:${data.wikimedia_links.commons}`}
>
<Button
sx={styles.button}
Expand All @@ -160,6 +160,23 @@ const ShowJobInformation = (props) => {
</Link>
) : null}
</CardActions>
<CardActions sx={{ marginTop: "-15px" }}>
{data.wikimedia_links.wikidata !== "Not Integrated" ? (
<Link
passHref
href={`https://www.wikidata.org/wiki/${data.wikimedia_links.wikidata}`}
>
<Button
sx={styles.button}
target="_blank"
size="large"
color="primary"
>
View on Wikidata
</Button>
</Link>
) : null}
</CardActions>
</Card>
</div>
);
Expand Down
11 changes: 8 additions & 3 deletions server.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,9 +188,14 @@ app
uploadLink: getUploadLink(job, trueURI),
isUploaded: jobState === "completed" ? true : false,
},
wikimedia_links: job.progress().wikiLinks?.commons
? job.progress().wikiLinks.commons
: "Not Integrated",
wikimedia_links: {
commons: job.progress().wikiLinks?.commons
? job.progress().wikiLinks.commons
: "Not Integrated",
wikidata: job.progress().wikiLinks?.wikidata
? job.progress().wikiLinks.wikidata
: "Not Integrated",
},
};

res.send(
Expand Down
240 changes: 240 additions & 0 deletions utils/helper.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -397,4 +397,244 @@ module.exports = {
return error;
}
},
uploadToWikiData: async (metadata, commonsItemFilename) => {
const bot = await Mwn.init({
apiUrl: "https://www.wikidata.org/w/api.php",
OAuth2AccessToken: metadata.oauthToken,
userAgent: "bub2.toolforge ([[https://bub2.toolforge.org]])",
defaultParams: {
assert: "user",
},
});

async function createEntity(csrf_token) {
try {
const title = metadata.details.volumeInfo.title || "";
const id = metadata.details.id || "";
const authorsArr = metadata.details.volumeInfo.authors
? metadata.details.volumeInfo.authors.join().trim()
: null;
const authors = authorsArr || "";
// Mapping for the labels/properties defined in `payload` - https://prop-explorer.toolforge.org/
const payload = {
labels: {
en: {
language: "en",
value: commonsItemFilename,
},
},
descriptions: {
en: {
language: "en",
value: title,
},
},
claims: {
file_name: [
{
mainsnak: {
snaktype: "value",
property: "P18",
datavalue: {
value: commonsItemFilename,
type: "string",
},
},
type: "statement",
rank: "normal",
},
],
file_url: [
{
mainsnak: {
snaktype: "value",
property: "P4765",
datavalue: {
value: `https://commons.wikimedia.org/wiki/File:${commonsItemFilename}`,
type: "string",
},
},
type: "statement",
rank: "normal",
},
],
commons_category: [
{
mainsnak: {
snaktype: "value",
property: "P373",
datavalue: {
value: "Bub.wikimedia",
type: "string",
},
},
type: "statement",
rank: "normal",
},
],
internet_archive_id: id
? [
{
mainsnak: {
snaktype: "value",
property: "P724",
datavalue: {
value: id,
type: "string",
},
},
type: "statement",
rank: "normal",
},
]
: undefined,
collection: [
{
mainsnak: {
snaktype: "value",
property: "P195",
datavalue: {
value: {
"entity-type": "item",
"numeric-id": 39162,
id: "Q39162", //wikidataID for 'opensource'
},
type: "wikibase-entityid",
},
},
type: "statement",
rank: "normal",
},
],
title: title
? [
{
mainsnak: {
snaktype: "value",
property: "P1476",
datavalue: {
value: {
text: title,
language: "en",
},
type: "monolingualtext",
},
},
type: "statement",
rank: "normal",
},
]
: undefined,
name: title
? [
{
mainsnak: {
snaktype: "value",
property: "P2561",
datavalue: {
value: {
text: title,
language: "en",
},
type: "monolingualtext",
},
},
type: "statement",
rank: "normal",
},
]
: undefined,
file_format: [
{
mainsnak: {
snaktype: "value",
property: "P2701",
datavalue: {
value: {
"entity-type": "item",
"numeric-id": 42332,
id: "Q42332", // wikidataID for PDF
},
type: "wikibase-entityid",
},
},
type: "statement",
rank: "normal",
},
],
author_name: authors
? [
{
mainsnak: {
snaktype: "value",
property: "P2093",
datavalue: {
value: authors,
type: "string",
},
},
type: "statement",
rank: "normal",
},
]
: undefined,
URL: [
{
mainsnak: {
snaktype: "value",
property: "P2699",
datavalue: {
value: `https://commons.wikimedia.org/wiki/File:${commonsItemFilename}`,
type: "string",
},
},
type: "statement",
rank: "normal",
},
],
copyright_status: [
{
mainsnak: {
snaktype: "value",
property: "P6216",
datavalue: {
value: {
"entity-type": "item",
"numeric-id": 6938433,
id: "Q6938433", // wikidataID for CC0 license
},
type: "wikibase-entityid",
},
},
type: "statement",
rank: "normal",
},
],
},
};

const res = await bot.request({
action: "wbeditentity",
new: "item",
summary: "bub2.toolforge.org: upload commons item to wikidata",
tags: "wikimedia-commons-app",
data: JSON.stringify(payload),
token: csrf_token,
});
logger.log({
level: "info",
message: `uploadToWikidata: Upload of ${commonsItemFilename} metadata to wikidata successful`,
});
return res.entity.id;
} catch (error) {
logger.log({
level: "error",
message: `uploadToWikidata:${error}`,
});
return 404;
}
}
const csrf_token = await bot.getCsrfToken();
return await createEntity(csrf_token);
},
};