Skip to content

Fixes:[T348412] Handle PDL library failures #198

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
coderwassananmol merged 2 commits into from
Oct 20, 2023
Merged

Fixes:[T348412] Handle PDL library failures #198

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6503ddc
fix: handle pdl library failures
okerekechinweotito Oct 19, 2023
e695cfd
add link to PDL status-log
okerekechinweotito Oct 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 55 additions & 31 deletions bull/pdl-queue/consumer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,33 +28,42 @@ async function getZipAndBytelength(no_of_pages, id, title, job) {
title = title.replace(/ /g, "_");
var img = zip.folder(`${title}_images`);
let temp_pages = no_of_pages;
let downloadImageStatus;
let errorFlag = { status: false, page: "" };
var download_image = async function (uri, filename) {
await rp({
method: "GET",
uri,
encoding: null,
transform: function (body, response) {
return { headers: response.headers, data: body };
},
})
.then(async function (body) {
if (/image/.test(body.headers["content-type"])) {
var data = new Buffer(body.data);
img.file(filename, data.toString("base64"), { base64: true });
}
})
.catch(function (err) {
--no_of_pages;
try {
const body = await rp({
method: "GET",
uri,
encoding: null,
transform: function (body, response) {
return { headers: response.headers, data: body };
},
});
if (/image/.test(body.headers["content-type"])) {
var data = new Buffer(body.data);
img.file(filename, data.toString("base64"), { base64: true });
}
return 200;
} catch (err) {
--no_of_pages;
return err.statusCode;
}
};
for (let i = 1; i <= temp_pages; ++i) {
const str = `http://www.panjabdigilib.org/images?ID=${id}&page=${i}&pagetype=null&Searched=W3GX`;
await download_image(str, `${title}_${i}.jpeg`);
downloadImageStatus = await download_image(str, `${title}_${i}.jpeg`);
job.progress(Math.round((i / temp_pages) * 82));
if (downloadImageStatus >= 200 && downloadImageStatus < 300) {
continue;
} else {
errorFlag = { status: true, page: str };
break;
}
}
let { byteLength } = await zip.generateAsync({ type: "nodebuffer" });
byteLength = Number(byteLength + no_of_pages * 16); //No. of pages * 16
return [zip, byteLength];
return [zip, byteLength, errorFlag];
}

function setHeaders(metadata, byteLength, title) {
Expand Down Expand Up @@ -117,22 +126,37 @@ PDLQueue.process(async (job, done) => {
const trueURI = `http://archive.org/details/${job.data.details.IAIdentifier}`;
jobLogs["trueURI"] = trueURI;
jobLogs["userName"] = job.data.details.userName;
job.log(JSON.stringify(jobLogs));
logUserData(jobLogs["userName"], "Panjab Digital Library");
const [zip, byteLength] = await getZipAndBytelength(
const [zip, byteLength, errorFlag] = await getZipAndBytelength(
job.data.details.Pages,
job.data.details.bookID,
job.data.details.title,
job
);
job.progress(90);
await uploadToIA(
zip,
job.data.details,
byteLength,
job.data.details.email,
job
);
job.progress(100);
done(null, true);
if (errorFlag.status) {
job.log(JSON.stringify(jobLogs));
logUserData(jobLogs["userName"], "Panjab Digital Library");
logger.log({
Copy link
Owner

@coderwassananmol coderwassananmol Oct 18, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please log in the job logs as well including username.

Copy link
Contributor Author

@okerekechinweotito okerekechinweotito Oct 19, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@coderwassananmol
I have made the requested changes

level: "error",
message: `Upload to Internet Archive failed because ${errorFlag.page} is not reachable. Please try again or contact Panjab Digital Library for more details.`,
});
job.progress(100);
done(
new Error(
`Upload to Internet Archive failed because <a href=${errorFlag.page} target='_blank'>${errorFlag.page}</a> is not reachable. Please try again or contact Panjab Digital Library for more details.`
)
);
} else {
job.log(JSON.stringify(jobLogs));
logUserData(jobLogs["userName"], "Panjab Digital Library");
job.progress(90);
await uploadToIA(
zip,
job.data.details,
byteLength,
job.data.details.email,
job
);
job.progress(100);
done(null, true);
}
});
18 changes: 18 additions & 0 deletions components/QueueTable.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,22 @@ const ShowUploadQueue = (props) => {
label: "Status",
minWidth: 30,
align: "left",
format: (value) => {
const isPDLMissingPage = /<a [^>]*>([^<]+)<\/a>/;
const missingPageLink = isPDLMissingPage.exec(value);
return missingPageLink ? (
<span>
Failed! (Reason: Upload to Internet Archive failed because {""}
<a href={missingPageLink[1]} target="_blank">
{missingPageLink[1]}
</a>{" "}
is not reachable. Please try again or contact Panjab Digital Library
for more details. )
</span>
) : (
value
);
},
},
{
id: "upload_progress",
Expand Down Expand Up @@ -158,6 +174,8 @@ const ShowUploadQueue = (props) => {
return column.format((value === "-" ? "" : "User:") + value);
} else if (column.id === "date") {
return column.format(value);
} else if (column.id === "status") {
return column.format(value);
} else {
return value;
}
Expand Down