Merge pull request #198 from okerekechinweotito/fix/pdl-library-failure

coderwassananmol · web-flow · commit 008cb3e2b87d · 2023-10-21T00:34:28.000+05:30
Fixes:[T348412] Handle PDL library failures
diff --git a/bull/pdl-queue/consumer.js b/bull/pdl-queue/consumer.js
@@ -28,33 +28,42 @@ async function getZipAndBytelength(no_of_pages, id, title, job) {
   title = title.replace(/ /g, "_");
   var img = zip.folder(`${title}_images`);
   let temp_pages = no_of_pages;
+  let downloadImageStatus;
+  let errorFlag = { status: false, page: "" };
   var download_image = async function (uri, filename) {
-    await rp({
-      method: "GET",
-      uri,
-      encoding: null,
-      transform: function (body, response) {
-        return { headers: response.headers, data: body };
-      },
-    })
-      .then(async function (body) {
-        if (/image/.test(body.headers["content-type"])) {
-          var data = new Buffer(body.data);
-          img.file(filename, data.toString("base64"), { base64: true });
-        }
-      })
-      .catch(function (err) {
-        --no_of_pages;
+    try {
+      const body = await rp({
+        method: "GET",
+        uri,
+        encoding: null,
+        transform: function (body, response) {
+          return { headers: response.headers, data: body };
+        },
       });
+      if (/image/.test(body.headers["content-type"])) {
+        var data = new Buffer(body.data);
+        img.file(filename, data.toString("base64"), { base64: true });
+      }
+      return 200;
+    } catch (err) {
+      --no_of_pages;
+      return err.statusCode;
+    }
   };
   for (let i = 1; i <= temp_pages; ++i) {
     const str = `http://www.panjabdigilib.org/images?ID=${id}&page=${i}&pagetype=null&Searched=W3GX`;
-    await download_image(str, `${title}_${i}.jpeg`);
+    downloadImageStatus = await download_image(str, `${title}_${i}.jpeg`);
     job.progress(Math.round((i / temp_pages) * 82));
+    if (downloadImageStatus >= 200 && downloadImageStatus < 300) {
+      continue;
+    } else {
+      errorFlag = { status: true, page: str };
+      break;
+    }
   }
   let { byteLength } = await zip.generateAsync({ type: "nodebuffer" });
   byteLength = Number(byteLength + no_of_pages * 16); //No. of pages * 16
-  return [zip, byteLength];
+  return [zip, byteLength, errorFlag];
 }
 
 function setHeaders(metadata, byteLength, title) {
@@ -117,22 +126,37 @@ PDLQueue.process(async (job, done) => {
   const trueURI = `http://archive.org/details/${job.data.details.IAIdentifier}`;
   jobLogs["trueURI"] = trueURI;
   jobLogs["userName"] = job.data.details.userName;
-  job.log(JSON.stringify(jobLogs));
-  logUserData(jobLogs["userName"], "Panjab Digital Library");
-  const [zip, byteLength] = await getZipAndBytelength(
+  const [zip, byteLength, errorFlag] = await getZipAndBytelength(
     job.data.details.Pages,
     job.data.details.bookID,
     job.data.details.title,
     job
   );
-  job.progress(90);
-  await uploadToIA(
-    zip,
-    job.data.details,
-    byteLength,
-    job.data.details.email,
-    job
-  );
-  job.progress(100);
-  done(null, true);
+  if (errorFlag.status) {
+    job.log(JSON.stringify(jobLogs));
+    logUserData(jobLogs["userName"], "Panjab Digital Library");
+    logger.log({
+      level: "error",
+      message: `Upload to Internet Archive failed because ${errorFlag.page} is not reachable. Please try again or contact Panjab Digital Library for more details.`,
+    });
+    job.progress(100);
+    done(
+      new Error(
+        `Upload to Internet Archive failed because <a href=${errorFlag.page} target='_blank'>${errorFlag.page}</a>  is not reachable. Please try again or contact Panjab Digital Library for more details.`
+      )
+    );
+  } else {
+    job.log(JSON.stringify(jobLogs));
+    logUserData(jobLogs["userName"], "Panjab Digital Library");
+    job.progress(90);
+    await uploadToIA(
+      zip,
+      job.data.details,
+      byteLength,
+      job.data.details.email,
+      job
+    );
+    job.progress(100);
+    done(null, true);
+  }
 });
diff --git a/components/QueueTable.js b/components/QueueTable.js
@@ -117,6 +117,22 @@ const ShowUploadQueue = (props) => {
       label: "Status",
       minWidth: 30,
       align: "left",
+      format: (value) => {
+        const isPDLMissingPage = /<a [^>]*>([^<]+)<\/a>/;
+        const missingPageLink = isPDLMissingPage.exec(value);
+        return missingPageLink ? (
+          <span>
+            Failed! (Reason: Upload to Internet Archive failed because {""}
+            <a href={missingPageLink[1]} target="_blank">
+              {missingPageLink[1]}
+            </a>{" "}
+            is not reachable. Please try again or contact Panjab Digital Library
+            for more details. )
+          </span>
+        ) : (
+          value
+        );
+      },
     },
     {
       id: "upload_progress",
@@ -154,6 +170,8 @@ const ShowUploadQueue = (props) => {
       return column.format((value === "-" ? "" : "User:") + value);
     } else if (column.id === "date") {
       return column.format(value);
+    } else if (column.id === "status") {
+      return column.format(value);
     } else {
       return value;
     }
