fix and imp

PritamP20 · PritamP20 · commit 35de5829c77a · 2025-12-17T20:31:02.000+05:30
diff --git a/core/src/main/scala/org/dbpedia/extraction/util/DataQualityMonitor.scala b/core/src/main/scala/org/dbpedia/extraction/util/DataQualityMonitor.scala
@@ -5,56 +5,30 @@ import java.util.concurrent.atomic.AtomicLong
 import scala.collection.concurrent.TrieMap
 
 /**
- * Centralized monitoring and logging system for extraction quality and errors.
- *
- * Features:
- * - Structured logging with context (extractor, page, error type)
- * - Metrics collection for error rates
- * - Thread-safe error counting
- * - Export capabilities for failed extractions
- *
- * Usage:
- * {{{
- *   val monitor = DataQualityMonitor.forExtractor("HomepageExtractor")
- *   monitor.logInvalidData("Einstein", "Invalid IRI: malformed URL", exception)
- *   monitor.getMetrics() // Get error statistics
- * }}}
+ * Monitors data quality issues during extraction.
+ * Tracks errors per extractor and provides export capabilities.
  */
 object DataQualityMonitor {
 
   private val logger = Logger.getLogger(classOf[DataQualityMonitor].getName)
-
-  // Global metrics storage (thread-safe)
   private val errorCounts = new TrieMap[String, AtomicLong]()
   private val errorDetails = new TrieMap[String, collection.mutable.ListBuffer[ExtractionError]]()
 
-  /**
-   * Create a monitor for a specific extractor
-   */
   def forExtractor(extractorName: String): DataQualityMonitor = {
     new DataQualityMonitor(extractorName)
   }
 
-  /**
-   * Get global extraction statistics
-   */
   def getGlobalMetrics(): Map[String, Long] = {
     errorCounts.map { case (key, counter) => (key, counter.get()) }.toMap
   }
 
-  /**
-   * Get detailed errors for analysis
-   */
   def getErrorDetails(errorType: String, limit: Int = 100): List[ExtractionError] = {
     errorDetails.get(errorType) match {
       case Some(errors) => errors.take(limit).toList
       case None => List.empty
     }
   }
 
-  /**
-   * Export errors to CSV format for analysis
-   */
   def exportToCsv(errorType: String, limit: Int = 1000): String = {
     val errors = getErrorDetails(errorType, limit)
     val header = "Extractor,PageTitle,ErrorMessage,Timestamp\n"
@@ -64,80 +38,46 @@ object DataQualityMonitor {
     header + rows
   }
 
-  /**
-   * Reset all metrics (useful for testing)
-   */
   def reset(): Unit = {
     errorCounts.clear()
     errorDetails.clear()
   }
 }
 
-/**
- * Monitor instance for a specific extractor
- */
 class DataQualityMonitor(val extractorName: String) {
 
   private val logger = Logger.getLogger(s"org.dbpedia.extraction.monitor.$extractorName")
 
-  /**
-   * Log invalid data with context
-   *
-   * @param pageTitle The Wikipedia page being processed
-   * @param reason Description of why the data is invalid
-   * @param exception Optional exception that caused the error
-   * @param data Optional invalid data for debugging
-   */
   def logInvalidData(
     pageTitle: String,
     reason: String,
     exception: Option[Throwable] = None,
     data: Option[String] = None
   ): Unit = {
     val message = buildMessage(pageTitle, reason, data)
-
-    // Log with appropriate level
     exception match {
       case Some(ex) => logger.log(Level.WARNING, message, ex)
       case None => logger.warning(message)
     }
-
-    // Record metrics
     recordError(pageTitle, reason, exception)
   }
 
-  /**
-   * Log skipped extraction with reason
-   */
   def logSkipped(pageTitle: String, reason: String): Unit = {
     logger.fine(s"[$extractorName] Skipped '$pageTitle': $reason")
   }
 
-  /**
-   * Log successful extraction with statistics
-   */
   def logSuccess(pageTitle: String, triplesCount: Int): Unit = {
     logger.fine(s"[$extractorName] Extracted $triplesCount triples from '$pageTitle'")
   }
 
-  /**
-   * Get metrics for this extractor
-   */
   def getMetrics(): Map[String, Long] = {
     DataQualityMonitor.errorCounts
       .filter { case (key, _) => key.startsWith(s"$extractorName:") }
       .map { case (key, counter) => (key, counter.get()) }
       .toMap
   }
 
-  /**
-   * Get total error count for this extractor
-   */
-  def getTotalErrors(): Long = {
-    getMetrics().values.sum
-  }
-
-  // Private helper methods
+  def getTotalErrors(): Long = getMetrics().values.sum
 
   private def buildMessage(pageTitle: String, reason: String, data: Option[String]): String = {
     val dataStr = data.map(d => s" | Data: ${truncate(d, 200)}").getOrElse("")
@@ -147,12 +87,10 @@ class DataQualityMonitor(val extractorName: String) {
   private def recordError(pageTitle: String, reason: String, exception: Option[Throwable]): Unit = {
     val errorType = s"$extractorName:${categorizeError(reason, exception)}"
 
-    // Increment counter
     DataQualityMonitor.errorCounts
       .getOrElseUpdate(errorType, new AtomicLong(0))
       .incrementAndGet()
 
-    // Store details (limit to prevent memory issues)
     val errorDetail = ExtractionError(
       extractorName = extractorName,
       pageTitle = pageTitle,
@@ -161,14 +99,11 @@ class DataQualityMonitor(val extractorName: String) {
       timestamp = System.currentTimeMillis()
     )
 
-    DataQualityMonitor.errorDetails
-      .getOrElseUpdate(errorType, collection.mutable.ListBuffer.empty)
-      .synchronized {
-        val buffer = DataQualityMonitor.errorDetails(errorType)
-        if (buffer.size < 10000) { // Limit storage
-          buffer += errorDetail
-        }
-      }
+    DataQualityMonitor.errorDetails.synchronized {
+      val buffer = DataQualityMonitor.errorDetails
+        .getOrElseUpdate(errorType, collection.mutable.ListBuffer.empty)
+      if (buffer.size < 10000) buffer += errorDetail
+    }
   }
 
   private def categorizeError(reason: String, exception: Option[Throwable]): String = {
@@ -187,9 +122,6 @@ class DataQualityMonitor(val extractorName: String) {
   }
 }
 
-/**
- * Case class representing an extraction error
- */
 case class ExtractionError(
   extractorName: String,
   pageTitle: String,
diff --git a/core/src/test/scala/org/dbpedia/extraction/util/DataQualityMonitorTest.scala b/core/src/test/scala/org/dbpedia/extraction/util/DataQualityMonitorTest.scala
@@ -2,72 +2,63 @@ package org.dbpedia.extraction.util
 
 import org.scalatest.{BeforeAndAfter, FlatSpec, Matchers}
 
-/**
- * Test suite for DataQualityMonitor
- */
 class DataQualityMonitorTest extends FlatSpec with Matchers with BeforeAndAfter {
 
   before {
-    // Reset metrics before each test
     DataQualityMonitor.reset()
   }
 
-  "DataQualityMonitor" should "create a monitor for an extractor" in {
+  "DataQualityMonitor" should "create monitor" in {
     val monitor = DataQualityMonitor.forExtractor("TestExtractor")
     monitor should not be null
     monitor.extractorName should be("TestExtractor")
   }
 
-  it should "record invalid data errors" in {
+  it should "record errors" in {
     val monitor = DataQualityMonitor.forExtractor("TestExtractor")
-
     monitor.logInvalidData(
       "Albert_Einstein",
       "Invalid IRI syntax",
       Some(new IllegalArgumentException("Test exception")),
       Some("http://malformed url")
     )
 
-    val metrics = monitor.getMetrics()
-    metrics should not be empty
+    monitor.getMetrics() should not be empty
     monitor.getTotalErrors() should be(1)
   }
 
-  it should "categorize errors correctly" in {
+  it should "categorize errors" in {
     val monitor = DataQualityMonitor.forExtractor("TestExtractor")
 
-    // Log different types of errors
     monitor.logInvalidData("Page1", "Invalid data", exception = Some(new IllegalArgumentException()))
     monitor.logInvalidData("Page2", "Malformed URL", exception = None)
     monitor.logInvalidData("Page3", "Missing property", exception = None)
 
-    val metrics = monitor.getMetrics()
-    metrics.size should be >= 1
+    monitor.getMetrics().size should be >= 1
     monitor.getTotalErrors() should be(3)
   }
 
-  it should "track multiple extractors independently" in {
-    val monitor1 = DataQualityMonitor.forExtractor("ExtractorA")
-    val monitor2 = DataQualityMonitor.forExtractor("ExtractorB")
+  it should "track extractors independently" in {
+    val m1 = DataQualityMonitor.forExtractor("ExtractorA")
+    val m2 = DataQualityMonitor.forExtractor("ExtractorB")
 
-    monitor1.logInvalidData("Page1", "Error in A")
-    monitor1.logInvalidData("Page2", "Another error in A")
-    monitor2.logInvalidData("Page3", "Error in B")
+    m1.logInvalidData("Page1", "Error in A")
+    m1.logInvalidData("Page2", "Another error in A")
+    m2.logInvalidData("Page3", "Error in B")
 
-    monitor1.getTotalErrors() should be(2)
-    monitor2.getTotalErrors() should be(1)
+    m1.getTotalErrors() should be(2)
+    m2.getTotalErrors() should be(1)
   }
 
   it should "provide global metrics" in {
-    val monitor1 = DataQualityMonitor.forExtractor("ExtractorA")
-    val monitor2 = DataQualityMonitor.forExtractor("ExtractorB")
+    val m1 = DataQualityMonitor.forExtractor("ExtractorA")
+    val m2 = DataQualityMonitor.forExtractor("ExtractorB")
 
-    monitor1.logInvalidData("Page1", "Error 1")
-    monitor2.logInvalidData("Page2", "Error 2")
-    monitor2.logInvalidData("Page3", "Error 3")
+    m1.logInvalidData("Page1", "Error 1")
+    m2.logInvalidData("Page2", "Error 2")
+    m2.logInvalidData("Page3", "Error 3")
 
-    val globalMetrics = DataQualityMonitor.getGlobalMetrics()
-    globalMetrics.values.sum should be(3)
+    DataQualityMonitor.getGlobalMetrics().values.sum should be(3)
   }
 
   it should "retrieve error details" in {
@@ -81,60 +72,41 @@ class DataQualityMonitorTest extends FlatSpec with Matchers with BeforeAndAfter
     details.head.extractorName should be("TestExtractor")
   }
 
-  it should "export errors to CSV format" in {
+  it should "export to CSV" in {
     val monitor = DataQualityMonitor.forExtractor("TestExtractor")
 
     monitor.logInvalidData("Einstein", "Invalid IRI")
     monitor.logInvalidData("Tesla", "Malformed URL")
 
     val csv = DataQualityMonitor.exportToCsv("TestExtractor:InvalidData", limit = 100)
-
-    csv should include("Extractor,PageTitle,ErrorMessage,Timestamp")
-    csv should include("TestExtractor")
+    csv should include("Extractor,PageTitle,ErrorMessage")
     csv should include("Einstein")
   }
 
-  it should "limit stored error details to prevent memory issues" in {
+  it should "limit stored details" in {
     val monitor = DataQualityMonitor.forExtractor("TestExtractor")
+    (1 to 11000).foreach(i => monitor.logInvalidData(s"Page$i", "Error"))
 
-    // Try to log more than the limit (10000)
-    for (i <- 1 to 11000) {
-      monitor.logInvalidData(s"Page$i", "Error")
-    }
-
-    // Should still work without memory issues
-    val metrics = monitor.getMetrics()
     monitor.getTotalErrors() should be(11000)
-
-    // But details should be limited
     val details = DataQualityMonitor.getErrorDetails("TestExtractor:InvalidData", limit = 20000)
     details.size should be <= 10000
   }
 
   it should "log skipped extractions" in {
     val monitor = DataQualityMonitor.forExtractor("TestExtractor")
-
-    // Should not throw exception
     monitor.logSkipped("TestPage", "Not in main namespace")
-
-    // Skipped pages don't count as errors
     monitor.getTotalErrors() should be(0)
   }
 
   it should "log successful extractions" in {
     val monitor = DataQualityMonitor.forExtractor("TestExtractor")
-
-    // Should not throw exception
     monitor.logSuccess("Einstein", 5)
-
-    // Successful extractions don't count as errors
     monitor.getTotalErrors() should be(0)
   }
 
-  it should "handle concurrent logging safely" in {
+  it should "handle concurrent logging" in {
     val monitor = DataQualityMonitor.forExtractor("ConcurrentExtractor")
 
-    // Simulate concurrent logging from multiple threads
     val threads = (1 to 10).map { i =>
       new Thread {
         override def run(): Unit = {
@@ -147,12 +119,10 @@ class DataQualityMonitorTest extends FlatSpec with Matchers with BeforeAndAfter
 
     threads.foreach(_.start())
     threads.foreach(_.join())
-
-    // Should have logged all errors without data corruption
     monitor.getTotalErrors() should be(1000)
   }
 
-  "Error details" should "include all required fields" in {
+  "Error details" should "include required fields" in {
     val monitor = DataQualityMonitor.forExtractor("TestExtractor")
 
     monitor.logInvalidData(
diff --git a/server/src/test/scala/org/dbpedia/extraction/server/resources/DataQualityTest.scala b/server/src/test/scala/org/dbpedia/extraction/server/resources/DataQualityTest.scala
