hadyelsahar · hadyelsahar · Jan 23, 2014 · Dec 6, 2013 · Dec 3, 2013 · Dec 6, 2013
diff --git a/core/pom.xml b/core/pom.xml
@@ -105,15 +105,15 @@
     <repositories>
 
         <repository>
-            <id>osr-public-releases</id>
-            <name>OSR Public Releases</name>
-            <url>http://mojo.informatik.uni-erlangen.de/nexus/content/repositories/public-releases</url>
+          <id>osr-public-releases</id>
+          <name>OSR Public Releases</name>
+          <url>http://mojo.informatik.uni-erlangen.de/nexus/content/repositories/public-releases</url>
         </repository>
 
         <repository>
-            <id>osr-public-snapshots</id>
-            <name>OSR Public snapshots</name>
-            <url>http://mojo.informatik.uni-erlangen.de/nexus/content/repositories/public-snapshots</url>
+          <id>osr-public-snapshots</id>
+          <name>OSR Public snapshots</name>
+          <url>http://mojo.informatik.uni-erlangen.de/nexus/content/repositories/public-snapshots</url>
         </repository>
 
     </repositories>

diff --git a/core/src/main/scala/org/dbpedia/extraction/config/mappings/HomepageExtractorConfig.scala b/core/src/main/scala/org/dbpedia/extraction/config/mappings/HomepageExtractorConfig.scala
@@ -8,7 +8,7 @@ object HomepageExtractorConfig
     // For "ar" configuration, rendering right-to-left may seems like a bug, but it's not.
     // Don't change this else if you know how it is done.
 
-    val propertyNamesMap = Map(
+    private val propertyNamesMap = Map(
         "ar" -> Set("الموقع", "الصفحة الرسمية", "موقع", "الصفحة الرئيسية", "صفحة ويب", "موقع ويب"),
         "ca" -> Set("pàgina", "web", "lloc"),
         "de" -> Set("website", "homepage", "webpräsenz", "web", "site", "siteweb", "site web"),/*cleanup*/
@@ -26,9 +26,13 @@ object HomepageExtractorConfig
         "ru" -> Set("сайт")
     )
 
+    def propertyNames(lang : String) : Set[String] = {
+        propertyNamesMap.getOrElse(lang, Set())
+    }
+
     val supportedLanguages = propertyNamesMap.keySet
 
-    val externalLinkSectionsMap = Map(
+    private val externalLinkSectionsMap = Map(
         "ar" -> "وصلات خارجية",
         "ca" -> "(?:Enllaços externs|Enllaço extern)",
         "de" -> "Weblinks?",
@@ -46,7 +50,11 @@ object HomepageExtractorConfig
         "ru" -> "Ссылки"
     )
 
-    val officialMap = Map(
+    def externalLinkSections(lang : String) : String = {
+        externalLinkSectionsMap.getOrElse(lang, "")
+    }
+
+    private val officialMap = Map(
         "ar" -> "رسمي",
         "ca" -> "oficial",
         "de" -> "offizielle",
@@ -64,4 +72,26 @@ object HomepageExtractorConfig
         "ru" -> "официальный"
     )
 
+    def official(lang : String) : String = {
+        officialMap.getOrElse(lang, "")
+    }
+
+    // Map(language -> Map(templateName -> templatePropertyKey))
+    private val templateOfficialWebsiteMap = Map(
+        "ca" -> Map("Oficial" -> "1"),
+        /* "it" -> Map("Sito Ufficiale" -> "1"), This does not exist, yet */
+        "el" -> Map("Επίσημη ιστοσελίδα" -> "1"),
+        "en" -> Map("Official website" -> "1"),
+        "eo" -> Map("Oficiala_retejo" -> "1"),
+        "es" -> Map("Página_web" -> "1"),
+        "fr" -> Map("Site_officiel" -> "url"),
+        "ga" -> Map("Páxina_web" -> "1"),
+        "pt" -> Map("Oficial" -> "1"),
+        "ru" -> Map("Официальный сайт" -> "1")
+    )
+
+    def templateOfficialWebsite(lang : String) : Map[String, String] = {
+        templateOfficialWebsiteMap.getOrElse(lang, Map())
+    }
+
 }
diff --git a/core/src/main/scala/org/dbpedia/extraction/config/mappings/InfoboxExtractorConfig.scala b/core/src/main/scala/org/dbpedia/extraction/config/mappings/InfoboxExtractorConfig.scala
@@ -27,9 +27,13 @@ object InfoboxExtractorConfig
 
     // When you generate statistics, set the following to true. To get full coverage, you should
     // probably set most other parameters here to zero or empty values. 
-    val extractTemplateStatistics = false 
+    val extractTemplateStatistics =
+      try {
+        System.getProperty("extract.template.stats", "false").toBoolean
+      } catch {
+        case ex : Exception => false
+      }
 
     val minPropertyCount = 2
-
     val minRatioOfExplicitPropertyKeys = 0.75
 }
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/HomepageExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/HomepageExtractor.scala
@@ -22,11 +22,13 @@ extends PageNodeExtractor
 {
   private val language = context.language.wikiCode
 
-  private val propertyNames = HomepageExtractorConfig.propertyNamesMap(language)
+  private val propertyNames = HomepageExtractorConfig.propertyNames(language)
 
-  private val official = HomepageExtractorConfig.officialMap(language)
+  private val official = HomepageExtractorConfig.official(language)
 
-  private val externalLinkSections = HomepageExtractorConfig.externalLinkSectionsMap(language)
+  private val externalLinkSections = HomepageExtractorConfig.externalLinkSections(language)
+
+  private val templateOfficialWebsite = HomepageExtractorConfig.templateOfficialWebsite(language)
 
   private val homepageProperty = context.ontology.properties("foaf:homepage")
 
@@ -35,37 +37,49 @@ extends PageNodeExtractor
   private val officialAndLineEndRegex = ("""(?msiu)[^$]*\b""" + official + """\b.*$.*""").r
   private val officialAndNoLineEndRegex = ("""(?msiu)[^$]*\b""" + official + """\b[^$]*""").r
   private val lineEndRegex = "(?ms).*$.+".r
+  // Similar to org.dbpedia.extraction.config.dataparser.DataParserConfig.splitPropertyNodeRegexLink - without '/' and ';'
+  private val splitPropertyNodeLinkStrict = """<br\s*\/?>|\n| and | or |,| """
 
   override val datasets = Set(DBpediaDatasets.Homepages)
 
   override def extract(page: PageNode, subjectUri: String, pageContext: PageContext): Seq[Quad] =
   {
     if(page.title.namespace != Namespace.Main) return Seq.empty
-
-    val list = collectProperties(page).filter(p => propertyNames.contains(p.key.toLowerCase))
-    list.foreach((property) => {
-      property.children match
-      {
-        case (textNode @ TextNode(text, _)) :: _ =>
+
+    val list = collectProperties(page).filter(p => propertyNames.contains(p.key.toLowerCase)).flatMap {
+      NodeUtil.splitPropertyNode(_, splitPropertyNodeLinkStrict, true)
+    }
+
+    list.foreach((property) =>
+
+      // Find among children
+      for (child <- property.children) {
+        child match
         {
-          val url = if (!text.startsWith("http")) "http://" + text else text
-          val graph = generateStatement(subjectUri, pageContext, url, textNode)
-          if (!graph.isEmpty)
+          case (textNode @ TextNode(text, _)) =>
           {
-            return graph
+            val cleaned = cleanProperty(text)
+            if (cleaned.nonEmpty) { // do not proceed if the property value is not a valid candidate
+              val url = if (!cleaned.startsWith("http")) "http://" + cleaned else cleaned
+              val graph = generateStatement(subjectUri, pageContext, url, textNode)
+              if (!graph.isEmpty)
+              {
+                return graph
+              }
+            }
           }
-        }
-        case (linkNode @ ExternalLinkNode(destination, _, _, _)) :: _ =>
-        {
-          val graph = generateStatement(subjectUri, pageContext, destination.toString, linkNode)
-          if (!graph.isEmpty)
+          case (linkNode @ ExternalLinkNode(destination, _, _, _)) =>
           {
-            return graph
+            val graph = generateStatement(subjectUri, pageContext, destination.toString, linkNode)
+            if (!graph.isEmpty)
+            {
+              return graph
+            }
           }
+          case _ =>
         }
-        case _ =>
       }
-    })
+    )
 
     for(externalLinkSectionChildren <- collectExternalLinkSection(page.children))
     {
@@ -84,6 +98,18 @@ extends PageNodeExtractor
     Seq.empty
   }
 
+  private def cleanProperty(text: String) : String = {
+
+    val candidateUrl = text.stripLineEnd.trim // remove ending new line
+
+    // While it is perfectly legal to have hostnames without dots in URLs
+    // it is very unlikely that such URLs will be present in Wikipedia
+    // Most of the times such values represent texts inserted by editors
+    // to convey a "missing homepage" info, such as None, N/A, missing, down etc.
+    if (candidateUrl.matches(""".*\w\.\w.*""")) candidateUrl
+    else ""
+  }
+
   private def generateStatement(subjectUri: String, pageContext: PageContext, url: String, node: Node): Seq[Quad] =
   {
     try
@@ -101,20 +127,50 @@ extends PageNodeExtractor
     Seq.empty
   }
 
+  private def extractUrlFromProperty(node: PropertyNode): Option[String] = {
+
+    /*
+    It could be:
+    1) {{template | key = example.com }}
+    2) {{template | key = http://example.com }}
+
+    In 1) => PropertyNode("key", List(TextNode("example.com", _))
+    In 2) => PropertyNode("key", List(ExternalLinkNode(URI("http://example.com"), ...)))
+     */
+    val url = node.children.collect {
+      case TextNode(t, _) => t
+      case ExternalLinkNode(destination, _, _, _) => destination.toString
+    }.mkString.trim
+
+    if (url.isEmpty) {
+      None
+    } else {
+      try {
+        val uri = new URI(url)
+        if (uri.getScheme == null) Some("http://" + uri.toString)
+        else Some(uri.toString)
+      } catch {
+        case _ : Exception => None
+      }
+    }
+  }
+
   private def findLinkTemplateInSection(nodes: List[Node]): Option[(String, Node)] =
   {
     // TODO: use for-loop instead of recursion
     nodes match
     {
-      // TODO: use language-specific name
-      case (templateNode @ TemplateNode(title, _, _, _)) :: _
-          if ((title.decoded == "Official") || ((context.redirects.map.contains(title.decoded)) && (context.redirects.map(title.decoded) == "Official"))) =>
+      case (templateNode @ TemplateNode(title, _, _, _)) :: tail =>
       {
-        templateNode.property("1") match
-        {
-          case Some(propertyNode) => propertyNode.retrieveText.map(url => (url, propertyNode))
-          case _ => None
+        val templateRedirect = context.redirects.resolve(title).decoded
+        if (templateOfficialWebsite.contains(templateRedirect)) {
+          templateNode.property(templateOfficialWebsite(templateRedirect)) match
+          {
+            case Some(propertyNode) => extractUrlFromProperty(propertyNode).map(url => (url, propertyNode))
+            case None => findLinkTemplateInSection(tail) // do not stop the recursion - there might be other templates
+          }
         }
+        else findLinkTemplateInSection(tail)
       }
       case head :: tail => findLinkTemplateInSection(tail)
       case Nil => None

diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/WikidataExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/WikidataExtractor.scala
@@ -5,6 +5,7 @@ import org.dbpedia.extraction.util.Language
 import org.dbpedia.extraction.destinations.{Quad, DBpediaDatasets}
 import org.dbpedia.extraction.wikiparser.{JsonNode, Namespace, PageNode}
 import collection.mutable.ArrayBuffer
+import scala.language.reflectiveCalls
 
 /**
  * Extracts data from Wikidata sources.

diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/WikidataFactsExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/WikidataFactsExtractor.scala
@@ -5,6 +5,7 @@ import org.dbpedia.extraction.util.Language
 import org.dbpedia.extraction.destinations.{Quad, DBpediaDatasets}
 import org.dbpedia.extraction.wikiparser.{JsonNode, PageNode}
 import collection.mutable.ArrayBuffer
+import scala.language.reflectiveCalls
 
 /**
  * Extracts Wikidata claims

diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/WikidataLLExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/WikidataLLExtractor.scala
@@ -5,6 +5,7 @@ import org.dbpedia.extraction.util.Language
 import org.dbpedia.extraction.destinations.{Quad, DBpediaDatasets}
 import org.dbpedia.extraction.wikiparser.{JsonNode, Namespace, PageNode}
 import collection.mutable.ArrayBuffer
+import scala.language.reflectiveCalls
 
 /**
  * Extracts data from Wikidata sources.

diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/WikidataLabelExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/WikidataLabelExtractor.scala
@@ -6,7 +6,7 @@ import org.dbpedia.extraction.util.Language
 import org.dbpedia.extraction.destinations.{Quad, DBpediaDatasets}
 import org.dbpedia.extraction.wikiparser.{JsonNode, PageNode}
 import collection.mutable.ArrayBuffer
-import  org.dbpedia.extraction.wikiparser.Namespace
+import scala.language.reflectiveCalls
 
 /**
  * Extracts labels triples from Wikidata sources

diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/WikidataMappedFactsExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/WikidataMappedFactsExtractor.scala
@@ -8,6 +8,7 @@ import collection.mutable.ArrayBuffer
 import org.dbpedia.extraction.ontology.io.OntologyReader
 import org.dbpedia.extraction.dataparser.{DataParser, DateTimeParser}
 import org.dbpedia.extraction.ontology.datatypes.Datatype
+import scala.language.reflectiveCalls
 
 /**
  * Extracts Wikidata claims

diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/WikidataNameSpaceSameAsExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/WikidataNameSpaceSameAsExtractor.scala
@@ -5,6 +5,7 @@ import org.dbpedia.extraction.util.Language
 import org.dbpedia.extraction.destinations.{Quad, DBpediaDatasets}
 import org.dbpedia.extraction.wikiparser.{JsonNode, PageNode}
 import collection.mutable.ArrayBuffer
+import scala.language.reflectiveCalls
 
 /**
  * it's an extractor to extract Mappings between Wikidata URIs to WikiData URIs inside DBpedia, in the form of :

diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/WikidataSameAsExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/WikidataSameAsExtractor.scala
@@ -5,6 +5,7 @@ import org.dbpedia.extraction.util.Language
 import org.dbpedia.extraction.destinations.{Quad, DBpediaDatasets}
 import org.dbpedia.extraction.wikiparser.{JsonNode, PageNode}
 import collection.mutable.ArrayBuffer
+import scala.language.reflectiveCalls
 
 /**
  * it's an extractor to extract sameas data from DBpedia-WikiData on the form of

diff --git a/core/src/main/scala/org/dbpedia/extraction/util/WikiApi.scala b/core/src/main/scala/org/dbpedia/extraction/util/WikiApi.scala
@@ -53,7 +53,7 @@ class WikiApi(url: URL, language: Language)
         // -> "generator" instead of "list" and "gapnamespace" instead of "apnamespace" ("gap" is for "generator all pages")
 
         //Retrieve list of pages
-        val response = query("?action=query&format=xml&list=allpages&apfrom=" + fromPage + "&aplimit=" + pageListLimit + "&apnamespace=" + namespace.code)
+        val response = query("?action=query&format=xml&list=allpages&apfrom=" + URLEncoder.encode(fromPage, "UTF-8") + "&aplimit=" + pageListLimit + "&apnamespace=" + namespace.code)
 
         //Extract page ids
         val pageIds = for(p <- response \ "query" \ "allpages" \ "p") yield (p \ "@pageid").head.text.toLong
@@ -119,7 +119,7 @@ class WikiApi(url: URL, language: Language)
         {
             for(titleGroup <- titles.grouped(pageDownloadLimit))
             {
-                val response = query("?action=query&format=xml&prop=revisions&titles=" + titleGroup.map(_.encodedWithNamespace).mkString("|") + "&rvprop=ids|content|timestamp|user|userid")
+                val response = query("?action=query&format=xml&prop=revisions&titles=" + titleGroup.map(t => URLEncoder.encode(t.encodedWithNamespace, "UTF-8")).mkString("|") + "&rvprop=ids|content|timestamp|user|userid")
                 processPages(response, proc)
             }
         }