change Extractor Trait to accept [T] type argument

hadyelsahar · hadyelsahar · commit e26ef813dad0 · 2013-11-18T12:01:00.000+02:00
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/AbstractExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/AbstractExtractor.scala
@@ -24,7 +24,7 @@ class AbstractExtractor(
     def language : Language
   }
 )
-extends Extractor
+extends Extractor[PageNode]
 {
     private val maxRetries = 3
 
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/ArticleCategoriesExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/ArticleCategoriesExtractor.scala
@@ -11,7 +11,7 @@ import org.dbpedia.extraction.wikiparser._
  */
 class ArticleCategoriesExtractor( context : {
                                       def ontology : Ontology
-                                      def language : Language } ) extends Extractor
+                                      def language : Language } ) extends Extractor[PageNode]
 {
     private val dctermsSubjectProperty = context.ontology.properties("dct:subject")
 
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/ArticleTemplatesExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/ArticleTemplatesExtractor.scala
@@ -17,7 +17,7 @@ class ArticleTemplatesExtractor(
      def language: Language
      def redirects: Redirects
     }
-  ) extends Extractor {
+  ) extends Extractor[PageNode] {
 
   // FIXME: this uses the http://xx.dbpedia.org/property/ namespace, but the
   // http://dbpedia.org/ontology/ namespace would probably make more sense.
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/CategoryLabelExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/CategoryLabelExtractor.scala
@@ -11,7 +11,7 @@ import org.dbpedia.extraction.util.Language
  */
 class CategoryLabelExtractor( context : {
                                   def ontology : Ontology
-                                  def language : Language } ) extends Extractor
+                                  def language : Language } ) extends Extractor[PageNode]
 {
     private val labelProperty = context.ontology.properties("rdfs:label")
     
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/CompositeExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/CompositeExtractor.scala
@@ -2,10 +2,11 @@ package org.dbpedia.extraction.mappings
 
 import org.dbpedia.extraction.destinations.Quad
 import org.dbpedia.extraction.wikiparser.PageNode
+import org.dbpedia.extraction.mappings.Extractor
 
-class CompositeExtractor(extractors: Mapping[PageNode]*)
+class CompositeExtractor(extractors: Seq[Extractor[PageNode]])
 extends CompositeMapping[PageNode](extractors: _*)
-with Extractor
+with Extractor[PageNode]
 
 /**
  * Creates new extractors.
@@ -14,15 +15,25 @@ object CompositeExtractor
 {
     /**
      * Creates a new extractor.
-     * 
+     *
      * TODO: using reflection here loses compile-time type safety.
      *
      * @param extractors List of extractor classes to be instantiated
      * @param context Any type of object that implements the required parameter methods for the extractors
      */
-    def load(classes: Seq[Class[_ <: Extractor]], context: AnyRef): Extractor =
+    def load(classes: Seq[Class[_ <: Extractor[_]]], context: AnyRef): Extractor[PageNode] =
     {
-        val extractors = classes.map(_.getConstructor(classOf[AnyRef]).newInstance(context))
-        new CompositeExtractor(extractors: _*)
+      val extractors = classes.map(_.getConstructor(classOf[AnyRef]).newInstance(context))
+
+      var pageNodeExtractors : Seq[Extractor[PageNode]] = Seq.empty
+
+      extractors foreach { extractor : Extractor[_] =>
+        extractor match {
+        case ex : Extractor[PageNode] =>  pageNodeExtractors :+ ex
+        case _ =>
+      }
+      }
+
+      new CompositeExtractor(pageNodeExtractors: Seq[Extractor[PageNode]])
     }
 }
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/ContributorExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/ContributorExtractor.scala
@@ -16,7 +16,7 @@ import java.net.{URLEncoder, URI}
 
 class ContributorExtractor( context : {
   def ontology : Ontology
-  def language : Language } ) extends Extractor
+  def language : Language } ) extends Extractor[PageNode]
 {
 
   override val datasets = Set(DBpediaDatasets.RevisionMeta)
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/DisambiguationExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/DisambiguationExtractor.scala
@@ -16,7 +16,7 @@ class DisambiguationExtractor(
     def language        : Language
   }
 )
-extends Extractor
+extends Extractor[PageNode]
 {
   private val language = context.language
 
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/ExternalLinksExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/ExternalLinksExtractor.scala
@@ -16,7 +16,7 @@ class ExternalLinksExtractor (
     def language : Language
   }
 )
-extends Extractor
+extends Extractor[PageNode]
 {
   val wikiPageExternalLinkProperty = context.ontology.properties("wikiPageExternalLink")
 
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/Extractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/Extractor.scala
@@ -8,4 +8,4 @@ import org.dbpedia.extraction.wikiparser._
  * Necessary to get some type safety in CompositeExtractor: 
  * Class[_ <: Extractor] can be checked at runtime, but Class[_ <: Mapping[PageNode]] can not.
  */
-trait Extractor extends Mapping[PageNode]
+trait Extractor[T] extends Mapping[T]
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/FlickrWrapprLinkExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/FlickrWrapprLinkExtractor.scala
@@ -17,7 +17,7 @@ class FlickrWrapprLinkExtractor (
     def language : Language
   }
 )
-extends Extractor
+extends Extractor[PageNode]
 {
   private val language = context.language
   
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/GeoExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/GeoExtractor.scala
@@ -17,7 +17,7 @@ class GeoExtractor(
     def language : Language 
   } 
 ) 
-extends Extractor
+extends Extractor[PageNode]
 {
   private val geoCoordinateParser = new GeoCoordinateParser(context)
 
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/HomepageExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/HomepageExtractor.scala
@@ -17,7 +17,7 @@ class HomepageExtractor(
     def redirects : Redirects
   }
 )
-extends Extractor
+extends Extractor[PageNode]
 {
   private val language = context.language.wikiCode
 
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/ImageExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/ImageExtractor.scala
@@ -29,7 +29,7 @@ class ImageExtractor(
     def commonsSource: Source 
   } 
 ) 
-extends Extractor
+extends Extractor[PageNode]
 {
   private val wikiCode = context.language.wikiCode
   private val language = context.language
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/InfoboxExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/InfoboxExtractor.scala
@@ -29,7 +29,7 @@ class InfoboxExtractor(
     def redirects : Redirects 
   } 
 ) 
-extends Extractor
+extends Extractor[PageNode]
 {
     ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
     // Configuration
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/InterLanguageLinksExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/InterLanguageLinksExtractor.scala
@@ -10,7 +10,7 @@ import scala.collection.mutable.ArrayBuffer
 /**
  * Extracts interwiki links
  */
-class InterLanguageLinksExtractor(context: { def ontology : Ontology; def language : Language }) extends Extractor
+class InterLanguageLinksExtractor(context: { def ontology : Ontology; def language : Language }) extends Extractor[PageNode]
 {
   private val interLanguageLinksProperty = context.ontology.properties("wikiPageInterLanguageLink")
 
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/IriSameAsUriExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/IriSameAsUriExtractor.scala
@@ -17,7 +17,7 @@ class IriSameAsUriExtractor (
     def language : Language
   }
 )
-extends Extractor
+extends Extractor[PageNode]
 {
   private val language = context.language
 
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/LabelExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/LabelExtractor.scala
@@ -14,7 +14,7 @@ class LabelExtractor(
     def language : Language 
   } 
 ) 
-extends Extractor
+extends Extractor[PageNode]
 {
   val labelProperty = context.ontology.properties("rdfs:label")
   
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/MappingExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/MappingExtractor.scala
@@ -14,7 +14,7 @@ class MappingExtractor(
     def redirects : Redirects
   }
 )
-extends Extractor
+extends Extractor[PageNode]
 {
   private val templateMappings = context.mappings.templateMappings
   private val tableMappings = context.mappings.tableMappings
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/MetaInformationExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/MetaInformationExtractor.scala
@@ -17,7 +17,7 @@ import java.net.URI
 
 class MetaInformationExtractor( context : {
   def ontology : Ontology
-  def language : Language } ) extends Extractor
+  def language : Language } ) extends Extractor[PageNode]
 {
   val modificationDatePredicate = context.ontology.properties("wikiPageModified")
   val extractionDatePredicate = context.ontology.properties("wikiPageExtracted")
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/PageIdExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/PageIdExtractor.scala
@@ -15,7 +15,7 @@ class PageIdExtractor (
     def language: Language
   }
 )
-extends Extractor
+extends Extractor[PageNode]
 {
   private val wikiPageIdProperty = context.ontology.properties("wikiPageID")
 
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/PageLinksExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/PageLinksExtractor.scala
@@ -16,7 +16,7 @@ class PageLinksExtractor (
     def language : Language
   }
 )
-extends Extractor
+extends Extractor[PageNode]
 {
   val wikiPageWikiLinkProperty = context.ontology.properties("wikiPageWikiLink")
 
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/PersondataExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/PersondataExtractor.scala
@@ -19,7 +19,7 @@ class PersondataExtractor(
     def language : Language 
   }
 )
-extends Extractor
+extends Extractor[PageNode]
 {
     private val language = context.language
     private val wikiCode = language.wikiCode
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/PndExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/PndExtractor.scala
@@ -20,7 +20,7 @@ class PndExtractor (
     def language : Language
   }
 )
-extends Extractor
+extends Extractor[PageNode]
 {
   private val language = context.language
   
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/RedirectExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/RedirectExtractor.scala
@@ -14,7 +14,7 @@ class RedirectExtractor (
     def language : Language
   }
 )
-extends Extractor
+extends Extractor[PageNode]
 {
   private val language = context.language
   
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/RevisionIdExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/RevisionIdExtractor.scala
@@ -15,7 +15,7 @@ class RevisionIdExtractor (
     def language: Language
   }
 )
-extends Extractor
+extends Extractor[PageNode]
 {
   private val wikiPageRevisionIdProperty = context.ontology.properties("wikiPageRevisionID")
 
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/RevisionUriExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/RevisionUriExtractor.scala
@@ -15,7 +15,7 @@ class RevisionUriExtractor (
     def language: Language
   }
 )
-extends Extractor
+extends Extractor[PageNode]
 {
   private val derivedFromProperty = "http://www.w3.org/ns/prov#wasDerivedFrom"
 
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/SkosCategoriesExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/SkosCategoriesExtractor.scala
@@ -17,7 +17,7 @@ class SkosCategoriesExtractor(
     def language : Language
   }
 )
-extends Extractor
+extends Extractor[PageNode]
 {
   private val rdfTypeProperty = context.ontology.properties("rdf:type")
   private val skosConceptClass = context.ontology.classes("skos:Concept")
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/TemplateParameterExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/TemplateParameterExtractor.scala
@@ -15,7 +15,7 @@ class TemplateParameterExtractor(
     def language : Language 
   } 
 ) 
-extends Extractor
+extends Extractor[PageNode]
 {
   private val templateParameterProperty = context.language.propertyUri.append("templateUsesParameter")
 
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/TopicalConceptsExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/TopicalConceptsExtractor.scala
@@ -23,7 +23,7 @@ class TopicalConceptsExtractor(
     def language : Language
   }
 )
-extends Extractor
+extends Extractor[PageNode]
 {
     private val skosSubjectProperty = context.ontology.properties("skos:subject")
 
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/WikiPageExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/WikiPageExtractor.scala
@@ -15,7 +15,7 @@ class WikiPageExtractor(
     def language : Language
   }
 )
-extends Extractor
+extends Extractor[PageNode]
 {
   // We used foaf:page here, but foaf:isPrimaryTopicOf is probably better.
   private val isPrimaryTopicOf = context.ontology.properties("foaf:isPrimaryTopicOf")
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/WikidataExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/WikidataExtractor.scala
@@ -16,7 +16,7 @@ class WikidataExtractor(
                            def language : Language
                          }
                          )
-  extends Extractor
+  extends Extractor[PageNode]
 {
   // Here we define all the ontology predicates we will use
   private val isPrimaryTopicOf = context.ontology.properties("foaf:isPrimaryTopicOf")
diff --git a/core/src/main/scala/org/dbpedia/extraction/mappings/fr/PopulationExtractor.scala b/core/src/main/scala/org/dbpedia/extraction/mappings/fr/PopulationExtractor.scala
@@ -18,7 +18,7 @@ class PopulationExtractor(
     	def redirects : Redirects
     }
 )
-extends Extractor {
+extends Extractor[PageNode] {
     private val populationProperty = context.ontology.properties("populationTotal")
     private val populationRegex = """pop=(\d+)""".r
     override val datasets = Set(DBpediaDatasets.FrenchPopulation)
diff --git a/dump/src/main/scala/org/dbpedia/extraction/dump/extract/Config.scala b/dump/src/main/scala/org/dbpedia/extraction/dump/extract/Config.scala
@@ -63,7 +63,7 @@ extends ConfigParser(config)
    *
    * @return A Map which contains the extractor classes for each language
    */
-  private def loadExtractorClasses() : Map[Language, List[Class[_ <: Extractor]]] =
+  private def loadExtractorClasses() : Map[Language, List[Class[_ <: Extractor[_]]]] =
   {
     val languages = loadLanguages()
 
@@ -73,7 +73,8 @@ extends ConfigParser(config)
     val stdExtractors = splitValue("extractors", ',').map(loadExtractorClass)
 
     //Create extractor map
-    val classes = new HashMap[Language, List[Class[_ <: Extractor]]]()
+    // type of Extractor would be 'any' because they are different types of extractors and not categorized yet
+    val classes = new HashMap[Language, List[Class[_ <: Extractor[_]]]]()
     
     /*
     TODO: maybe we should check in the first loop if property "extractors."+language.wikiCode
@@ -142,10 +143,10 @@ extends ConfigParser(config)
     SortedSet[Language](languages.toSeq: _*)
   }
 
-  private def loadExtractorClass(name: String): Class[_ <: Extractor] = {
-    val className = if (! name.contains(".")) classOf[Extractor].getPackage.getName+'.'+name else name
+  private def loadExtractorClass(name: String): Class[_ <: Extractor[_]] = {
+    val className = if (! name.contains(".")) classOf[Extractor[_]].getPackage.getName+'.'+name else name
     // TODO: class loader of Extractor.class is probably wrong for some users.
-    classOf[Extractor].getClassLoader.loadClass(className).asSubclass(classOf[Extractor])
+    classOf[Extractor[_]].getClassLoader.loadClass(className).asSubclass(classOf[Extractor[_]])
   }
   
 }
diff --git a/dump/src/main/scala/org/dbpedia/extraction/dump/extract/ConfigLoader.scala b/dump/src/main/scala/org/dbpedia/extraction/dump/extract/ConfigLoader.scala
@@ -44,7 +44,7 @@ class ConfigLoader(config: Config)
     /**
      * Creates ab extraction job for a specific language.
      */
-    private def createExtractionJob(lang : Language, extractorClasses: List[Class[_ <: Extractor]], parser : WikiParser) : ExtractionJob =
+    private def createExtractionJob(lang : Language, extractorClasses: Seq[Class[_ <: Extractor[_]]], parser : WikiParser) : ExtractionJob =
     {
         val finder = new Finder[File](config.dumpDir, lang, config.wikiName)
 

Original file line number	Diff line number	Diff line change
`@@ -24,7 +24,7 @@ class AbstractExtractor(`
`24`	`24`	`def language : Language`
`25`	`25`	`}`
`26`	`26`	`)`
`27`		`-extends Extractor`
	`27`	`+extends Extractor[PageNode]`
`28`	`28`	`{`
`29`	`29`	`private val maxRetries = 3`
`30`	`30`
Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,7 @@ import org.dbpedia.extraction.wikiparser._`
`11`	`11`	`*/`
`12`	`12`	`class ArticleCategoriesExtractor( context : {`
`13`	`13`	`def ontology : Ontology`
`14`		`- def language : Language } ) extends Extractor`
	`14`	`+ def language : Language } ) extends Extractor[PageNode]`
`15`	`15`	`{`
`16`	`16`	`private val dctermsSubjectProperty = context.ontology.properties("dct:subject")`
`17`	`17`
Original file line number	Diff line number	Diff line change
`@@ -17,7 +17,7 @@ class ArticleTemplatesExtractor(`
`17`	`17`	`def language: Language`
`18`	`18`	`def redirects: Redirects`
`19`	`19`	`}`
`20`		`- ) extends Extractor {`
	`20`	`+ ) extends Extractor[PageNode] {`
`21`	`21`
`22`	`22`	`// FIXME: this uses the http://xx.dbpedia.org/property/ namespace, but the`
`23`	`23`	`// http://dbpedia.org/ontology/ namespace would probably make more sense.`
Original file line number	Diff line number	Diff line change
`@@ -16,7 +16,7 @@ import java.net.{URLEncoder, URI}`
`16`	`16`
`17`	`17`	`class ContributorExtractor( context : {`
`18`	`18`	`def ontology : Ontology`
`19`		`- def language : Language } ) extends Extractor`
	`19`	`+ def language : Language } ) extends Extractor[PageNode]`
`20`	`20`	`{`
`21`	`21`
`22`	`22`	`override val datasets = Set(DBpediaDatasets.RevisionMeta)`
Original file line number	Diff line number	Diff line change
`@@ -16,7 +16,7 @@ class DisambiguationExtractor(`
`16`	`16`	`def language : Language`
`17`	`17`	`}`
`18`	`18`	`)`
`19`		`-extends Extractor`
	`19`	`+extends Extractor[PageNode]`
`20`	`20`	`{`
`21`	`21`	`private val language = context.language`
`22`	`22`
Original file line number	Diff line number	Diff line change
`@@ -16,7 +16,7 @@ class ExternalLinksExtractor (`
`16`	`16`	`def language : Language`
`17`	`17`	`}`
`18`	`18`	`)`
`19`		`-extends Extractor`
	`19`	`+extends Extractor[PageNode]`
`20`	`20`	`{`
`21`	`21`	`val wikiPageExternalLinkProperty = context.ontology.properties("wikiPageExternalLink")`
`22`	`22`
Original file line number	Diff line number	Diff line change
`@@ -17,7 +17,7 @@ class FlickrWrapprLinkExtractor (`
`17`	`17`	`def language : Language`
`18`	`18`	`}`
`19`	`19`	`)`
`20`		`-extends Extractor`
	`20`	`+extends Extractor[PageNode]`
`21`	`21`	`{`
`22`	`22`	`private val language = context.language`
`23`	`23`