Merge branch 'master' of https://github.com/dbpedia/extraction-framework

jcsahnwaldt · jcsahnwaldt · commit 240df02b4765 · 2014-02-07T13:43:10.000+01:00
diff --git a/core/src/main/scala/org/dbpedia/extraction/util/WikiApi.scala b/core/src/main/scala/org/dbpedia/extraction/util/WikiApi.scala
@@ -111,15 +111,15 @@ class WikiApi(url: URL, language: Language)
     /**
      * Retrieves multiple pages by their title.
      *
-     * @param pageIds The titles of the pages to be downloaded.
+     * @param titles The titles of the pages to be downloaded.
      */
     def retrievePagesByTitle[U](titles : Iterable[WikiTitle]) = new Traversable[WikiPage]
     {
         override def foreach[U](proc : WikiPage => U) : Unit =
         {
             for(titleGroup <- titles.grouped(pageDownloadLimit))
             {
-                val response = query("?action=query&format=xml&prop=revisions&titles=" + titleGroup.map(t => URLEncoder.encode(t.encodedWithNamespace, "UTF-8")).mkString("|") + "&rvprop=ids|content|timestamp|user|userid")
+                val response = query("?action=query&format=xml&prop=revisions&titles=" + titleGroup.map(formatWikiTitle).mkString("|") + "&rvprop=ids|content|timestamp|user|userid")
                 processPages(response, proc)
             }
         }
@@ -138,7 +138,7 @@ class WikiApi(url: URL, language: Language)
 
         proc(
           new WikiPage(
-            title        = WikiTitle.parse((page \ "@title").head.text, language),
+            title           = WikiTitle.parse((page \ "@title").head.text, language),
             redirect        = null, // TODO: read redirect from XML
             id              = (page \ "@pageid").head.text,
             revision        = (rev \ "@revid").head.text,
@@ -243,4 +243,14 @@ class WikiApi(url: URL, language: Language)
 
         throw new IllegalStateException("Should never get there")
     }
+
+  /**
+   * Formats {@param title} to be used with MediaWiki API
+   *
+   * @param title
+   * @return
+   */
+    private def formatWikiTitle(title: WikiTitle) : String = {
+        URLEncoder.encode(title.decodedWithNamespace.replace(' ', '_'), "UTF-8")
+    }
 }

Original file line number	Diff line number	Diff line change
`@@ -111,15 +111,15 @@ class WikiApi(url: URL, language: Language)`
`111`	`111`	`/**`
`112`	`112`	`* Retrieves multiple pages by their title.`
`113`	`113`	`*`
`114`		`- * @param pageIds The titles of the pages to be downloaded.`
	`114`	`+ * @param titles The titles of the pages to be downloaded.`
`115`	`115`	`*/`
`116`	`116`	`def retrievePagesByTitle[U](titles : Iterable[WikiTitle]) = new Traversable[WikiPage]`
`117`	`117`	`{`
`118`	`118`	`override def foreach[U](proc : WikiPage => U) : Unit =`
`119`	`119`	`{`
`120`	`120`	`for(titleGroup <- titles.grouped(pageDownloadLimit))`
`121`	`121`	`{`
`122`		`- val response = query("?action=query&format=xml&prop=revisions&titles=" + titleGroup.map(t => URLEncoder.encode(t.encodedWithNamespace, "UTF-8")).mkString("\|") + "&rvprop=ids\|content\|timestamp\|user\|userid")`
	`122`	`+ val response = query("?action=query&format=xml&prop=revisions&titles=" + titleGroup.map(formatWikiTitle).mkString("\|") + "&rvprop=ids\|content\|timestamp\|user\|userid")`
`123`	`123`	`processPages(response, proc)`
`124`	`124`	`}`
`125`	`125`	`}`
`@@ -138,7 +138,7 @@ class WikiApi(url: URL, language: Language)`
`138`	`138`
`139`	`139`	`proc(`
`140`	`140`	`new WikiPage(`
`141`		`- title = WikiTitle.parse((page \ "@title").head.text, language),`
	`141`	`+ title = WikiTitle.parse((page \ "@title").head.text, language),`
`142`	`142`	`redirect = null, // TODO: read redirect from XML`
`143`	`143`	`id = (page \ "@pageid").head.text,`
`144`	`144`	`revision = (rev \ "@revid").head.text,`
`@@ -243,4 +243,14 @@ class WikiApi(url: URL, language: Language)`
`243`	`243`
`244`	`244`	`throw new IllegalStateException("Should never get there")`
`245`	`245`	`}`
	`246`	`+`
	`247`	`+ /**`
	`248`	`+ * Formats {@param title} to be used with MediaWiki API`
	`249`	`+ *`
	`250`	`+ * @param title`
	`251`	`+ * @return`
	`252`	`+ */`
	`253`	`+ private def formatWikiTitle(title: WikiTitle) : String = {`
	`254`	`+ URLEncoder.encode(title.decodedWithNamespace.replace(' ', '_'), "UTF-8")`
	`255`	`+ }`
`246`	`256`	`}`