*(XWiki)UserLinkAdditionFinder: filter out government domains.

MER-C · MER-C · commit ec5a039e5075 · 2024-06-21T18:51:50.000+01:00
*Arabic Wikipedia main page was moved.
diff --git a/src/org/wikipedia/tools/UserLinkAdditionFinder.java b/src/org/wikipedia/tools/UserLinkAdditionFinder.java
@@ -1,6 +1,6 @@
 /**
- *  @(#)UserLinkAdditionFinder.java 0.02 05/11/2017
- *  Copyright (C) 2015-2017 MER-C
+ *  @(#)UserLinkAdditionFinder.java 0.03 15/06/2024
+ *  Copyright (C) 2015-2024 MER-C
  *
  *  This program is free software; you can redistribute it and/or
  *  modify it under the terms of the GNU General Public License
@@ -29,12 +29,15 @@
 /**
  *  Finds links added by a user in the main namespace.
  *  @author MER-C
- *  @version 0.02
+ *  @version 0.03
  */
 public class UserLinkAdditionFinder
 {
     private final WMFWiki wiki;
+    private final Pages pages;
+    private final ExternalLinks el;
     private static final WMFWikiFarm sessions = WMFWikiFarm.instance();
+    private final List<Pattern> whitelist_regexes = new ArrayList<>();
     
     /**
      *  Runs this program.
@@ -81,40 +84,19 @@ public static void main(String[] args) throws IOException
             System.out.println("No links found.");
             System.exit(0);
         }
-        
         Map<String, String> linkdomains = new HashMap<>();
         for (Map.Entry<Wiki.Revision, List<String>> entry : results.entrySet())
         {
             for (String link : entry.getValue())
             {
                 String domain = ExternalLinks.extractDomain(link);
-                if (domain != null) // must be parseable
-                {
-                    boolean nomatch = true;
-                    for (String wmfsite : WMFWikiFarm.WMF_DOMAINS)
-                        if (domain.endsWith(wmfsite))
-                            nomatch = false;
-                    if (nomatch)
-                        linkdomains.put(link, domain);
-                }
-            }
-        }
-        
-        // remove blacklisted links
-        Collection<String> domains = new TreeSet(linkdomains.values());
-        ExternalLinks el = ExternalLinks.of(thiswiki);
-        if (removeblacklisted)
-        {
-            Iterator<String> iter = domains.iterator();
-            while (iter.hasNext())
-            {
-                String link = iter.next();
-                if (el.isSpamBlacklisted(linkdomains.get(link)))
-                    iter.remove();
+                if (domain != null && !finder.canSkipDomain(domain, removeblacklisted))
+                    linkdomains.put(link, domain);
             }
         }
-        
+                
         // remove commonly used domains
+        Collection<String> domains = new TreeSet(linkdomains.values());
         Map<String, Integer> linkcounts = null;
         if (linksearch)
         {
@@ -149,6 +131,21 @@ public static void main(String[] args) throws IOException
     public UserLinkAdditionFinder(WMFWiki wiki)
     {
         this.wiki = wiki;
+        this.pages = Pages.of(wiki);
+        this.el = ExternalLinks.of(wiki);
+        
+        String[] regex = new String[] 
+        {
+            ".*\\.(?:gov|int|mil)$",
+            "(.*\\.)?gov\\.(?:au|br|cn|ie|in|il|ph|ru|scot|sg|ua|uk|wales|za)$",
+            "(.*\\.)?gob\\.(?:ar|cl|es|mx|pe)$",
+            "(.*\\.)?(?:bl|judiciary|mod|nhs|parliament|police|royal)\\.uk$",
+            "(.*\\.)?\\bgouv\\.fr",
+            "(.*\\.)?\\bgovt\\.nz$",
+            "(.*\\.)?\\beuropa\\.eu$"
+        };
+        for (String r : regex)
+            whitelist_regexes.add(Pattern.compile(r));
     }
     
     /**
@@ -191,6 +188,33 @@ public Map<Wiki.Revision, List<String>> getLinksAdded(List<String> users, Offset
         return results;
     }
     
+    /**
+     *  Filters added links for inclusion in search results. 
+     * 
+     *  @param domain the domain to check
+     *  @param removeblacklisted remove already blacklisted links
+     *  @return whether this domain can be skipped for the purpose of this
+     *  search
+     *  @throws IOException if a network error occurs when fetching the spam
+     *  blacklist (highly unlikely)
+     *  @since 0.03
+     */
+    public boolean canSkipDomain(String domain, boolean removeblacklisted) throws IOException
+    {
+        // WMF domains
+        for (String wmfsite : WMFWikiFarm.WMF_DOMAINS)
+            if (domain.endsWith(wmfsite))
+                return true;
+        // government domains
+        for (Pattern p : whitelist_regexes)
+            if (p.matcher(domain).matches())
+                return true;
+        // blacklisted domains
+        if (removeblacklisted && el.isSpamBlacklisted(domain))
+            return true;
+        return false;
+    }
+    
     /**
      *  For a map that contains revision data &#8594; links added in that 
      *  revision, check whether the links still exist in the current version of
@@ -216,7 +240,7 @@ public Map<String, Map<String, Boolean>> checkIfLinksAreStillPresent(Map<Wiki.Re
             }
             list.addAll(listoflinks);
         });
-        return Pages.of(wiki).containExternalLinks(resultsbypage);
+        return pages.containExternalLinks(resultsbypage);
     }
     
     public String outputWikitableResults(Map<Wiki.Revision, List<String>> data, 
diff --git a/src/org/wikipedia/tools/XWikiUserLinkAdditionFinder.java b/src/org/wikipedia/tools/XWikiUserLinkAdditionFinder.java
@@ -78,7 +78,7 @@ public static void main(String[] args) throws Exception
                     for (String link : entry.getValue())
                     {
                         String domain = ExternalLinks.extractDomain(link);
-                        if (domain != null) // must be parseable
+                        if (domain != null && !finder.canSkipDomain(domain, false)) // must be parseable
                             linkdomains.put(link, domain);
                     }
                 }
diff --git a/test/org/wikipedia/WikiTest.java b/test/org/wikipedia/WikiTest.java
@@ -369,7 +369,7 @@ public void resolveRedirects() throws Exception
         List<String> expected = List.of("Main Page", "Main Page", "Sdkghsdklg",
             "Goatse.cx", "Main Page", "Fish and chips", "Fish and chips");
         assertEquals(expected, enWiki.resolveRedirects(titles));
-        assertEquals(List.of("الصفحة الرئيسية"), arWiki.resolveRedirects(List.of("الصفحه الرئيسيه")), "rtl");
+        assertEquals(List.of("الصفحة الرئيسة"), arWiki.resolveRedirects(List.of("الصفحه الرئيسيه")), "rtl");
     }
 
     @Test
diff --git a/test/org/wikipedia/tools/UserLinkAdditionFinderTest.java b/test/org/wikipedia/tools/UserLinkAdditionFinderTest.java
@@ -54,6 +54,21 @@ public void getWiki() throws Exception
         assertEquals("test.wikipedia.org", finder_test.getWiki().getDomain());
         assertEquals("en.wikipedia.org", finder_en.getWiki().getDomain());
     }
+    
+    @Test
+    public void canSkipDomain() throws Exception
+    {
+        List<String> domains = List.of("en.wikipedia.org", "www.fda.gov", "nasa.gov",
+            "army.mil", "www.consilium.europa.eu", "gov.uk", "wa.gov.au", "europa.eu",
+            "govt.nz", "www.govt.nz", "bl.uk", "parliament.uk", "un.int");
+        for (String domain : domains)
+            assertTrue(finder_en.canSkipDomain(domain, true), "domain: " + domain);
+        
+        domains = List.of("www.example.com", "blah.gov.invalid", "fakegov.uk", "blahbl.uk",
+            "fake-gov.uk");
+        for (String domain : domains)
+            assertFalse(finder_en.canSkipDomain(domain, true), "domain: " + domain);
+    }
 
     @Test
     public void getLinksAdded() throws Exception

Original file line number	Diff line number	Diff line change
`@@ -78,7 +78,7 @@ public static void main(String[] args) throws Exception`
`78`	`78`	`for (String link : entry.getValue())`
`79`	`79`	`{`
`80`	`80`	`String domain = ExternalLinks.extractDomain(link);`
`81`		`- if (domain != null) // must be parseable`
	`81`	`+ if (domain != null && !finder.canSkipDomain(domain, false)) // must be parseable`
`82`	`82`	`linkdomains.put(link, domain);`
`83`	`83`	`}`
`84`	`84`	`}`
Original file line number	Diff line number	Diff line change
`@@ -369,7 +369,7 @@ public void resolveRedirects() throws Exception`
`369`	`369`	`List<String> expected = List.of("Main Page", "Main Page", "Sdkghsdklg",`
`370`	`370`	`"Goatse.cx", "Main Page", "Fish and chips", "Fish and chips");`
`371`	`371`	`assertEquals(expected, enWiki.resolveRedirects(titles));`
`372`		`- assertEquals(List.of("الصفحة الرئيسية"), arWiki.resolveRedirects(List.of("الصفحه الرئيسيه")), "rtl");`
	`372`	`+ assertEquals(List.of("الصفحة الرئيسة"), arWiki.resolveRedirects(List.of("الصفحه الرئيسيه")), "rtl");`
`373`	`373`	`}`
`374`	`374`
`375`	`375`	`@Test`