Merge pull request #2 from mark-watson/improve-webscraper

mark-watson · web-flow · commit fbd0873237ee · 2025-05-19T14:52:34.000-07:00
Refactor and enhance webscraping utility.
diff --git a/webscraping/project.clj b/webscraping/project.clj
@@ -4,5 +4,5 @@
   :license {:name "EPL-2.0 OR GPL-2.0-or-later WITH Classpath-exception-2.0"
             :url "https://www.eclipse.org/legal/epl-2.0/"}
   :dependencies [[org.clojure/clojure "1.11.1"]
-                 [org.jsoup/jsoup "1.14.3"]]
+                 [org.jsoup/jsoup "1.17.2"]]
   :repl-options {:init-ns webscraping.core})
diff --git a/webscraping/src/webscraping/core.clj b/webscraping/src/webscraping/core.clj
@@ -5,18 +5,15 @@
 
 (defn get-html-anchors [jsoup-web-page-contents]
   (let [anchors (. jsoup-web-page-contents select "a[href]")]
-    (for [anchor anchors]
-      (try
-        (let [anchor-text (. (first (. anchor childNodes)) text)
-              anchor-uri-base (. (first (. anchor childNodes)) baseUri)
-              href-attribute (. (. anchor attributes) get "href")
-              anchor-uri
-              (if (str/starts-with? href-attribute "http")
-                href-attribute
-                (str/join "" [anchor-uri-base (. (. anchor attributes) get "href")]))
-              furi (first (. anchor childNodes))]
-          {:text (str/trim anchor-text) :uri anchor-uri})
-        (catch Exception e {:text (ex-message e) :uri ""})))))
+    (->> anchors
+         (map (fn [anchor]
+                (try
+                  {:text (str/trim (. anchor text))
+                   :uri (. anchor absUrl "href")}
+                  (catch Exception e
+                    (binding [*out* *err*] (println (str "Error processing anchor: " (.getMessage e) " on page: " (. jsoup-web-page-contents title))))
+                    nil))))
+         (filterv some?))))
 
 (defn fetch-web-page-data
   "Get the <a> anchor data and full text from a web URI"
diff --git a/webscraping/test/webscraping/core_test.clj b/webscraping/test/webscraping/core_test.clj
@@ -1,10 +1,49 @@
 (ns webscraping.core-test
   (:require [clojure.test :refer :all]
             [clojure.pprint :as pp]
-            [webscraping.core :refer :all]))
+            [clojure.string :as str] ;; Added for string checks
+            [webscraping.core :refer :all])
+  (:import (org.jsoup Jsoup))) ;; Added Jsoup import
 
-(deftest a-test
-  (testing
-    "Fetch my personal website and check number key/values in results"
+(deftest mark-watson-website-test
+  (testing "Fetch Mark Watson's website and verify basic anchor extraction"
     (let [page-data (fetch-web-page-data "https://markwatson.com")]
-      (pp/pprint page-data))))
+      (is (string? (:page-text page-data)))
+      (is (not (str/blank? (:page-text page-data))))
+      (is (vector? (:anchors page-data)))
+      (is (pos? (count (:anchors page-data))))
+      (let [anchors (:anchors page-data)]
+        (is (some #(str/includes? (:text %) "Read My Blog on Blogspot") anchors)
+            "Expected to find an anchor with 'Read My Blog on Blogspot' in its text")
+        (is (some #(= (:uri %) "https://mark-watson.blogspot.com/") anchors)
+            "Expected to find an anchor linking to https://mark-watson.blogspot.com/")
+
+        (is (some #(str/includes? (str/lower-case (:text %)) "clojure") anchors)
+            "Expected to find an anchor with 'Clojure' (case-insensitive) in its text")
+        (is (some #(= (:uri %) "https://leanpub.com/clojureai") anchors)
+            "Expected to find an anchor linking to Leanpub Clojure AI book")
+
+        ;; Add a check for one more reasonably stable link, e.g., "My Books"
+        (is (some #(and (str/includes? (:text %) "My Books")
+                        (= (:uri %) "https://markwatson.com#books"))
+                  anchors)
+            "Expected to find an anchor 'My Books' linking to '#books'")))))
+
+(deftest no-anchors-test
+  (testing "Page with no anchor tags"
+    (let [html-doc (Jsoup/parse "<html><body><p>No links here.</p></body></html>")
+          anchors (get-html-anchors html-doc)]
+      (is (empty? anchors) "Expected no anchors from HTML with no links"))))
+
+(deftest relative-and-absolute-uris-test
+  (testing "Anchor URI resolution for relative and absolute paths"
+    (let [base-uri "http://example.com/docs/"
+          html-content "<html><body><a href=\"/page1\">Page 1</a> <a href=\"http://domain.com/page2\">Page 2</a> <a href=\"../page3\">Page 3</a> <a href=\"sub/page4\">Page 4</a><a href=\"page5.html\">Page 5</a></body></html>"
+          html-doc (. Jsoup parse html-content base-uri)
+          anchors (get-html-anchors html-doc)
+          uris (set (map :uri anchors))]
+      (is (contains? uris "http://example.com/page1") "Relative /page1 should resolve to http://example.com/page1")
+      (is (contains? uris "http://domain.com/page2") "Absolute http://domain.com/page2 should remain unchanged")
+      (is (contains? uris "http://example.com/page3") "Relative ../page3 should resolve to http://example.com/page3")
+      (is (contains? uris "http://example.com/docs/sub/page4") "Relative sub/page4 should resolve to http://example.com/docs/sub/page4")
+      (is (contains? uris "http://example.com/docs/page5.html") "Relative page5.html should resolve to http://example.com/docs/page5.html"))))
