scrape2.clj

(ns tutorial.scrape2
  (:require [net.cgrand.enlive-html :as html]))

(def ^:dynamic *base-url* "https://news.ycombinator.com/")

(defn fetch-url [url]
  (with-open [inputstream (-> (java.net.URL. url)
                            .openConnection
                            (doto (.setRequestProperty "User-Agent"
                                                       "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1;)"))
                            .getContent)]
  (html/html-resource inputstream)))

(defn hn-headlines-and-points []
  (map html/text
       (html/select (fetch-url *base-url*)
                    #{[:td.title :a] [:td.subtext html/first-child]})))

(defn print-headlines-and-points []
  (doseq [line (map (fn [[h s]] (str h " (" s ")"))
                    (partition 2 (hn-headlines-and-points)))]
    (println line)))