init

blogfetch.cr

@@ -0,0 +1,164 @@
+#!/usr/bin/env crystal
+
+require "http/client"
+require "uri"
+require "set"
+
+USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64; rv:154.0) Gecko/20100101 Firefox/154.0"
+MAX_BLOGS = 5000
+
+def fetch(url : String) : String?
+  begin
+    response = HTTP::Client.get(
+      url,
+      headers: HTTP::Headers{
+        "User-Agent" => USER_AGENT,
+      }
+    )
+
+    return nil unless response.success?
+
+    response.body
+  rescue
+    nil
+  end
+end
+
+def blogspot_host(url : String) : String?
+  begin
+    host = URI.parse(url).host
+    return nil unless host
+
+    host = host.downcase
+
+    blogspot_domains = [
+      "blogspot.com",
+      "blogspot.co.uk",
+      "blogspot.de",
+      "blogspot.fr",
+      "blogspot.ca",
+      "blogspot.com.au",
+      "blogspot.in",
+      "blogspot.it",
+      "blogspot.nl",
+      "blogspot.es",
+    ]
+
+    blogspot_domains.each do |domain|
+      if host == domain || host.ends_with?(".#{domain}")
+        return host
+      end
+    end
+
+    nil
+  rescue
+    nil
+  end
+end
+
+def blog_root(url : String) : String?
+  host = blogspot_host(url)
+  return nil unless host
+
+  "https://#{host}/"
+end
+
+def make_absolute(href : String, base : URI) : String?
+  href = href.strip
+
+  return nil if href.empty?
+  return nil if href.starts_with?("#")
+  return nil if href.starts_with?("javascript:")
+  return nil if href.starts_with?("mailto:")
+  return nil if href.starts_with?("tel:")
+  return nil if href.starts_with?("data:")
+
+  begin
+    uri = URI.parse(href)
+
+    if uri.scheme && uri.host
+      return href
+    end
+
+    host = base.host
+    return nil unless host
+
+    scheme = base.scheme || "https"
+
+    if href.starts_with?("/")
+      return "#{scheme}://#{host}#{href}"
+    end
+
+    base_path = base.path
+    base_path = "/" if base_path.nil? || base_path.empty?
+
+    directory =
+      if base_path.ends_with?("/")
+        base_path
+      else
+        base_path.sub(/\/[^\/]*$/, "/")
+      end
+
+    "#{scheme}://#{host}#{directory}#{href}"
+  rescue
+    nil
+  end
+end
+
+def extract_blog_roots(html : String, page_url : String) : Array(String)
+  results = Set(String).new
+  base = URI.parse(page_url)
+
+  html.scan(/<a\b[^>]*\bhref\s*=\s*["']([^"']+)["']/im) do |match|
+    href = match[1]
+
+    if absolute = make_absolute(href, base)
+      if root = blog_root(absolute)
+        results << root
+      end
+    end
+  end
+
+  results.to_a
+end
+
+def crawl(start_urls : Array(String))
+  queue = Array(String).new
+  discovered = Set(String).new
+  crawled = Set(String).new
+
+  start_urls.each do |url|
+    if root = blog_root(url)
+      unless discovered.includes?(root)
+        discovered << root
+        queue << root
+      end
+    end
+  end
+
+  while !queue.empty? && crawled.size < MAX_BLOGS
+    root = queue.shift
+
+    next if crawled.includes?(root)
+
+    crawled << root
+
+    puts root
+
+    html = fetch(root)
+    next unless html
+
+    extract_blog_roots(html, root).each do |new_root|
+      next if discovered.includes?(new_root)
+
+      discovered << new_root
+      queue << new_root
+    end
+  end
+end
+
+if ARGV.empty?
+  exit 1
+end
+
+crawl(ARGV.to_a)