#!/usr/bin/env crystal

require "http/client"
require "uri"
require "set"

USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64; rv:154.0) Gecko/20100101 Firefox/154.0"
MAX_BLOGS = 5000

def fetch(url : String) : String?
  begin
    response = HTTP::Client.get(
      url,
      headers: HTTP::Headers{
        "User-Agent" => USER_AGENT,
      }
    )

    return nil unless response.success?

    response.body
  rescue
    nil
  end
end

def blogspot_host(url : String) : String?
  begin
    host = URI.parse(url).host
    return nil unless host

    host = host.downcase

    blogspot_domains = [
      "blogspot.com",
      "blogspot.co.uk",
      "blogspot.de",
      "blogspot.fr",
      "blogspot.ca",
      "blogspot.com.au",
      "blogspot.in",
      "blogspot.it",
      "blogspot.nl",
      "blogspot.es",
    ]

    blogspot_domains.each do |domain|
      if host == domain || host.ends_with?(".#{domain}")
        return host
      end
    end

    nil
  rescue
    nil
  end
end

def blog_root(url : String) : String?
  host = blogspot_host(url)
  return nil unless host

  "https://#{host}/"
end

def make_absolute(href : String, base : URI) : String?
  href = href.strip

  return nil if href.empty?
  return nil if href.starts_with?("#")
  return nil if href.starts_with?("javascript:")
  return nil if href.starts_with?("mailto:")
  return nil if href.starts_with?("tel:")
  return nil if href.starts_with?("data:")

  begin
    uri = URI.parse(href)

    if uri.scheme && uri.host
      return href
    end

    host = base.host
    return nil unless host

    scheme = base.scheme || "https"

    if href.starts_with?("/")
      return "#{scheme}://#{host}#{href}"
    end

    base_path = base.path
    base_path = "/" if base_path.nil? || base_path.empty?

    directory =
      if base_path.ends_with?("/")
        base_path
      else
        base_path.sub(/\/[^\/]*$/, "/")
      end

    "#{scheme}://#{host}#{directory}#{href}"
  rescue
    nil
  end
end

def extract_blog_roots(html : String, page_url : String) : Array(String)
  results = Set(String).new
  base = URI.parse(page_url)

  html.scan(/<a\b[^>]*\bhref\s*=\s*["']([^"']+)["']/im) do |match|
    href = match[1]

    if absolute = make_absolute(href, base)
      if root = blog_root(absolute)
        results << root
      end
    end
  end

  results.to_a
end

def crawl(start_urls : Array(String))
  queue = Array(String).new
  discovered = Set(String).new
  crawled = Set(String).new

  start_urls.each do |url|
    if root = blog_root(url)
      unless discovered.includes?(root)
        discovered << root
        queue << root
      end
    end
  end

  while !queue.empty? && crawled.size < MAX_BLOGS
    root = queue.shift

    next if crawled.includes?(root)

    crawled << root

    puts root

    html = fetch(root)
    next unless html

    extract_blog_roots(html, root).each do |new_root|
      next if discovered.includes?(new_root)

      discovered << new_root
      queue << new_root
    end
  end
end

if ARGV.empty?
  exit 1
end

crawl(ARGV.to_a)