Query packaged versions of CMake and Boost on various distros tracked by pkgs.org

#! ruby
# License: PUBLIC DOMAIN

begin
  require 'markdown-tables'
  require "net/http"
  require 'nokogumbo'
  require 'pp'
  require "uri"
rescue LoadError => e
  puts "Missing dependency, try 'gem install #{e.path}'"
  exit 1
end

@pkgs_org_uri_base = URI.parse('https://pkgs.org')

def load_from_pkgs_org(config)
  types = config.keys
  res = {}
  Net::HTTP.start(@pkgs_org_uri_base.host, use_ssl: true) do |http|
    # Handle CSRF protection
    request = Net::HTTP::Get.new(@pkgs_org_uri_base)
    response = http.request(request)
    cookie = response.response['set-cookie']
    csrftoken = cookie.split(';').map { |kv| kv.split('=') }.find { |kv| kv.first == 'csrftoken' }.last
    post_form_data = { 'csrfmiddlewaretoken' => csrftoken }

    types.each do |type|
      res[type] = {}
      config[type][:queries].each do |query|
        uri = @pkgs_org_uri_base.clone
        uri.path += query
        request = Net::HTTP::Post.new(uri)
        request['content-type'] = 'application/x-www-form-urlencoded'
        request['cookie'] = cookie
        request['referer'] = uri.to_s
        request.form_data = post_form_data
        response = http.request(request)

        document = Nokogiri::HTML5(response.body)
        document.search('div.card').each do |card|
          distro = card.search('a.card-title').first.text
          res[type][distro] = [] unless res[type].has_key?(distro)
          card.search('div.card-body a').each do |pkg|
            res[type][distro] << pkg.text
          end
        end
      end
    end
  end
  res
end

def extract_versions(config, data)
  types = config.keys
  res = {}
  types.each do |type|
    res[type] = {}
    data[type].each do |distro, pkgs|
      res[type][distro] = [] unless res[type].has_key?(distro)
      pkgs.each do |pkg|
        # res[type][distro] << pkg
        pkg.match(config[type][:regex]) { |m|
          res[type][distro] << m.captures.last
        }
      end
      res[type][distro].sort!.uniq!
    end
  end
  res
end

def print_table(config, data)
  types = config.keys

  labels = ['distro']
  types.each { |type| labels << type.to_s }
  table_data = []
  types = config.keys

  distros = []
  types.each { |type| distros.concat(data[type].keys) }
  distros.sort!.uniq!

  distros.each do |distro|
    row = [distro]
    types.each do |type|
      row << data[type][distro].sort_by { |v| Gem::Version.new(v) }.join(', ')
    end
    table_data << row
  end

  puts MarkdownTables.make_table(labels, table_data, is_rows: true)
end

###########

config = {}
config[:boost] = {
  queries: [ '/download/boost' ],
  regex: /^(lib)?boost[-_a-z0-9]*(1\.[0-9]{2}\.[0-9]{1,2})/
}
config[:cmake] = {
  queries: [ '/download/cmake' ],
  regex: /^cmake3?[-_]?([2-3]\.[0-9]{1,2}\.[0-9]{1,2})/
}

raw_data = load_from_pkgs_org(config)

version_data = extract_versions(config, raw_data)

print_table(config, version_data)