#! ruby # License: PUBLIC DOMAIN begin require 'markdown-tables' require "net/http" require 'nokogumbo' require 'pp' require "uri" rescue LoadError => e puts "Missing dependency, try 'gem install #{e.path}'" exit 1 end @pkgs_org_uri_base = URI.parse('https://pkgs.org') def load_from_pkgs_org(config) types = config.keys res = {} Net::HTTP.start(@pkgs_org_uri_base.host, use_ssl: true) do |http| # Handle CSRF protection request = Net::HTTP::Get.new(@pkgs_org_uri_base) response = http.request(request) cookie = response.response['set-cookie'] csrftoken = cookie.split(';').map { |kv| kv.split('=') }.find { |kv| kv.first == 'csrftoken' }.last post_form_data = { 'csrfmiddlewaretoken' => csrftoken } types.each do |type| res[type] = {} config[type][:queries].each do |query| uri = @pkgs_org_uri_base.clone uri.path += query request = Net::HTTP::Post.new(uri) request['content-type'] = 'application/x-www-form-urlencoded' request['cookie'] = cookie request['referer'] = uri.to_s request.form_data = post_form_data response = http.request(request) document = Nokogiri::HTML5(response.body) document.search('div.card').each do |card| distro = card.search('a.card-title').first.text res[type][distro] = [] unless res[type].has_key?(distro) card.search('div.card-body a').each do |pkg| res[type][distro] << pkg.text end end end end end res end def extract_versions(config, data) types = config.keys res = {} types.each do |type| res[type] = {} data[type].each do |distro, pkgs| res[type][distro] = [] unless res[type].has_key?(distro) pkgs.each do |pkg| # res[type][distro] << pkg pkg.match(config[type][:regex]) { |m| res[type][distro] << m.captures.last } end res[type][distro].sort!.uniq! end end res end def print_table(config, data) types = config.keys labels = ['distro'] types.each { |type| labels << type.to_s } table_data = [] types = config.keys distros = [] types.each { |type| distros.concat(data[type].keys) } distros.sort!.uniq! distros.each do |distro| row = [distro] types.each do |type| row << data[type][distro].sort_by { |v| Gem::Version.new(v) }.join(', ') end table_data << row end puts MarkdownTables.make_table(labels, table_data, is_rows: true) end ########### config = {} config[:boost] = { queries: [ '/download/boost' ], regex: /^(lib)?boost[-_a-z0-9]*(1\.[0-9]{2}\.[0-9]{1,2})/ } config[:cmake] = { queries: [ '/download/cmake' ], regex: /^cmake3?[-_]?([2-3]\.[0-9]{1,2}\.[0-9]{1,2})/ } raw_data = load_from_pkgs_org(config) version_data = extract_versions(config, raw_data) print_table(config, version_data)