Rewrite parser for iv-org/documentation#76

Dockerfile

@@ -3,14 +3,15 @@ WORKDIR /app
 COPY ./shard.yml ./shard.yml
 RUN shards install
 COPY ./src/ ./src/
-RUN crystal build ./src/instances.cr --release
+RUN crystal build ./src/instances.cr -s -p -t
 
 FROM alpine:latest
-RUN apk add --no-cache gc pcre libgcc
+RUN apk add --no-cache gc pcre libgcc yaml
 WORKDIR /app
 RUN addgroup -g 1000 -S invidious && \
     adduser -u 1000 -S invidious -G invidious
 COPY ./assets/ ./assets/
+COPY ./config.yml ./config.yml
 COPY --from=builder /app/instances .
 
 EXPOSE 3000

config.yml

@@ -0,0 +1,9 @@
+# Uses the system's CURL binary to do so.
+fetch_onion_instance_stats: true
+
+# TOR's Sock proxy address that CURL uses to connect to hidden services
+tor_sock_proxy_address: "127.0.0.1"
+tor_sock_proxy_port: 9050
+
+# Minutes before refreshing the instance stats
+minutes_between_refresh: 30

docker-compose.yml

@@ -1,5 +1,13 @@
 version: '3'
 services:
+  tor-socks-proxy:
+    container_name: tor-socks-proxy
+    image: peterdavehello/tor-socks-proxy:latest
+    ports:
+      - "127.0.0.1:8853:53/udp"
+      - "127.0.0.1:9150:9150/tcp"
+    restart: unless-stopped
+
   instances:
     build: .
     restart: unless-stopped

src/fetch.cr

@@ -0,0 +1,125 @@
+# Fetch a country's emoji flag from their country code (ISO 3166 alpha-2).
+#
+# A flag is made out of two regional indicator symbols.
+# So in order to convert from an ISO 3166 alpha-2 code into unicode we'll have to
+# add a specific offset to each character to make them into the required regional
+# indicator symbols. This offset is exactly 0x1f1a5.
+#
+# Reference implementation https://schinckel.net/2015/10/29/unicode-flags-in-python/
+private def fetch_flag(country_code)
+  return country_code.codepoints.map { |codepoint| (codepoint + 0x1f1a5).chr }.join("")
+end
+
+# Extracts the nested modified information containing source url and changes.
+private def extract_modified_information(modified_hash)
+  if modified = modified_hash.as_h?
+    return Modified.new(
+      source: modified["source"].as_s,
+      changes: modified["changes"].as_s
+    )
+  end
+end
+
+# Extracts information common to all instance types.
+private def extract_prerequisites(instance_data)
+  uri = URI.parse(instance_data["url"].to_s)
+  host = uri.host
+
+  # Fetch country data
+  region = instance_data["country"].to_s
+  flag = fetch_flag(region)
+
+  privacy_policy = instance_data["privacy_policy"].as_s?
+  owner = {name: instance_data["owner"].to_s.split("/")[-1].to_s, url: instance_data["owner"].as_s}
+  modified = extract_modified_information(instance_data["modified"])
+  notes = instance_data["notes"].as_a?
+
+  mirrors = [] of Mirrors
+  instance_data["mirrors"].as_a?.try &.each do |m|
+    mirrors << Mirrors.new(
+      url: m["url"].as_s,
+      region: m["country"].as_s,
+      flag: fetch_flag(m["country"].as_s)
+    )
+  end
+
+  return uri, host, region, flag, privacy_policy, owner, modified, notes, mirrors
+end
+
+def prepare_http_instance(instance_data, instances_storage, monitors)
+  uri, host, region, flag, privacy_policy, owner, modified, notes, mirrors = extract_prerequisites(instance_data)
+
+  # Fetch status information
+  if status = instance_data["status"].as_h?
+    status_url = status["url"].as_s
+  else
+    status_url = nil
+  end
+
+  ddos_mitm_protection = instance_data["ddos_mitm_protection"].as_s?
+
+  client = HTTP::Client.new(uri)
+  client.connect_timeout = 5.seconds
+  client.read_timeout = 5.seconds
+
+  begin
+    stats = JSON.parse(client.get("/api/v1/stats").body)
+  rescue ex
+    stats = nil
+  end
+
+  monitor = monitors.try &.select { |monitor| monitor["name"].try &.as_s == host }[0]?
+  return {
+    region:               region,
+    flag:                 flag,
+    stats:                stats,
+    type:                 "https",
+    uri:                  uri.to_s,
+    status_url:           status_url,
+    privacy_policy:       privacy_policy,
+    ddos_mitm_protection: ddos_mitm_protection,
+    owner:                owner,
+    modified:             modified,
+    mirrors:              mirrors,
+    notes:                notes,
+    monitor:              monitor || instances_storage[host]?.try &.[:monitor]?,
+  }
+end
+
+def prepare_onion_instance(instance_data, instances_storage)
+  uri, host, region, flag, privacy_policy, owner, modified, notes, mirrors = extract_prerequisites(instance_data)
+
+  associated_clearnet_instance = instance_data["associated_clearnet_instance"].as_s?
+
+  if CONFIG["fetch_onion_instance_stats"]?
+    begin
+      args = Process.parse_arguments("--socks5-hostname '#{CONFIG["tor_sock_proxy_address"]}:#{CONFIG["tor_sock_proxy_port"]}' 'http://#{uri.host}/api/v1/stats'")
+      response = nil
+      Process.run("curl", args: args) do |result|
+        data = result.output.read_line
+        response = JSON.parse(data)
+      end
+
+      stats = response
+    rescue ex
+      stats = nil
+    end
+  else
+    stats = nil
+  end
+
+  return {
+    region:                       region,
+    flag:                         flag,
+    stats:                        stats,
+    type:                         "onion",
+    uri:                          uri.to_s,
+    associated_clearnet_instance: associated_clearnet_instance,
+    privacy_policy:               privacy_policy,
+    owner:                        owner,
+    modified:                     modified,
+    mirrors:                      mirrors,
+    notes:                        notes,
+    monitor:                      nil,
+  }
+end

src/helpers/helpers.cr

@@ -0,0 +1,9 @@
+require "yaml"
+
+def load_config
+  return YAML.parse(File.read("config.yml"))
+end
+
+def load_instance_yaml(contents)
+  return YAML.parse(contents)
+end

src/instances.cr

@@ -18,15 +18,52 @@ require "http/client"
 require "kemal"
 require "uri"
 
+require "./fetch.cr"
+require "./helpers/*"
+
+CONFIG = load_config()
+
 Kemal::CLI.new ARGV
 
 macro rendered(filename)
   render "src/instances/views/#{{{filename}}}.ecr"
 end
 
-alias Instance = NamedTuple(flag: String?, region: String?, stats: JSON::Any?, type: String, uri: String, monitor: JSON::Any?)
-
-INSTANCES = {} of String => Instance
+# Nested data within instances
+alias Owner = NamedTuple(name: String, url: String)
+alias Modified = NamedTuple(source: String, changes: String)
+alias Mirrors = NamedTuple(url: String, region: String, flag: String)
+
+alias ClearNetInstance = NamedTuple(
+  flag: String,
+  region: String,
+  stats: JSON::Any?,
+  type: String,
+  uri: String,
+  status_url: String?,
+  privacy_policy: String?,
+  ddos_mitm_protection: String?,
+  owner: Owner,
+  modified: Modified?,
+  mirrors: Array(Mirrors)?,
+  notes: Array(YAML::Any)?,
+  monitor: JSON::Any?)
+
+alias OnionInstance = NamedTuple(
+  flag: String,
+  region: String,
+  stats: JSON::Any?,
+  type: String,
+  uri: String,
+  associated_clearnet_instance: String?,
+  privacy_policy: String?,
+  owner: Owner,
+  modified: Modified?,
+  mirrors: Array(Mirrors)?,
+  notes: Array(YAML::Any)?,
+  monitor: JSON::Any?)
+
+INSTANCES = {} of String => ClearNetInstance | OnionInstance
 
 spawn do
   loop do
@@ -50,45 +87,25 @@ spawn do
         break
       end
     end
+
     begin
-      body = HTTP::Client.get(URI.parse("https://raw.githubusercontent.com/iv-org/documentation/master/Invidious-Instances.md")).body
+      # Needs to be replaced once merged!
+      body = HTTP::Client.get(URI.parse("https://raw.githubusercontent.com/syeopite/documentation/alt-instance-list/instances.yaml")).body
     rescue ex
       body = ""
     end
 
-    instances = {} of String => Instance
-
-    body = body.split("### Blocked:")[0]
-    body.scan(/\[(?<host>[^ \]]+)\]\((?<uri>[^\)]+)\)( .(?<region>[\x{1f100}-\x{1f1ff}]{2}))?/mx).each do |md|
-      region = md["region"]?.try { |region| region.codepoints.map { |codepoint| (codepoint - 0x1f1a5).chr }.join("") }
-      flag = md["region"]?
+    instance_yaml = load_instance_yaml(body)
 
-      uri = URI.parse(md["uri"])
-      host = md["host"]
+    instance_storage = {} of String => ClearNetInstance | OnionInstance
 
-      case type = host.split(".")[-1]
-      when "onion"
-      when "i2p"
-      else
-        type = uri.scheme.not_nil!
-        client = HTTP::Client.new(uri)
-        client.connect_timeout = 5.seconds
-        client.read_timeout = 5.seconds
-        begin
-          stats = JSON.parse(client.get("/api/v1/stats").body)
-        rescue ex
-          stats = nil
-        end
-      end
-
-      monitor = monitors.try &.select { |monitor| monitor["name"].try &.as_s == host }[0]?
-      instances[host] = {flag: flag, region: region, stats: stats, type: type, uri: uri.to_s, monitor: monitor || instances[host]?.try &.[:monitor]?}
-    end
+    instance_yaml["instances"]["https"].as_a.each { |i| instance_storage[URI.parse(i["url"].to_s).host.not_nil!] = prepare_http_instance(i, instance_storage, monitors) }
+    instance_yaml["instances"]["onion"].as_a.each { |i| instance_storage[URI.parse(i["url"].to_s).host.not_nil!] = prepare_onion_instance(i, instance_storage) }
 
     INSTANCES.clear
-    INSTANCES.merge! instances
+    INSTANCES.merge! instance_storage
 
-    sleep 5.minutes
+    sleep CONFIG["minutes_between_refresh"].as_i.minutes
   end
 end
 
@@ -134,13 +151,13 @@ static_headers do |response, filepath, filestat|
 end
 
 SORT_PROCS = {
-  "health"   => ->(name : String, instance : Instance) { -(instance[:monitor]?.try &.["30dRatio"]["ratio"].as_s.to_f || 0.0) },
-  "location" => ->(name : String, instance : Instance) { instance[:region]? || "ZZ" },
-  "name"     => ->(name : String, instance : Instance) { name },
-  "signup"   => ->(name : String, instance : Instance) { instance[:stats]?.try &.["openRegistrations"]?.try { |bool| bool.as_bool ? 0 : 1 } || 2 },
-  "type"     => ->(name : String, instance : Instance) { instance[:type] },
-  "users"    => ->(name : String, instance : Instance) { -(instance[:stats]?.try &.["usage"]?.try &.["users"]["total"].as_i || 0) },
-  "version"  => ->(name : String, instance : Instance) { instance[:stats]?.try &.["software"]?.try &.["version"].as_s.try &.split("-", 2)[0].split(".").map { |a| -a.to_i } || [0, 0, 0] },
+  "health"   => ->(name : String, instance : ClearNetInstance | OnionInstance) { -(instance[:monitor]?.try &.["30dRatio"]["ratio"].as_s.to_f || 0.0) },
+  "location" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:region]? || "ZZ" },
+  "name"     => ->(name : String, instance : ClearNetInstance | OnionInstance) { name },
+  "signup"   => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:stats]?.try &.["openRegistrations"]?.try { |bool| bool.as_bool ? 0 : 1 } || 2 },
+  "type"     => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:type] },
+  "users"    => ->(name : String, instance : ClearNetInstance | OnionInstance) { -(instance[:stats]?.try &.["usage"]?.try &.["users"]["total"].as_i || 0) },
+  "version"  => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:stats]?.try &.["software"]?.try &.["version"].as_s.try &.split("-", 2)[0].split(".").map { |a| -a.to_i } || [0, 0, 0] },
 }
 
 def sort_instances(instances, sort_by)

src/instances/views/index.ecr

@@ -118,7 +118,7 @@
               <td><%= instance[:type] %></td>
               <td><%= instance[:stats]?.try &.["usage"]?.try &.["users"]["total"] || "-" %></td>
               <td><%= instance[:stats]?.try &.["openRegistrations"]?.try { |bool| bool.as_bool ? "✔" : "❌" } || "-" %></td>
-              <td><%= instance[:flag]? ? "#{instance[:flag]} #{instance[:region]}" : "-" %></td>
+              <td><%= instance[:region]? ? "#{instance[:flag]} #{instance[:region]}" : "-" %></td>
               <td><%= instance[:monitor]?.try &.["30dRatio"]["ratio"] || "-" %></td>
             </tr>
           <% end %>