diff --git a/Dockerfile b/Dockerfile index 2b3b239f..42044a12 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,14 +3,15 @@ WORKDIR /app COPY ./shard.yml ./shard.yml RUN shards install COPY ./src/ ./src/ -RUN crystal build ./src/instances.cr --release +RUN crystal build ./src/instances.cr -s -p -t FROM alpine:latest -RUN apk add --no-cache gc pcre libgcc +RUN apk add --no-cache gc pcre libgcc yaml WORKDIR /app RUN addgroup -g 1000 -S invidious && \ adduser -u 1000 -S invidious -G invidious COPY ./assets/ ./assets/ +COPY ./config.yml ./config.yml COPY --from=builder /app/instances . EXPOSE 3000 diff --git a/config.yml b/config.yml new file mode 100644 index 00000000..08f2a5e0 --- /dev/null +++ b/config.yml @@ -0,0 +1,9 @@ +# Uses the system's CURL binary to do so. +fetch_onion_instance_stats: true + +# TOR's Sock proxy address that CURL uses to connect to hidden services +tor_sock_proxy_address: "127.0.0.1" +tor_sock_proxy_port: 9050 + +# Minutes before refreshing the instance stats +minutes_between_refresh: 30 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 83e664a0..57fc7a1a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,13 @@ version: '3' services: + tor-socks-proxy: + container_name: tor-socks-proxy + image: peterdavehello/tor-socks-proxy:latest + ports: + - "127.0.0.1:8853:53/udp" + - "127.0.0.1:9150:9150/tcp" + restart: unless-stopped + instances: build: . restart: unless-stopped diff --git a/src/fetch.cr b/src/fetch.cr new file mode 100644 index 00000000..bc4eb3d0 --- /dev/null +++ b/src/fetch.cr @@ -0,0 +1,125 @@ +# Fetch a country's emoji flag from their country code (ISO 3166 alpha-2). +# +# A flag is made out of two regional indicator symbols. +# So in order to convert from an ISO 3166 alpha-2 code into unicode we'll have to +# add a specific offset to each character to make them into the required regional +# indicator symbols. This offset is exactly 0x1f1a5. +# +# Reference implementation https://schinckel.net/2015/10/29/unicode-flags-in-python/ +private def fetch_flag(country_code) + return country_code.codepoints.map { |codepoint| (codepoint + 0x1f1a5).chr }.join("") +end + +# Extracts the nested modified information containing source url and changes. +private def extract_modified_information(modified_hash) + if modified = modified_hash.as_h? + return Modified.new( + source: modified["source"].as_s, + changes: modified["changes"].as_s + ) + end +end + +# Extracts information common to all instance types. +private def extract_prerequisites(instance_data) + uri = URI.parse(instance_data["url"].to_s) + host = uri.host + + # Fetch country data + region = instance_data["country"].to_s + flag = fetch_flag(region) + + privacy_policy = instance_data["privacy_policy"].as_s? + owner = {name: instance_data["owner"].to_s.split("/")[-1].to_s, url: instance_data["owner"].as_s} + modified = extract_modified_information(instance_data["modified"]) + notes = instance_data["notes"].as_a? + + mirrors = [] of Mirrors + instance_data["mirrors"].as_a?.try &.each do |m| + mirrors << Mirrors.new( + url: m["url"].as_s, + region: m["country"].as_s, + flag: fetch_flag(m["country"].as_s) + ) + end + + return uri, host, region, flag, privacy_policy, owner, modified, notes, mirrors +end + +def prepare_http_instance(instance_data, instances_storage, monitors) + uri, host, region, flag, privacy_policy, owner, modified, notes, mirrors = extract_prerequisites(instance_data) + + # Fetch status information + if status = instance_data["status"].as_h? + status_url = status["url"].as_s + else + status_url = nil + end + + ddos_mitm_protection = instance_data["ddos_mitm_protection"].as_s? + + client = HTTP::Client.new(uri) + client.connect_timeout = 5.seconds + client.read_timeout = 5.seconds + + begin + stats = JSON.parse(client.get("/api/v1/stats").body) + rescue ex + stats = nil + end + + monitor = monitors.try &.select { |monitor| monitor["name"].try &.as_s == host }[0]? + return { + region: region, + flag: flag, + stats: stats, + type: "https", + uri: uri.to_s, + status_url: status_url, + privacy_policy: privacy_policy, + ddos_mitm_protection: ddos_mitm_protection, + owner: owner, + modified: modified, + mirrors: mirrors, + notes: notes, + monitor: monitor || instances_storage[host]?.try &.[:monitor]?, + } +end + +def prepare_onion_instance(instance_data, instances_storage) + uri, host, region, flag, privacy_policy, owner, modified, notes, mirrors = extract_prerequisites(instance_data) + + associated_clearnet_instance = instance_data["associated_clearnet_instance"].as_s? + + if CONFIG["fetch_onion_instance_stats"]? + begin + args = Process.parse_arguments("--socks5-hostname '#{CONFIG["tor_sock_proxy_address"]}:#{CONFIG["tor_sock_proxy_port"]}' 'http://#{uri.host}/api/v1/stats'") + response = nil + Process.run("curl", args: args) do |result| + data = result.output.read_line + response = JSON.parse(data) + end + + stats = response + rescue ex + stats = nil + end + else + stats = nil + end + + return { + region: region, + flag: flag, + stats: stats, + type: "onion", + uri: uri.to_s, + associated_clearnet_instance: associated_clearnet_instance, + privacy_policy: privacy_policy, + owner: owner, + modified: modified, + mirrors: mirrors, + notes: notes, + monitor: nil, + } +end diff --git a/src/helpers/helpers.cr b/src/helpers/helpers.cr new file mode 100644 index 00000000..7fcc83fa --- /dev/null +++ b/src/helpers/helpers.cr @@ -0,0 +1,9 @@ +require "yaml" + +def load_config + return YAML.parse(File.read("config.yml")) +end + +def load_instance_yaml(contents) + return YAML.parse(contents) +end diff --git a/src/instances.cr b/src/instances.cr index 71c45517..9459a0fd 100644 --- a/src/instances.cr +++ b/src/instances.cr @@ -18,15 +18,52 @@ require "http/client" require "kemal" require "uri" +require "./fetch.cr" +require "./helpers/*" + +CONFIG = load_config() + Kemal::CLI.new ARGV macro rendered(filename) render "src/instances/views/#{{{filename}}}.ecr" end -alias Instance = NamedTuple(flag: String?, region: String?, stats: JSON::Any?, type: String, uri: String, monitor: JSON::Any?) - -INSTANCES = {} of String => Instance +# Nested data within instances +alias Owner = NamedTuple(name: String, url: String) +alias Modified = NamedTuple(source: String, changes: String) +alias Mirrors = NamedTuple(url: String, region: String, flag: String) + +alias ClearNetInstance = NamedTuple( + flag: String, + region: String, + stats: JSON::Any?, + type: String, + uri: String, + status_url: String?, + privacy_policy: String?, + ddos_mitm_protection: String?, + owner: Owner, + modified: Modified?, + mirrors: Array(Mirrors)?, + notes: Array(YAML::Any)?, + monitor: JSON::Any?) + +alias OnionInstance = NamedTuple( + flag: String, + region: String, + stats: JSON::Any?, + type: String, + uri: String, + associated_clearnet_instance: String?, + privacy_policy: String?, + owner: Owner, + modified: Modified?, + mirrors: Array(Mirrors)?, + notes: Array(YAML::Any)?, + monitor: JSON::Any?) + +INSTANCES = {} of String => ClearNetInstance | OnionInstance spawn do loop do @@ -50,45 +87,25 @@ spawn do break end end + begin - body = HTTP::Client.get(URI.parse("https://raw.githubusercontent.com/iv-org/documentation/master/Invidious-Instances.md")).body + # Needs to be replaced once merged! + body = HTTP::Client.get(URI.parse("https://raw.githubusercontent.com/syeopite/documentation/alt-instance-list/instances.yaml")).body rescue ex body = "" end - instances = {} of String => Instance - - body = body.split("### Blocked:")[0] - body.scan(/\[(?[^ \]]+)\]\((?[^\)]+)\)( .(?[\x{1f100}-\x{1f1ff}]{2}))?/mx).each do |md| - region = md["region"]?.try { |region| region.codepoints.map { |codepoint| (codepoint - 0x1f1a5).chr }.join("") } - flag = md["region"]? + instance_yaml = load_instance_yaml(body) - uri = URI.parse(md["uri"]) - host = md["host"] + instance_storage = {} of String => ClearNetInstance | OnionInstance - case type = host.split(".")[-1] - when "onion" - when "i2p" - else - type = uri.scheme.not_nil! - client = HTTP::Client.new(uri) - client.connect_timeout = 5.seconds - client.read_timeout = 5.seconds - begin - stats = JSON.parse(client.get("/api/v1/stats").body) - rescue ex - stats = nil - end - end - - monitor = monitors.try &.select { |monitor| monitor["name"].try &.as_s == host }[0]? - instances[host] = {flag: flag, region: region, stats: stats, type: type, uri: uri.to_s, monitor: monitor || instances[host]?.try &.[:monitor]?} - end + instance_yaml["instances"]["https"].as_a.each { |i| instance_storage[URI.parse(i["url"].to_s).host.not_nil!] = prepare_http_instance(i, instance_storage, monitors) } + instance_yaml["instances"]["onion"].as_a.each { |i| instance_storage[URI.parse(i["url"].to_s).host.not_nil!] = prepare_onion_instance(i, instance_storage) } INSTANCES.clear - INSTANCES.merge! instances + INSTANCES.merge! instance_storage - sleep 5.minutes + sleep CONFIG["minutes_between_refresh"].as_i.minutes end end @@ -134,13 +151,13 @@ static_headers do |response, filepath, filestat| end SORT_PROCS = { - "health" => ->(name : String, instance : Instance) { -(instance[:monitor]?.try &.["30dRatio"]["ratio"].as_s.to_f || 0.0) }, - "location" => ->(name : String, instance : Instance) { instance[:region]? || "ZZ" }, - "name" => ->(name : String, instance : Instance) { name }, - "signup" => ->(name : String, instance : Instance) { instance[:stats]?.try &.["openRegistrations"]?.try { |bool| bool.as_bool ? 0 : 1 } || 2 }, - "type" => ->(name : String, instance : Instance) { instance[:type] }, - "users" => ->(name : String, instance : Instance) { -(instance[:stats]?.try &.["usage"]?.try &.["users"]["total"].as_i || 0) }, - "version" => ->(name : String, instance : Instance) { instance[:stats]?.try &.["software"]?.try &.["version"].as_s.try &.split("-", 2)[0].split(".").map { |a| -a.to_i } || [0, 0, 0] }, + "health" => ->(name : String, instance : ClearNetInstance | OnionInstance) { -(instance[:monitor]?.try &.["30dRatio"]["ratio"].as_s.to_f || 0.0) }, + "location" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:region]? || "ZZ" }, + "name" => ->(name : String, instance : ClearNetInstance | OnionInstance) { name }, + "signup" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:stats]?.try &.["openRegistrations"]?.try { |bool| bool.as_bool ? 0 : 1 } || 2 }, + "type" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:type] }, + "users" => ->(name : String, instance : ClearNetInstance | OnionInstance) { -(instance[:stats]?.try &.["usage"]?.try &.["users"]["total"].as_i || 0) }, + "version" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:stats]?.try &.["software"]?.try &.["version"].as_s.try &.split("-", 2)[0].split(".").map { |a| -a.to_i } || [0, 0, 0] }, } def sort_instances(instances, sort_by) diff --git a/src/instances/views/index.ecr b/src/instances/views/index.ecr index 2ba32075..1a025951 100644 --- a/src/instances/views/index.ecr +++ b/src/instances/views/index.ecr @@ -118,7 +118,7 @@ <%= instance[:type] %> <%= instance[:stats]?.try &.["usage"]?.try &.["users"]["total"] || "-" %> <%= instance[:stats]?.try &.["openRegistrations"]?.try { |bool| bool.as_bool ? "✔" : "❌" } || "-" %> - <%= instance[:flag]? ? "#{instance[:flag]} #{instance[:region]}" : "-" %> + <%= instance[:region]? ? "#{instance[:flag]} #{instance[:region]}" : "-" %> <%= instance[:monitor]?.try &.["30dRatio"]["ratio"] || "-" %> <% end %>