Migrated away from Excon and to async-http, fixes issues with ipv6 dns resolving but not reachable- and is the start towards more migration to async libs, websocket based server list updater is temporarily broken

Gemfile

@@ -1,7 +1,8 @@
 source "https://rubygems.org"
 
 gem "base64"
-gem "excon"
+gem "async-http"
+gem "async-websocket"
 gem "cyberarm_engine"
 gem "sdl2-bindings"
 gem "libui", platforms: [:windows]

Gemfile.lock

@@ -1,8 +1,35 @@
 GEM
   remote: https://rubygems.org/
   specs:
+    async (2.35.1)
+      console (~> 1.29)
+      fiber-annotation
+      io-event (~> 1.11)
+      metrics (~> 0.12)
+      traces (~> 0.18)
+    async-http (0.92.1)
+      async (>= 2.10.2)
+      async-pool (~> 0.11)
+      io-endpoint (~> 0.14)
+      io-stream (~> 0.6)
+      metrics (~> 0.12)
+      protocol-http (~> 0.49)
+      protocol-http1 (~> 0.30)
+      protocol-http2 (~> 0.22)
+      protocol-url (~> 0.2)
+      traces (~> 0.10)
+    async-pool (0.11.1)
+      async (>= 2.0)
+    async-websocket (0.30.0)
+      async-http (~> 0.76)
+      protocol-http (~> 0.34)
+      protocol-rack (~> 0.7)
+      protocol-websocket (~> 0.17)
     base64 (0.3.0)
-    concurrent-ruby (1.3.5)
+    console (1.34.2)
+      fiber-annotation
+      fiber-local (~> 1.1)
+      json
     cri (2.15.12)
     cyberarm_engine (0.24.5)
       gosu (~> 1.1)
@@ -14,17 +41,39 @@ GEM
     ffi (1.17.0)
     ffi-win32-extensions (1.1.0)
       ffi (>= 1.15.5, <= 1.17.0)
+    fiber-annotation (0.2.0)
+    fiber-local (1.1.0)
+      fiber-storage
+    fiber-storage (1.0.1)
     fiddle (1.1.8)
     gosu (1.4.6)
-    i18n (1.14.7)
+    io-endpoint (0.16.0)
-      concurrent-ruby (~> 1.0)
+    io-event (1.14.2)
+    io-stream (0.11.1)
     ircparser (1.0.0)
+    json (2.18.0)
     libui (0.2.0-x64-mingw-ucrt)
       fiddle
     logger (1.7.0)
+    metrics (0.15.0)
     mutex_m (0.3.0)
     ocran (1.3.17)
       fiddle (~> 1.0)
+    protocol-hpack (1.5.1)
+    protocol-http (0.57.0)
+    protocol-http1 (0.35.2)
+      protocol-http (~> 0.22)
+    protocol-http2 (0.23.0)
+      protocol-hpack (~> 1.4)
+      protocol-http (~> 0.47)
+    protocol-rack (0.20.0)
+      io-stream (>= 0.10)
+      protocol-http (~> 0.43)
+      rack (>= 1.0)
+    protocol-url (0.4.0)
+    protocol-websocket (0.20.2)
+      protocol-http (~> 0.2)
+    rack (3.2.4)
     rake (13.3.1)
     releasy (0.2.4)
       bundler (>= 1.2.1)
@@ -35,6 +84,7 @@ GEM
     rubyzip (3.2.2)
     sdl2-bindings (0.2.3)
       ffi (~> 1.15)
+    traces (0.18.2)
     websocket (1.2.11)
     websocket-client-simple (0.9.0)
       base64
@@ -51,12 +101,13 @@ PLATFORMS
   x64-mingw-ucrt
 
 DEPENDENCIES
+  async-http
+  async-websocket
   base64
   bundler (~> 2.4.3)
   cyberarm_engine
   digest-crc
   excon
-  i18n
   ircparser
   libui
   ocran

lib/api.rb

@@ -10,35 +10,37 @@ class W3DHub
 
     API_TIMEOUT = 30 # seconds
     USER_AGENT = "Cyberarm's Linux Friendly W3D Hub Launcher v#{W3DHub::VERSION}".freeze
-    DEFAULT_HEADERS = {
+    DEFAULT_HEADERS = [
-      "User-Agent": USER_AGENT,
+      ["user-agent", USER_AGENT],
-      "Accept": "application/json"
+      ["accept", "application/json"]
-    }.freeze
+    ].freeze
-    FORM_ENCODED_HEADERS = {
+    FORM_ENCODED_HEADERS = [
-      "User-Agent": USER_AGENT,
+      ["user-agent", USER_AGENT],
-      "Accept": "application/json",
+      ["accept", "application/json"],
-      "Content-Type": "application/x-www-form-urlencoded"
+      ["content-type", "application/x-www-form-urlencoded"]
-    }.freeze
+    ].freeze
 
     def self.on_thread(method, *args, &callback)
       BackgroundWorker.foreground_job(-> { Api.send(method, *args) }, callback)
     end
 
-    class DummyResponse
+    class Response
-      def initialize(error)
+      def initialize(error: nil, status: -1, body: "")
+        @status = status
+        @body = body
         @error = error
       end
 
       def success?
-        false
+        @status == 200
       end
 
       def status
-        -1
+        @status
       end
 
       def body
-        ""
+        @body
       end
 
       def error
@@ -48,103 +50,60 @@ class W3DHub
 
     #! === W3D Hub API === !#
     W3DHUB_API_ENDPOINT = "https://secure.w3dhub.com".freeze # "https://example.com" # "http://127.0.0.1:9292".freeze #
-    W3DHUB_API_CONNECTION = Excon.new(W3DHUB_API_ENDPOINT, persistent: true)
-
     ALT_W3DHUB_API_ENDPOINT = "https://w3dhub-api.w3d.cyberarm.dev".freeze # "https://secure.w3dhub.com".freeze # "https://example.com" # "http://127.0.0.1:9292".freeze #
-    ALT_W3DHUB_API_API_CONNECTION = Excon.new(ALT_W3DHUB_API_ENDPOINT, persistent: true)
 
-    def self.excon(method, url, headers = DEFAULT_HEADERS, body = nil, backend = :w3dhub)
+    def self.async_http(method, url, headers = DEFAULT_HEADERS, body = nil, backend = :w3dhub)
       case backend
       when :w3dhub
-        connection = W3DHUB_API_CONNECTION
         endpoint   = W3DHUB_API_ENDPOINT
       when :alt_w3dhub
-        connection = ALT_W3DHUB_API_API_CONNECTION
         endpoint   = ALT_W3DHUB_API_ENDPOINT
       when :gsh
-        connection = GSH_CONNECTION
         endpoint   = SERVER_LIST_ENDPOINT
       end
 
-      logger.debug(LOG_TAG) { "Fetching #{method.to_s.upcase} \"#{endpoint}#{url}\"..." }
+      url = "#{endpoint}#{url}" unless url.start_with?("http")
+
+      logger.debug(LOG_TAG) { "Fetching #{method.to_s.upcase} \"#{url}\"..." }
 
       # Inject Authorization header if account data is populated
       if Store.account
         logger.debug(LOG_TAG) { "  Injecting Authorization header..." }
         headers = headers.dup
-        headers["Authorization"] = "Bearer #{Store.account.access_token}"
+        headers << ["authorization", "Bearer #{Store.account.access_token}"]
       end
 
+      Sync do
         begin
-        connection.send(
+         response = Async::HTTP::Internet.send(method, url, headers, body)
-          method,
+
-          path: url.sub(endpoint, ""),
+          Response.new(status: response.status, body: response.read)
-          headers: headers,
+        rescue Async::TimeoutError => e
-          body: body,
-          nonblock: true,
-          tcp_nodelay: true,
-          write_timeout: API_TIMEOUT,
-          read_timeout: API_TIMEOUT,
-          connect_timeout: API_TIMEOUT,
-          idempotent: true,
-          retry_limit: 3,
-          retry_interval: 1,
-          retry_errors: [Excon::Error::Socket, Excon::Error::HTTPStatus] # Don't retry on timeout
-        )
-      rescue Excon::Error::Timeout => e
           logger.error(LOG_TAG) { "Connection to \"#{url}\" timed out after: #{API_TIMEOUT} seconds" }
 
-        DummyResponse.new(e)
+          Response.new(error: e)
-      rescue Excon::Error => e
+        rescue StandardError => e
           logger.error(LOG_TAG) { "Connection to \"#{url}\" errored:" }
           logger.error(LOG_TAG) { e }
 
-        DummyResponse.new(e)
+          Response.new(error: e)
+        ensure
+          response&.close
+        end
       end
     end
 
     def self.post(url, headers = DEFAULT_HEADERS, body = nil, backend = :w3dhub)
-      excon(:post, url, headers, body, backend)
+      async_http(:post, url, headers, body, backend)
     end
 
     def self.get(url, headers = DEFAULT_HEADERS, body = nil, backend = :w3dhub)
-      excon(:get, url, headers, body, backend)
+      async_http(:get, url, headers, body, backend)
     end
 
     # Api.get but handles any URL instead of known hosts
     def self.fetch(url, headers = DEFAULT_HEADERS, body = nil, backend = nil)
-      uri = URI(url)
+      async_http(:get, url, headers, body, backend)
-
-      # Use Api.get for `W3DHUB_API_ENDPOINT` URL's to exploit keep alive and connection reuse (faster responses)
-      return excon(:get, url, headers, body, backend) if "#{uri.scheme}://#{uri.host}" == W3DHUB_API_ENDPOINT
-
-      logger.debug(LOG_TAG) { "Fetching GET \"#{url}\"..." }
-
-      begin
-        Excon.get(
-          url,
-          headers: headers,
-          body: body,
-          nonblock: true,
-          tcp_nodelay: true,
-          write_timeout: API_TIMEOUT,
-          read_timeout: API_TIMEOUT,
-          connect_timeout: API_TIMEOUT,
-          idempotent: true,
-          retry_limit: 3,
-          retry_interval: 1,
-          retry_errors: [Excon::Error::Socket, Excon::Error::HTTPStatus] # Don't retry on timeout
-        )
-      rescue Excon::Error::Timeout => e
-        logger.error(LOG_TAG) { "Connection to \"#{url}\" timed out after: #{API_TIMEOUT} seconds" }
-
-        DummyResponse.new(e)
-      rescue Excon::Error => e
-        logger.error(LOG_TAG) { "Connection to \"#{url}\" errored:" }
-        logger.error(LOG_TAG) { e }
-
-        DummyResponse.new(e)
-      end
     end
 
     # Method: POST
@@ -163,7 +122,7 @@ class W3DHub
     # On a failed login the service responds with:
     # {"error":"login-failed"}
     def self.refresh_user_login(refresh_token, backend = :w3dhub)
-      body = "data=#{JSON.dump({refreshToken: refresh_token})}"
+      body = URI.encode_www_form("data": JSON.dump({refreshToken: refresh_token}))
       response = post("/apis/launcher/1/user-login", FORM_ENCODED_HEADERS, body, backend)
 
       if response.status == 200
@@ -183,7 +142,7 @@ class W3DHub
 
     # See #user_refresh_token
     def self.user_login(username, password, backend = :w3dhub)
-      body = "data=#{JSON.dump({username: username, password: password})}"
+      body = URI.encode_www_form("data": JSON.dump({username: username, password: password}))
       response = post("/apis/launcher/1/user-login", FORM_ENCODED_HEADERS, body, backend)
 
       if response.status == 200
@@ -205,7 +164,7 @@ class W3DHub
     #
     # Response: avatar-uri (Image download uri), id, username
     def self.user_details(id, backend = :w3dhub)
-      body = "data=#{JSON.dump({ id: id })}"
+      body = URI.encode_www_form("data": JSON.dump({ id: id }))
       user_details = post("/apis/w3dhub/1/get-user-details", FORM_ENCODED_HEADERS, body, backend)
 
       if user_details.status == 200
@@ -322,7 +281,7 @@ class W3DHub
     # Client requests news for a specific application/game e.g.: data={"category":"ia"} ("launcher-home" retrieves the weekly hub updates)
     # Response is a JSON hash with a "highlighted" and "news" keys; the "news" one seems to be the desired one
     def self.news(category, backend = :w3dhub)
-      body = "data=#{JSON.dump({category: category})}"
+      body = URI.encode_www_form("data": JSON.dump({category: category}))
       response = post("/apis/w3dhub/1/get-news", FORM_ENCODED_HEADERS, body, backend)
 
       if response.status == 200
@@ -383,7 +342,6 @@ class W3DHub
     # SERVER_LIST_ENDPOINT = "https://gsh.w3dhub.com".freeze
     SERVER_LIST_ENDPOINT = "https://gsh.w3d.cyberarm.dev".freeze
     # SERVER_LIST_ENDPOINT = "http://127.0.0.1:9292".freeze
-    GSH_CONNECTION = Excon.new(SERVER_LIST_ENDPOINT, persistent: true)
 
     # Method: GET
     # FORMAT: JSON

lib/api/server_list_updater.rb

@@ -23,12 +23,15 @@ class W3DHub
       end
 
       def run
+        return
+
         Thread.new do
+          Sync do |task|
             begin
-            connect
+              async_connect(task)
 
               while W3DHub::BackgroundWorker.alive?
-              connect if @auto_reconnect
+                async_connect(task) if @auto_reconnect
                 sleep 1
               end
             rescue => e
@@ -39,11 +42,45 @@ class W3DHub
               retry
             end
           end
+        end
 
         logger.debug(LOG_TAG) { "Cleaning up..." }
         @@instance = nil
       end
 
+      def async_connect(task)
+        @auto_reconnect = false
+
+        logger.debug(LOG_TAG) { "Requesting connection token..." }
+        response = Api.post("/listings/push/v2/negotiate?negotiateVersion=1", Api::DEFAULT_HEADERS, "", :gsh)
+
+        if response.status != 200
+          @auto_reconnect = true
+          return
+        end
+
+        data = JSON.parse(response.body, symbolize_names: true)
+
+        @invocation_id = 0 if @invocation_id > 9095
+        id = data[:connectionToken]
+        endpoint = "#{Api::SERVER_LIST_ENDPOINT}/listings/push/v2?id=#{id}"
+
+        logger.debug(LOG_TAG) { "Connecting to websocket..." }
+
+        Async::WebSocket::Client.connect(Async::HTTP::Endpoint.parse(endpoint)) do |connection|
+          logger.debug(LOG_TAG) { "Requesting json protocol, v1..." }
+          async_websocket_send(connection, { protocol: "json", version: 1 }.to_json)
+        end
+      end
+
+      def async_websocket_send(connection, payload)
+        connection.write("#{payload}\x1e")
+        connection.flush
+      end
+
+      def async_websocket_read(connection, payload)
+      end
+
       def connect
         @auto_reconnect = false
 

lib/cache.rb

@@ -50,54 +50,16 @@ class W3DHub
     end
 
     # Download a W3D Hub package
-    # TODO: More work needed to make this work reliably
+    def self.async_fetch_package(package, block)
-    def self._async_fetch_package(package, block)
-      path = package_path(package.category, package.subcategory, package.name, package.version)
-      headers = Api::FORM_ENCODED_HEADERS
-      start_from_bytes = package.custom_partially_valid_at_bytes
-
-      logger.info(LOG_TAG) { "    Start from bytes: #{start_from_bytes} of #{package.size}" }
-
-      create_directories(path)
-
-      file = File.open(path, start_from_bytes.positive? ? "r+b" : "wb")
-
-      if start_from_bytes.positive?
-        headers = Api::FORM_ENCODED_HEADERS + [["Range", "bytes=#{start_from_bytes}-"]]
-        file.pos = start_from_bytes
-      end
-
-      body = "data=#{JSON.dump({ category: package.category, subcategory: package.subcategory, name: package.name, version: package.version })}"
-
-      response = Api.post("/apis/launcher/1/get-package", headers, body)
-
-      total_bytes = package.size
-      remaining_bytes = total_bytes - start_from_bytes
-
-      response.each do |chunk|
-        file.write(chunk)
-
-        remaining_bytes -= chunk.size
-
-        block.call(chunk, remaining_bytes, total_bytes)
-      end
-
-      response.status == 200
-    ensure
-      file&.close
-    end
-
-    # Download a W3D Hub package
-    def self.fetch_package(package, block)
       endpoint_download_url = package.download_url || "#{Api::W3DHUB_API_ENDPOINT}/apis/launcher/1/get-package"
       if package.download_url
         uri_path = package.download_url.split("/").last
         endpoint_download_url = package.download_url.sub(uri_path, URI.encode_uri_component(uri_path))
       end
       path = package_path(package.category, package.subcategory, package.name, package.version)
-      headers = { "Content-Type": "application/x-www-form-urlencoded", "User-Agent": Api::USER_AGENT }
+      headers = [["content-type", "application/x-www-form-urlencoded"], ["user-agent", Api::USER_AGENT]]
-      headers["Authorization"] = "Bearer #{Store.account.access_token}" if Store.account && !package.download_url
+      headers << ["authorization", "Bearer #{Store.account.access_token}"] if Store.account && !package.download_url
-      body = "data=#{JSON.dump({ category: package.category, subcategory: package.subcategory, name: package.name, version: package.version })}"
+      body = URI.encode_www_form("data": JSON.dump({ category: package.category, subcategory: package.subcategory, name: package.name, version: package.version }))
       start_from_bytes = package.custom_partially_valid_at_bytes
 
       logger.info(LOG_TAG) { "    Start from bytes: #{start_from_bytes} of #{package.size}" }
@@ -107,54 +69,63 @@ class W3DHub
       file = File.open(path, start_from_bytes.positive? ? "r+b" : "wb")
 
       if start_from_bytes.positive?
-        headers["Range"] = "bytes=#{start_from_bytes}-"
+        headers << ["range", "bytes=#{start_from_bytes}-"]
         file.pos = start_from_bytes
       end
 
-      streamer = lambda do |chunk, remaining_bytes, total_bytes|
+      result = false
+      Sync do
+        response = nil
+
+        Async::HTTP::Internet.send(package.download_url ? :get : :post, endpoint_download_url, headers, body) do |r|
+          response = r
+          if r.success?
+            total_bytes = package.size
+
+            r.each do |chunk|
               file.write(chunk)
 
-        block.call(chunk, remaining_bytes, total_bytes)
+              block.call(chunk, total_bytes - file.pos, total_bytes)
             end
 
-      # Create a new connection due to some weirdness somewhere in Excon
+            result = true
-      response = Excon.send(
+          end
-        package.download_url ? :get : :post,
+        end
-        endpoint_download_url,
-        tcp_nodelay: true,
-        headers: headers,
-        body: package.download_url ? "" : body,
-        chunk_size: 50_000,
-        response_block: streamer,
-        middlewares: Excon.defaults[:middlewares] + [Excon::Middleware::RedirectFollower]
-      )
 
         if response.status == 200 || response.status == 206
-        return true
+          result = true
         else
           logger.debug(LOG_TAG) { "    Failed to retrieve package: (#{package.category}:#{package.subcategory}:#{package.name}:#{package.version})" }
           logger.debug(LOG_TAG) { "      Download URL: #{endpoint_download_url}, response: #{response&.status || -1}" }
 
-        return false
+          result = false
         end
-    rescue Excon::Error::Timeout => e
+      rescue Async::Timeout => e
         logger.error(LOG_TAG) { "    Connection to \"#{endpoint_download_url}\" timed out after: #{W3DHub::Api::API_TIMEOUT} seconds" }
         logger.error(LOG_TAG) { e }
         logger.debug(LOG_TAG) { "    Failed to retrieve package: (#{package.category}:#{package.subcategory}:#{package.name}:#{package.version})" }
         logger.debug(LOG_TAG) { "      Download URL: #{endpoint_download_url}, response: #{response&.status || -1}" }
 
-      return false
+        result = false
-    rescue Excon::Error => e
+      rescue StandardError => e
         logger.error(LOG_TAG) { "    Connection to \"#{endpoint_download_url}\" errored:" }
         logger.error(LOG_TAG) { e }
         logger.debug(LOG_TAG) { "    Failed to retrieve package: (#{package.category}:#{package.subcategory}:#{package.name}:#{package.version})" }
         logger.debug(LOG_TAG) { "      Download URL: #{endpoint_download_url}, response: #{response&.status || -1}" }
 
-      return false
+        result = false
+      end
+
+      result
     ensure
       file&.close
     end
 
+    # Download a W3D Hub package
+    def self.fetch_package(package, block)
+      async_fetch_package(package, block)
+    end
+
     def self.acquire_net_lock(key)
       Store["net_locks"] ||= {}
 

w3d_hub_linux_launcher.rb

@@ -14,7 +14,10 @@ require "logger"
 require "time"
 require "base64"
 require "zip"
-require "excon"
+require "async"
+require "async/http/endpoint"
+require "async/websocket/client"
+require "async/http/internet/instance"
 
 class W3DHub
   W3DHUB_DEBUG = ARGV.join.include?("--debug")
@@ -32,7 +35,7 @@ class W3DHub
   FileUtils.mkdir_p(CACHE_PATH) unless Dir.exist?(CACHE_PATH)
   FileUtils.mkdir_p(LOGS_PATH) unless Dir.exist?(LOGS_PATH)
 
-  LOGGER = Logger.new("#{LOGS_PATH}/w3d_hub_linux_launcher.log", "daily")
+  LOGGER = W3DHUB_DEBUG ? Logger.new(STDOUT) : Logger.new("#{LOGS_PATH}/w3d_hub_linux_launcher.log", "daily")
   LOGGER.level = Logger::Severity::DEBUG # W3DHUB_DEBUG ? Logger::Severity::DEBUG : Logger::Severity::WARN
 
   LOG_TAG = "W3DHubLinuxLauncher"