From 840bc849d342782f6d403560d906ab2c775c08a6 Mon Sep 17 00:00:00 2001 From: Cyberarm Date: Wed, 7 Jan 2026 14:18:48 -0600 Subject: [PATCH] Migrated away from Excon and to async-http, fixes issues with ipv6 dns resolving but not reachable- and is the start towards more migration to async libs, websocket based server list updater is temporarily broken --- Gemfile | 3 +- Gemfile.lock | 59 +++++++++++++-- lib/api.rb | 128 +++++++++++---------------------- lib/api/server_list_updater.rb | 59 ++++++++++++--- lib/cache.rb | 121 ++++++++++++------------------- w3d_hub_linux_launcher.rb | 7 +- 6 files changed, 199 insertions(+), 178 deletions(-) diff --git a/Gemfile b/Gemfile index f2dd7f1..da5157d 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,8 @@ source "https://rubygems.org" gem "base64" -gem "excon" +gem "async-http" +gem "async-websocket" gem "cyberarm_engine" gem "sdl2-bindings" gem "libui", platforms: [:windows] diff --git a/Gemfile.lock b/Gemfile.lock index c29b13a..ed99aa1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,8 +1,35 @@ GEM remote: https://rubygems.org/ specs: + async (2.35.1) + console (~> 1.29) + fiber-annotation + io-event (~> 1.11) + metrics (~> 0.12) + traces (~> 0.18) + async-http (0.92.1) + async (>= 2.10.2) + async-pool (~> 0.11) + io-endpoint (~> 0.14) + io-stream (~> 0.6) + metrics (~> 0.12) + protocol-http (~> 0.49) + protocol-http1 (~> 0.30) + protocol-http2 (~> 0.22) + protocol-url (~> 0.2) + traces (~> 0.10) + async-pool (0.11.1) + async (>= 2.0) + async-websocket (0.30.0) + async-http (~> 0.76) + protocol-http (~> 0.34) + protocol-rack (~> 0.7) + protocol-websocket (~> 0.17) base64 (0.3.0) - concurrent-ruby (1.3.5) + console (1.34.2) + fiber-annotation + fiber-local (~> 1.1) + json cri (2.15.12) cyberarm_engine (0.24.5) gosu (~> 1.1) @@ -14,17 +41,39 @@ GEM ffi (1.17.0) ffi-win32-extensions (1.1.0) ffi (>= 1.15.5, <= 1.17.0) + fiber-annotation (0.2.0) + fiber-local (1.1.0) + fiber-storage + fiber-storage (1.0.1) fiddle (1.1.8) gosu (1.4.6) - i18n (1.14.7) - concurrent-ruby (~> 1.0) + io-endpoint (0.16.0) + io-event (1.14.2) + io-stream (0.11.1) ircparser (1.0.0) + json (2.18.0) libui (0.2.0-x64-mingw-ucrt) fiddle logger (1.7.0) + metrics (0.15.0) mutex_m (0.3.0) ocran (1.3.17) fiddle (~> 1.0) + protocol-hpack (1.5.1) + protocol-http (0.57.0) + protocol-http1 (0.35.2) + protocol-http (~> 0.22) + protocol-http2 (0.23.0) + protocol-hpack (~> 1.4) + protocol-http (~> 0.47) + protocol-rack (0.20.0) + io-stream (>= 0.10) + protocol-http (~> 0.43) + rack (>= 1.0) + protocol-url (0.4.0) + protocol-websocket (0.20.2) + protocol-http (~> 0.2) + rack (3.2.4) rake (13.3.1) releasy (0.2.4) bundler (>= 1.2.1) @@ -35,6 +84,7 @@ GEM rubyzip (3.2.2) sdl2-bindings (0.2.3) ffi (~> 1.15) + traces (0.18.2) websocket (1.2.11) websocket-client-simple (0.9.0) base64 @@ -51,12 +101,13 @@ PLATFORMS x64-mingw-ucrt DEPENDENCIES + async-http + async-websocket base64 bundler (~> 2.4.3) cyberarm_engine digest-crc excon - i18n ircparser libui ocran diff --git a/lib/api.rb b/lib/api.rb index 77f390d..dd11e3c 100644 --- a/lib/api.rb +++ b/lib/api.rb @@ -10,35 +10,37 @@ class W3DHub API_TIMEOUT = 30 # seconds USER_AGENT = "Cyberarm's Linux Friendly W3D Hub Launcher v#{W3DHub::VERSION}".freeze - DEFAULT_HEADERS = { - "User-Agent": USER_AGENT, - "Accept": "application/json" - }.freeze - FORM_ENCODED_HEADERS = { - "User-Agent": USER_AGENT, - "Accept": "application/json", - "Content-Type": "application/x-www-form-urlencoded" - }.freeze + DEFAULT_HEADERS = [ + ["user-agent", USER_AGENT], + ["accept", "application/json"] + ].freeze + FORM_ENCODED_HEADERS = [ + ["user-agent", USER_AGENT], + ["accept", "application/json"], + ["content-type", "application/x-www-form-urlencoded"] + ].freeze def self.on_thread(method, *args, &callback) BackgroundWorker.foreground_job(-> { Api.send(method, *args) }, callback) end - class DummyResponse - def initialize(error) + class Response + def initialize(error: nil, status: -1, body: "") + @status = status + @body = body @error = error end def success? - false + @status == 200 end def status - -1 + @status end def body - "" + @body end def error @@ -48,103 +50,60 @@ class W3DHub #! === W3D Hub API === !# W3DHUB_API_ENDPOINT = "https://secure.w3dhub.com".freeze # "https://example.com" # "http://127.0.0.1:9292".freeze # - W3DHUB_API_CONNECTION = Excon.new(W3DHUB_API_ENDPOINT, persistent: true) - ALT_W3DHUB_API_ENDPOINT = "https://w3dhub-api.w3d.cyberarm.dev".freeze # "https://secure.w3dhub.com".freeze # "https://example.com" # "http://127.0.0.1:9292".freeze # - ALT_W3DHUB_API_API_CONNECTION = Excon.new(ALT_W3DHUB_API_ENDPOINT, persistent: true) - def self.excon(method, url, headers = DEFAULT_HEADERS, body = nil, backend = :w3dhub) + def self.async_http(method, url, headers = DEFAULT_HEADERS, body = nil, backend = :w3dhub) case backend when :w3dhub - connection = W3DHUB_API_CONNECTION endpoint = W3DHUB_API_ENDPOINT when :alt_w3dhub - connection = ALT_W3DHUB_API_API_CONNECTION endpoint = ALT_W3DHUB_API_ENDPOINT when :gsh - connection = GSH_CONNECTION endpoint = SERVER_LIST_ENDPOINT end - logger.debug(LOG_TAG) { "Fetching #{method.to_s.upcase} \"#{endpoint}#{url}\"..." } + url = "#{endpoint}#{url}" unless url.start_with?("http") + + logger.debug(LOG_TAG) { "Fetching #{method.to_s.upcase} \"#{url}\"..." } # Inject Authorization header if account data is populated if Store.account logger.debug(LOG_TAG) { " Injecting Authorization header..." } headers = headers.dup - headers["Authorization"] = "Bearer #{Store.account.access_token}" + headers << ["authorization", "Bearer #{Store.account.access_token}"] end - begin - connection.send( - method, - path: url.sub(endpoint, ""), - headers: headers, - body: body, - nonblock: true, - tcp_nodelay: true, - write_timeout: API_TIMEOUT, - read_timeout: API_TIMEOUT, - connect_timeout: API_TIMEOUT, - idempotent: true, - retry_limit: 3, - retry_interval: 1, - retry_errors: [Excon::Error::Socket, Excon::Error::HTTPStatus] # Don't retry on timeout - ) - rescue Excon::Error::Timeout => e - logger.error(LOG_TAG) { "Connection to \"#{url}\" timed out after: #{API_TIMEOUT} seconds" } + Sync do + begin + response = Async::HTTP::Internet.send(method, url, headers, body) - DummyResponse.new(e) - rescue Excon::Error => e - logger.error(LOG_TAG) { "Connection to \"#{url}\" errored:" } - logger.error(LOG_TAG) { e } + Response.new(status: response.status, body: response.read) + rescue Async::TimeoutError => e + logger.error(LOG_TAG) { "Connection to \"#{url}\" timed out after: #{API_TIMEOUT} seconds" } - DummyResponse.new(e) + Response.new(error: e) + rescue StandardError => e + logger.error(LOG_TAG) { "Connection to \"#{url}\" errored:" } + logger.error(LOG_TAG) { e } + + Response.new(error: e) + ensure + response&.close + end end end def self.post(url, headers = DEFAULT_HEADERS, body = nil, backend = :w3dhub) - excon(:post, url, headers, body, backend) + async_http(:post, url, headers, body, backend) end def self.get(url, headers = DEFAULT_HEADERS, body = nil, backend = :w3dhub) - excon(:get, url, headers, body, backend) + async_http(:get, url, headers, body, backend) end # Api.get but handles any URL instead of known hosts def self.fetch(url, headers = DEFAULT_HEADERS, body = nil, backend = nil) - uri = URI(url) - - # Use Api.get for `W3DHUB_API_ENDPOINT` URL's to exploit keep alive and connection reuse (faster responses) - return excon(:get, url, headers, body, backend) if "#{uri.scheme}://#{uri.host}" == W3DHUB_API_ENDPOINT - - logger.debug(LOG_TAG) { "Fetching GET \"#{url}\"..." } - - begin - Excon.get( - url, - headers: headers, - body: body, - nonblock: true, - tcp_nodelay: true, - write_timeout: API_TIMEOUT, - read_timeout: API_TIMEOUT, - connect_timeout: API_TIMEOUT, - idempotent: true, - retry_limit: 3, - retry_interval: 1, - retry_errors: [Excon::Error::Socket, Excon::Error::HTTPStatus] # Don't retry on timeout - ) - rescue Excon::Error::Timeout => e - logger.error(LOG_TAG) { "Connection to \"#{url}\" timed out after: #{API_TIMEOUT} seconds" } - - DummyResponse.new(e) - rescue Excon::Error => e - logger.error(LOG_TAG) { "Connection to \"#{url}\" errored:" } - logger.error(LOG_TAG) { e } - - DummyResponse.new(e) - end + async_http(:get, url, headers, body, backend) end # Method: POST @@ -163,7 +122,7 @@ class W3DHub # On a failed login the service responds with: # {"error":"login-failed"} def self.refresh_user_login(refresh_token, backend = :w3dhub) - body = "data=#{JSON.dump({refreshToken: refresh_token})}" + body = URI.encode_www_form("data": JSON.dump({refreshToken: refresh_token})) response = post("/apis/launcher/1/user-login", FORM_ENCODED_HEADERS, body, backend) if response.status == 200 @@ -183,7 +142,7 @@ class W3DHub # See #user_refresh_token def self.user_login(username, password, backend = :w3dhub) - body = "data=#{JSON.dump({username: username, password: password})}" + body = URI.encode_www_form("data": JSON.dump({username: username, password: password})) response = post("/apis/launcher/1/user-login", FORM_ENCODED_HEADERS, body, backend) if response.status == 200 @@ -205,7 +164,7 @@ class W3DHub # # Response: avatar-uri (Image download uri), id, username def self.user_details(id, backend = :w3dhub) - body = "data=#{JSON.dump({ id: id })}" + body = URI.encode_www_form("data": JSON.dump({ id: id })) user_details = post("/apis/w3dhub/1/get-user-details", FORM_ENCODED_HEADERS, body, backend) if user_details.status == 200 @@ -322,7 +281,7 @@ class W3DHub # Client requests news for a specific application/game e.g.: data={"category":"ia"} ("launcher-home" retrieves the weekly hub updates) # Response is a JSON hash with a "highlighted" and "news" keys; the "news" one seems to be the desired one def self.news(category, backend = :w3dhub) - body = "data=#{JSON.dump({category: category})}" + body = URI.encode_www_form("data": JSON.dump({category: category})) response = post("/apis/w3dhub/1/get-news", FORM_ENCODED_HEADERS, body, backend) if response.status == 200 @@ -383,7 +342,6 @@ class W3DHub # SERVER_LIST_ENDPOINT = "https://gsh.w3dhub.com".freeze SERVER_LIST_ENDPOINT = "https://gsh.w3d.cyberarm.dev".freeze # SERVER_LIST_ENDPOINT = "http://127.0.0.1:9292".freeze - GSH_CONNECTION = Excon.new(SERVER_LIST_ENDPOINT, persistent: true) # Method: GET # FORMAT: JSON diff --git a/lib/api/server_list_updater.rb b/lib/api/server_list_updater.rb index fb2709f..9e98208 100644 --- a/lib/api/server_list_updater.rb +++ b/lib/api/server_list_updater.rb @@ -23,20 +23,24 @@ class W3DHub end def run + return + Thread.new do - begin - connect + Sync do |task| + begin + async_connect(task) - while W3DHub::BackgroundWorker.alive? - connect if @auto_reconnect - sleep 1 + while W3DHub::BackgroundWorker.alive? + async_connect(task) if @auto_reconnect + sleep 1 + end + rescue => e + puts e + puts e.backtrace + + sleep 30 + retry end - rescue => e - puts e - puts e.backtrace - - sleep 30 - retry end end @@ -44,6 +48,39 @@ class W3DHub @@instance = nil end + def async_connect(task) + @auto_reconnect = false + + logger.debug(LOG_TAG) { "Requesting connection token..." } + response = Api.post("/listings/push/v2/negotiate?negotiateVersion=1", Api::DEFAULT_HEADERS, "", :gsh) + + if response.status != 200 + @auto_reconnect = true + return + end + + data = JSON.parse(response.body, symbolize_names: true) + + @invocation_id = 0 if @invocation_id > 9095 + id = data[:connectionToken] + endpoint = "#{Api::SERVER_LIST_ENDPOINT}/listings/push/v2?id=#{id}" + + logger.debug(LOG_TAG) { "Connecting to websocket..." } + + Async::WebSocket::Client.connect(Async::HTTP::Endpoint.parse(endpoint)) do |connection| + logger.debug(LOG_TAG) { "Requesting json protocol, v1..." } + async_websocket_send(connection, { protocol: "json", version: 1 }.to_json) + end + end + + def async_websocket_send(connection, payload) + connection.write("#{payload}\x1e") + connection.flush + end + + def async_websocket_read(connection, payload) + end + def connect @auto_reconnect = false diff --git a/lib/cache.rb b/lib/cache.rb index d331899..003d822 100644 --- a/lib/cache.rb +++ b/lib/cache.rb @@ -50,54 +50,16 @@ class W3DHub end # Download a W3D Hub package - # TODO: More work needed to make this work reliably - def self._async_fetch_package(package, block) - path = package_path(package.category, package.subcategory, package.name, package.version) - headers = Api::FORM_ENCODED_HEADERS - start_from_bytes = package.custom_partially_valid_at_bytes - - logger.info(LOG_TAG) { " Start from bytes: #{start_from_bytes} of #{package.size}" } - - create_directories(path) - - file = File.open(path, start_from_bytes.positive? ? "r+b" : "wb") - - if start_from_bytes.positive? - headers = Api::FORM_ENCODED_HEADERS + [["Range", "bytes=#{start_from_bytes}-"]] - file.pos = start_from_bytes - end - - body = "data=#{JSON.dump({ category: package.category, subcategory: package.subcategory, name: package.name, version: package.version })}" - - response = Api.post("/apis/launcher/1/get-package", headers, body) - - total_bytes = package.size - remaining_bytes = total_bytes - start_from_bytes - - response.each do |chunk| - file.write(chunk) - - remaining_bytes -= chunk.size - - block.call(chunk, remaining_bytes, total_bytes) - end - - response.status == 200 - ensure - file&.close - end - - # Download a W3D Hub package - def self.fetch_package(package, block) + def self.async_fetch_package(package, block) endpoint_download_url = package.download_url || "#{Api::W3DHUB_API_ENDPOINT}/apis/launcher/1/get-package" if package.download_url uri_path = package.download_url.split("/").last endpoint_download_url = package.download_url.sub(uri_path, URI.encode_uri_component(uri_path)) end path = package_path(package.category, package.subcategory, package.name, package.version) - headers = { "Content-Type": "application/x-www-form-urlencoded", "User-Agent": Api::USER_AGENT } - headers["Authorization"] = "Bearer #{Store.account.access_token}" if Store.account && !package.download_url - body = "data=#{JSON.dump({ category: package.category, subcategory: package.subcategory, name: package.name, version: package.version })}" + headers = [["content-type", "application/x-www-form-urlencoded"], ["user-agent", Api::USER_AGENT]] + headers << ["authorization", "Bearer #{Store.account.access_token}"] if Store.account && !package.download_url + body = URI.encode_www_form("data": JSON.dump({ category: package.category, subcategory: package.subcategory, name: package.name, version: package.version })) start_from_bytes = package.custom_partially_valid_at_bytes logger.info(LOG_TAG) { " Start from bytes: #{start_from_bytes} of #{package.size}" } @@ -107,54 +69,63 @@ class W3DHub file = File.open(path, start_from_bytes.positive? ? "r+b" : "wb") if start_from_bytes.positive? - headers["Range"] = "bytes=#{start_from_bytes}-" + headers << ["range", "bytes=#{start_from_bytes}-"] file.pos = start_from_bytes end - streamer = lambda do |chunk, remaining_bytes, total_bytes| - file.write(chunk) + result = false + Sync do + response = nil - block.call(chunk, remaining_bytes, total_bytes) - end + Async::HTTP::Internet.send(package.download_url ? :get : :post, endpoint_download_url, headers, body) do |r| + response = r + if r.success? + total_bytes = package.size - # Create a new connection due to some weirdness somewhere in Excon - response = Excon.send( - package.download_url ? :get : :post, - endpoint_download_url, - tcp_nodelay: true, - headers: headers, - body: package.download_url ? "" : body, - chunk_size: 50_000, - response_block: streamer, - middlewares: Excon.defaults[:middlewares] + [Excon::Middleware::RedirectFollower] - ) + r.each do |chunk| + file.write(chunk) - if response.status == 200 || response.status == 206 - return true - else + block.call(chunk, total_bytes - file.pos, total_bytes) + end + + result = true + end + end + + if response.status == 200 || response.status == 206 + result = true + else + logger.debug(LOG_TAG) { " Failed to retrieve package: (#{package.category}:#{package.subcategory}:#{package.name}:#{package.version})" } + logger.debug(LOG_TAG) { " Download URL: #{endpoint_download_url}, response: #{response&.status || -1}" } + + result = false + end + rescue Async::Timeout => e + logger.error(LOG_TAG) { " Connection to \"#{endpoint_download_url}\" timed out after: #{W3DHub::Api::API_TIMEOUT} seconds" } + logger.error(LOG_TAG) { e } logger.debug(LOG_TAG) { " Failed to retrieve package: (#{package.category}:#{package.subcategory}:#{package.name}:#{package.version})" } logger.debug(LOG_TAG) { " Download URL: #{endpoint_download_url}, response: #{response&.status || -1}" } - return false + result = false + rescue StandardError => e + logger.error(LOG_TAG) { " Connection to \"#{endpoint_download_url}\" errored:" } + logger.error(LOG_TAG) { e } + logger.debug(LOG_TAG) { " Failed to retrieve package: (#{package.category}:#{package.subcategory}:#{package.name}:#{package.version})" } + logger.debug(LOG_TAG) { " Download URL: #{endpoint_download_url}, response: #{response&.status || -1}" } + + result = false end - rescue Excon::Error::Timeout => e - logger.error(LOG_TAG) { " Connection to \"#{endpoint_download_url}\" timed out after: #{W3DHub::Api::API_TIMEOUT} seconds" } - logger.error(LOG_TAG) { e } - logger.debug(LOG_TAG) { " Failed to retrieve package: (#{package.category}:#{package.subcategory}:#{package.name}:#{package.version})" } - logger.debug(LOG_TAG) { " Download URL: #{endpoint_download_url}, response: #{response&.status || -1}" } - return false - rescue Excon::Error => e - logger.error(LOG_TAG) { " Connection to \"#{endpoint_download_url}\" errored:" } - logger.error(LOG_TAG) { e } - logger.debug(LOG_TAG) { " Failed to retrieve package: (#{package.category}:#{package.subcategory}:#{package.name}:#{package.version})" } - logger.debug(LOG_TAG) { " Download URL: #{endpoint_download_url}, response: #{response&.status || -1}" } - - return false + result ensure file&.close end + # Download a W3D Hub package + def self.fetch_package(package, block) + async_fetch_package(package, block) + end + def self.acquire_net_lock(key) Store["net_locks"] ||= {} diff --git a/w3d_hub_linux_launcher.rb b/w3d_hub_linux_launcher.rb index 6a42336..ac73b04 100644 --- a/w3d_hub_linux_launcher.rb +++ b/w3d_hub_linux_launcher.rb @@ -14,7 +14,10 @@ require "logger" require "time" require "base64" require "zip" -require "excon" +require "async" +require "async/http/endpoint" +require "async/websocket/client" +require "async/http/internet/instance" class W3DHub W3DHUB_DEBUG = ARGV.join.include?("--debug") @@ -32,7 +35,7 @@ class W3DHub FileUtils.mkdir_p(CACHE_PATH) unless Dir.exist?(CACHE_PATH) FileUtils.mkdir_p(LOGS_PATH) unless Dir.exist?(LOGS_PATH) - LOGGER = Logger.new("#{LOGS_PATH}/w3d_hub_linux_launcher.log", "daily") + LOGGER = W3DHUB_DEBUG ? Logger.new(STDOUT) : Logger.new("#{LOGS_PATH}/w3d_hub_linux_launcher.log", "daily") LOGGER.level = Logger::Severity::DEBUG # W3DHUB_DEBUG ? Logger::Severity::DEBUG : Logger::Severity::WARN LOG_TAG = "W3DHubLinuxLauncher"