From f6fbd1bce181ef0289d07ceb21175eb8fe499575 Mon Sep 17 00:00:00 2001 From: Cyberarm Date: Thu, 8 Oct 2026 14:12:18 -0500 Subject: [PATCH] Fixed unable to download ECW due to spaces in the file names, added checksum caching for Task to make updating much faster for most games (IA with its mostly loose files doesn't gain anything and might even be slower... Consider not performing any verification for non-repair tasks?) --- lib/pages/games.rb | 2 +- lib/worker/api/legacy_manifest_package.rb | 6 ++ lib/worker/api/verification_data.rb | 107 ++++++++++++++++++++++ lib/worker/task.rb | 54 +++++++++-- lib/worker/tasks/repair_application.rb | 3 + w3d_hub_linux_launcher.rb | 1 + 6 files changed, 164 insertions(+), 9 deletions(-) create mode 100644 lib/worker/api/verification_data.rb diff --git a/lib/pages/games.rb b/lib/pages/games.rb index e4fe3de..8da77f0 100644 --- a/lib/pages/games.rb +++ b/lib/pages/games.rb @@ -47,7 +47,7 @@ module W3DHubLauncher def blur @parent.hide_battleview_panel - CyberarmEngine::EventBus.unsubscribe("application_changed") + CyberarmEngine::EventBus.unsubscribe("application_changed", self) end def populate_game_content_container diff --git a/lib/worker/api/legacy_manifest_package.rb b/lib/worker/api/legacy_manifest_package.rb index 88bf519..9b2054b 100644 --- a/lib/worker/api/legacy_manifest_package.rb +++ b/lib/worker/api/legacy_manifest_package.rb @@ -19,6 +19,12 @@ module W3DHubLauncher @sha256_checksum = hash["checksum"] @checksum_chunks = hash["checksum-chunks"] @download_url = hash["download_url"] + + # Fix crashing on spaces + if @download_url + uri_path = @download_url.split("/").last + @download_url = @download_url.sub(uri_path, URI.encode_uri_component(uri_path)) + end end def error? diff --git a/lib/worker/api/verification_data.rb b/lib/worker/api/verification_data.rb new file mode 100644 index 0000000..d33e83a --- /dev/null +++ b/lib/worker/api/verification_data.rb @@ -0,0 +1,107 @@ +module W3DHubLauncher + class Worker + class Api + + # Cached verifcation data for application and pending package files + class VerificationData + SCHEMA = 0 + + def initialize(path) + @path = path + + data = {} + if File.exist?(@path) + begin + data = JSON.parse(File.read(@path)) + rescue JSON::ParserError + data = {} + end + end + data = {} unless data.is_a?(Hash) + + @schema = data["schema"] || SCHEMA + @data = {} + data["data"]&.each do |key, value| + @data[key] = Data.new(value) + end + end + + def key(file_name, file_version: "", package: false) + file_name = file_name.gsub("\\", "/").downcase + + if package + "package:{file_name}:#{file_version}" + else + file_name + end + end + + def data(key) + @data[key] + end + + def unchanged?(key, checksum:, file_path:) + data = data(key) + return nil unless data + + return data if data.checksum.upcase.strip == checksum.upcase.strip && + data.modification_time == File.mtime(file_path).to_i && + data.file_size == File.size(file_path) + + nil + end + + def set(key, checksum:, file_path:) + data = Data.new( + { + "checksum" => checksum.upcase.strip, + "modification_time" => File.mtime(file_path).to_i, + "file_size" => File.size(file_path) + } + ) + + @data[key] = data + save + + data + end + + def delete(key) + data = @data.delete(key) + save + + data + end + + def save + File.write(@path, JSON.pretty_generate(self)) + end + + def to_json(options = {}) + { + schema: SCHEMA, + data: @data + }.to_json(options) + end + + class Data + attr_reader :checksum, :modification_time, :file_size + + def initialize(data) + @checksum = data["checksum"] + @modification_time = data["modification_time"] + @file_size = data["file_size"] + end + + def to_json(options = {}) + { + checksum: @checksum, + modification_time: @modification_time, + file_size: @file_size + }.to_json(options) + end + end + end + end + end +end diff --git a/lib/worker/task.rb b/lib/worker/task.rb index c9e33c8..67e778c 100644 --- a/lib/worker/task.rb +++ b/lib/worker/task.rb @@ -19,6 +19,8 @@ module W3DHubLauncher @status = Worker::Request::STATUS_PENDING + @verification_data = Worker::Api::VerificationData.new("#{CACHE_PATH}/verification_data-#{@application.id}-#{@channel.id}.json") + # caching mechinism for file path normalization @file_index = {} # manifests! @@ -32,6 +34,10 @@ module W3DHubLauncher # packages to verify and/or download @packages = [] + # whether task should perform a per file checksum or just read it from the cache and + # check file modification time and size + @force_verification = false + setup end @@ -136,7 +142,7 @@ module W3DHubLauncher # Process manifest game files in NEWEST to OLDEST order so that we don't erroneously flag # valid files as invalid due to an OLDER version of the file being checked FIRST. - @manifest_files.reverse.each do |file| + @manifest_files.reverse.each_with_index do |file, i| break unless File.directory?(@installation_directory) # skip irrelevant older versions of files @@ -147,8 +153,15 @@ module W3DHubLauncher next unless File.exist?(file_path) # && !File.directory?(file_path) puts "verifying #{file_path}..." - checksum = cached_checksums[file_path] || Digest::SHA256.file(file_path).hexdigest.upcase - cached_checksums[file_path] ||= checksum + key = @verification_data.key(file.name) + pp [key, @force_verification] + verification_data = @force_verification ? nil : @verification_data.unchanged?(key, checksum: file.checksum, file_path: file_path) + + # Use cached checksum if the file appears unchanged, otherwise recalculate it + checksum = verification_data&.checksum || Digest::SHA256.file(file_path).hexdigest.upcase + + # Set or update cached verification data if files don't match + @verification_data.set(key, checksum: checksum, file_path: file_path) unless verification_data if checksum == file.checksum @required_manifest_files.delete(file) @@ -159,6 +172,8 @@ module W3DHubLauncher else puts " failed verification:#{file.version}" end + + update_status(label: "Verifying files...", fraction: i.to_f / @manifest_files.size) end end @@ -249,7 +264,7 @@ module W3DHubLauncher if manifest_file.patch? apply_patch(manifest_file, package) else - unzip(package_path) + unzip(manifest_file, package_path) end processed_packages[package_path] = manifest_file @@ -265,6 +280,8 @@ module W3DHubLauncher if File.exist?(file_path) && !File.directory?(file_path) puts "Removing file: #{file_path}" File.delete(file_path) + key = @verification_data.key(manifest_file.name) + @verification_data.delete(key) remove_from_file_index(file_path) end @@ -285,7 +302,7 @@ module W3DHubLauncher end end - # updated, and moved applications will overwrite existing application data in settings + # updated, repaired, and moved applications will overwrite existing application data in settings def mark_application_installed app = Worker::Api::Settings::Application.create( id: @application.id, @@ -469,7 +486,7 @@ module W3DHubLauncher package_path = package_cache_path(package) puts "unpacking patch..." - unzip(package_path, f.path) + unzip(false, package_path, output_path: f.path) puts "reading patch data.." patch_mix = W3DHubLauncher::WWMix.new(path: f.path) @@ -504,12 +521,16 @@ module W3DHubLauncher raise temp_mix.error_reason unless temp_mix.save puts "copying file..." - FileUtils.cp(temp.path, normalize_path(manifest_file.name)) + file_path = normalize_path(manifest_file.name) + FileUtils.cp(temp.path, file_path) + + key = @verification_data.key(manifest_file.name) + @verification_data.set(key, checksum: manifest_file.checksum, file_path: file_path) end end end - def unzip(zip_path, output_path = nil) + def unzip(manifest_file, zip_path, output_path: nil) stream = Zip::InputStream.new(File.open(zip_path)) while(entry = stream.get_next_entry) @@ -529,6 +550,23 @@ module W3DHubLauncher end end + + # the manifest file we receive MIGHT NOT be for the current zip file entry + # so look it up to be safe + if manifest_file + manifest = @manifests[manifest_file.version] + if manifest + entry_manifest_file = manifest.files.find do |f| + f.name.gsub("\\", "/").downcase == entry.name.gsub("\\", "/").downcase + end + + if entry_manifest_file + key = @verification_data.key(entry.name) + @verification_data.set(key, checksum: entry_manifest_file.checksum, file_path: file_path) + end + end + end + add_to_file_index(file_path) end end diff --git a/lib/worker/tasks/repair_application.rb b/lib/worker/tasks/repair_application.rb index 68cf100..933d021 100644 --- a/lib/worker/tasks/repair_application.rb +++ b/lib/worker/tasks/repair_application.rb @@ -4,6 +4,9 @@ module W3DHubLauncher def setup super + # User requested a repair, make sure everything is as it should be! + @force_verification = true + @task_verb = "Repairing".freeze end end diff --git a/w3d_hub_linux_launcher.rb b/w3d_hub_linux_launcher.rb index 59baa2b..0dc4dd2 100644 --- a/w3d_hub_linux_launcher.rb +++ b/w3d_hub_linux_launcher.rb @@ -60,6 +60,7 @@ require_relative "lib/worker/api/server_event" require_relative "lib/worker/api/test_event" require_relative "lib/worker/api/legacy_manifest" require_relative "lib/worker/api/legacy_manifest_package" +require_relative "lib/worker/api/verification_data" require_relative "lib/worker/request" require_relative "lib/worker/w3dhub_api" require_relative "lib/worker/task"