diff --git a/lib/worker.rb b/lib/worker.rb index a378228..328b245 100644 --- a/lib/worker.rb +++ b/lib/worker.rb @@ -8,7 +8,7 @@ module W3DHubLauncher Response = Data.define(:status, :request_id, :result) - attr_reader :w3dhub_api + attr_reader :w3dhub_api, :settings def initialize end @@ -35,7 +35,7 @@ module W3DHubLauncher @client = nil - @settings = 0# Settings.new + @settings = nil @game_servers = [] @host_pings = {} @@ -325,6 +325,7 @@ module W3DHubLauncher begin if File.exist?(path) && File.size(path).positive? json = File.read(path) + @settings = Api::Settings.new(JSON.parse(json)) result.data = json else result.error = RuntimeError.new("Launcher settings file does not exist or is empty.") diff --git a/lib/worker/api/legacy_manifest_package.rb b/lib/worker/api/legacy_manifest_package.rb index b8d9fcd..92fa47a 100644 --- a/lib/worker/api/legacy_manifest_package.rb +++ b/lib/worker/api/legacy_manifest_package.rb @@ -15,7 +15,7 @@ module W3DHubLauncher @name = hash["name"] @error = hash["error"] @file_size = Integer(hash["size"] || 0) - @checksum_chunk_size = Integer(hash["checksum-size"] || 0) + @checksum_chunk_size = Integer(hash["checksum-chunk-size"] || 0) @sha256_checksum = hash["checksum"] @checksum_chunks = hash["checksum-chunks"] @download_url = hash["download_url"] @@ -25,6 +25,10 @@ module W3DHubLauncher @error end + def version? + @version != Gem::Version.new("0.0.0.0") + end + # checksum whole file and file chunks until a mismatch occurs or the whole file is verified. def verify_file(filename) file_size = File.size(filename) diff --git a/lib/worker/api/settings.rb b/lib/worker/api/settings.rb index 6a4ff91..00adeed 100644 --- a/lib/worker/api/settings.rb +++ b/lib/worker/api/settings.rb @@ -76,15 +76,16 @@ module W3DHubLauncher attr_reader :id, :channel attr_accessor :version, :installation_path, :wine_prefix_path, :launch_command - def self.create(id:, channel:, version:, installation_path:, wine_prefix_path:, launch_command:) + def self.create(id:, channel:, version:, installation_path:, wine_prefix_path:, launch_command:, timestamp:) hash = { "schema" => SCHEMA, "id" => id, "channel" => channel, - "version" => Gem::Version.new(version), + "version" => version, "installation_path" => installation_path, "wine_prefix_path" => wine_prefix_path, "launch_command" => launch_command, + "timestamp" => timestamp } self.new(hash) @@ -99,6 +100,7 @@ module W3DHubLauncher @installation_path = data["installation_path"] @wine_prefix_path = data["wine_prefix_path"] @launch_command = data["launch_command"] + @timestamp = Time.at(data["timestamp"]) end def to_json(options = {}) @@ -109,7 +111,8 @@ module W3DHubLauncher version: @version.to_s, installation_path: @installation_path, wine_prefix_path: @wine_prefix_path, - launch_command: @launch_command + launch_command: @launch_command, + timestamp: @timestamp.to_i }.to_json(options) end end diff --git a/lib/worker/task.rb b/lib/worker/task.rb index 0a5abd1..844d6a6 100644 --- a/lib/worker/task.rb +++ b/lib/worker/task.rb @@ -1,5 +1,7 @@ module W3DHubLauncher class Task + IndexedFile = Data.define(:type, :path) + attr_reader :request_id, :application, :channel, :installed_version, :target_version def initialize(request_id:, application:, channel:, installed_version:, target_version:) @@ -11,7 +13,18 @@ module W3DHubLauncher pp self + # caching mechinism for file path normalization + @file_index = {} + # manifests! @manifests = {} + # files that must exist for a valid installation + @manifest_files = [] + # files that should be removed + @deleted_manifest_files = [] + # files needed to make installation valid for target version + @required_manifest_files = [] + # packages to verify and/or download + @packages = [] setup end @@ -20,7 +33,9 @@ module W3DHubLauncher end def start(worker) - @worker = worker + @worker = worker + + @installation_directory ||= application_target_installation_directory execute end @@ -33,6 +48,16 @@ module W3DHubLauncher # high level methods + def index_installation_directory + return unless File.directory?(@installation_directory) + + Dir.glob("#{@installation_directory}/**/**").each do |path| + path.gsub!("\\", "/") + + @file_index[path.downcase] = IndexedFile.new(File.directory?(path) ? :directory : :file, path) + end + end + def fetch_manifests(version = @target_version) while (manifest = fetch_manifest(version)) @manifests[version] = manifest @@ -74,23 +99,170 @@ module W3DHubLauncher end end - pp [:files, files.map(&:name), :deleted_files, deleted_files.map(&:name)] - end - - def remove_deleted_files + @manifest_files = files + @deleted_manifest_files = deleted_files end def verify_files + @required_manifest_files = @manifest_files.clone + + # Process manifest game files in NEWEST to OLDEST order so that we don't erroneously flag + # valid files as invalid due to an OLDER version of the file being checked FIRST. + @manifest_files.reverse.each do |file| + break unless File.directory?(@installation_directory) + + # skip irrelevant older versions of files + next unless @required_manifest_files.include?(file) + + file_path = normalize_path(file.name) + + next unless File.exist?(file_path) # && !File.directory?(file_path) + + puts "verifying #{file_path}..." + checksum = Digest::SHA256.file(file_path).hexdigest.upcase + + if checksum == file.checksum + @required_manifest_files.delete(file) + + # remove irrelevant older versions of the file from consideration + @required_manifest_files.delete_if { |f| f.name.casecmp?(file.name) && f.version < file.version } + end + end end def fetch_packages + required_packages = {} + @required_manifest_files.each do |file| + required_packages[file.version] ||= [] + + next if required_packages[file.version].find { |pkg| pkg.casecmp?(file.package) } + + required_packages[file.version] << file.package + end + + packages = required_packages.map do |version, packages| + packages.map do |package_name| + { + category: @application.category, + subcategory: @application.id, + name: format("%s.zip", package_name), + version: version.to_s + } + end + end.flatten + + result = @worker.w3dhub_api.fetch_package_details(packages) + + unless result.okay? + abort_task!("Failed to fetch required package details!") + end + + response = JSON.parse(result.data) + manifest_packages = response["packages"]&.map { |item| Worker::Api::LegacyManifestPackage.new(item) } + if (failed_packages = manifest_packages.select(&:error?)) && !failed_packages.empty? + abort_task!("Failed to retrieve packages details for: #{failed_packages.map { |pkg| "#{pkg.name}:#{pkg.version}: #{pkg.error}"}.join(', ') }") + end + + manifest_packages.each do |pkg| + # FIXME: verify local packages to prevent overdownloading! + # pkg.verify_file(normalize_path(pkg.name)) + + file_path = package_cache_path(pkg) + unless File.directory?(File.dirname(file_path)) + puts "creating directory: #{File.dirname(file_path)}" + FileUtils.mkdir_p(File.dirname(file_path)) + end + + result = if pkg.download_url + puts "downloading #{pkg.download_url} to #{file_path}" + @worker.w3dhub_api.download(pkg.download_url, path: file_path) + else + # TODO xD + @worker.w3dhub_api.fetch_package("TODO") + end + + abort_task!("Failed to download required package: #{pkg.name}:#{pkg.version}") unless result.okay? + end end def install_packages end + def remove_deleted_files + end + + def write_paths_ini + end + + # updated, and moved applications will overwrite existing application data in settings + def mark_application_installed + end + + def mark_application_uninstalled + end + # helper functions + def application_target_installation_directory + target_application = @worker.settings.applications.find { |app| app.id == @application.id && app.channel == @channel.id } + if target_application + target_application.installation_path + else + format("%s/%s/%s", @worker.settings.preferences.application_installation_directory, @application.id, @channel.id) + end + end + + def package_cache_path(package) + directory = @worker.settings.preferences.launcher_package_cache_directory + + if package.version? + format("%s/%s/%s", directory, package.version.to_s, package.name) + else + format("%s/%s", directory, package.name) + end + end + + # And behold, the backslashes were slain and the path deemed sane! + def normalize_path(base_path) + base_path = base_path.gsub("\\", "/") + path = "#{@installation_directory}/#{base_path}".gsub("\\", "/") + + # try to find exact, existing match + index_file = @file_index[path.downcase] + return index_file.path if index_file + + # try to find directory structure + if (index_directory = @file_index[File.dirname(path).downcase]) + return format("%s/%s", index_directory.path, File.basename(path)) + end + + # walk directories to see if we have a partial path match + sub_paths = base_path.split("/") + sub_paths.pop # drop file name from sub path reconstruction + unless sub_paths.empty? + partial_index_directory = nil + sub_path = format("%s/%s", @installation_directory, sub_paths.shift).downcase + while (partial_index_directory = @file_index[sub_path.downcase]) + sub_path = format("%s/%s", sub_path, sub_paths.shift) + end + + if partial_index_directory + if sub_paths.empty? # we've found the path! + return format("%s/%s", partial_index_directory.path, File.basename(base_path)) + else # we has a directory or two that don't exist, downcase them, and then head out to the pub! + return format("%s/%s/%s", partial_index_directory.path, sub_paths.map(&:downcase).join("/"), File.basename(base_path)) + end + end + end + + # path doesn't exist, convert directories to lower-case, and use provided file case + if File.basename(path) == base_path # base_path doesn't have a sub directory + format("%s/%s", File.dirname(path), File.basename(path)) + else # base_path has sub directory/directories + format("%s/%s/%s", @installation_directory, File.dirname(base_path).downcase, File.basename(path)) + end + end + def fetch_manifest(version) result = @worker.w3dhub_api.fetch_package_details([ { diff --git a/lib/worker/tasks/install_application.rb b/lib/worker/tasks/install_application.rb index 257de5b..c88da90 100644 --- a/lib/worker/tasks/install_application.rb +++ b/lib/worker/tasks/install_application.rb @@ -8,6 +8,9 @@ module W3DHubLauncher # TODO: # fail_fast + # get case-insensitive file list + index_installation_directory + # fetch manifests for currently installed version # AND the target version so we can detect files that need to be removed # between versions @@ -16,13 +19,23 @@ module W3DHubLauncher # build list of packages we'll need to download build_package_list + # verify our pre-existing files for presence and validity verify_files + # download required packages to get missing or out of date files fetch_packages + # unpack packages and patch files to reach target version install_packages + # delete files that are no longer part of the application (and whose presence my break the application) remove_deleted_files + + # tell the application to behive + write_paths_ini + + # store application install state and notify frontend/ui that the application is ready for use! + mark_application_installed end end end diff --git a/lib/worker/w3dhub_api.rb b/lib/worker/w3dhub_api.rb index 0b15d94..d280bdf 100644 --- a/lib/worker/w3dhub_api.rb +++ b/lib/worker/w3dhub_api.rb @@ -61,27 +61,25 @@ module W3DHubLauncher result = CyberarmEngine::Result.new Sync do |task| - task.with_timeout(API_TIMEOUT) do - Async::HTTP::Internet.send(method, url, headers, body) do |response| - if response.success? - content_length = response.headers["content-length"] || 0 + Async::HTTP::Internet.send(method, url, headers, body) do |response| + if response.success? + content_length = response.headers["content-length"] || 0 - total_downloaded_bytes = 0 - File.open(path, "wb") do |file| - response.each do |chunk| - file.write(chunk) - downloaded_bytes = chunk.length - total_downloaded_bytes += downloaded_bytes + total_downloaded_bytes = 0 + File.open(path, "wb") do |file| + response.each do |chunk| + file.write(chunk) + downloaded_bytes = chunk.length + total_downloaded_bytes += downloaded_bytes - block&.call(downloaded_bytes, total_downloaded_bytes, content_length) - end + block&.call(downloaded_bytes, total_downloaded_bytes, content_length) end - - result.data = true end - rescue StandardError => e - result.error = e + + result.data = true end + rescue StandardError => e + result.error = e rescue Async::TimeoutError result.error = e end