From 82add3cc9dffc626c1b6a963a5028af830349ef9 Mon Sep 17 00:00:00 2001 From: cyberarm Date: Tue, 28 Dec 2021 17:55:40 -0600 Subject: [PATCH] Brought back Excon for package downloading as the method recommended by async-http is unreliable, added support for importing games, repairer and updater tasks are both now simple subclasses of installer, implemented verify_files for checking installed files to prune download package list (currently causes Api.package_details to fail..., so disabled for now), misc. changes. --- lib/api.rb | 10 +- lib/application_manager.rb | 19 ++- lib/application_manager/pool.rb | 6 +- lib/application_manager/task.rb | 158 ++++++++++++++------- lib/application_manager/tasks/importer.rb | 43 +++++- lib/application_manager/tasks/installer.rb | 3 + lib/application_manager/tasks/repairer.rb | 26 ++-- lib/cache.rb | 69 +++------ lib/pages/games.rb | 12 +- lib/states/boot.rb | 2 + lib/states/interface.rb | 4 +- locales/en.yml | 1 + 12 files changed, 221 insertions(+), 132 deletions(-) diff --git a/lib/api.rb b/lib/api.rb index 6ffe5be..1a56337 100644 --- a/lib/api.rb +++ b/lib/api.rb @@ -2,7 +2,8 @@ class W3DHub class Api USER_AGENT = "Cyberarm's Linux Friendly W3D Hub Launcher v#{W3DHub::VERSION}".freeze DEFAULT_HEADERS = [ - ["User-Agent", USER_AGENT] + ["User-Agent", USER_AGENT], + ["Accept", "application/json"] ].freeze FORM_ENCODED_HEADERS = ( DEFAULT_HEADERS + [["Content-Type", "application/x-www-form-urlencoded"]] @@ -126,15 +127,14 @@ class W3DHub # /apis/launcher/1/get-package-details # client requests package details: data={"packages":[{"category":"games","name":"apb.ico","subcategory":"apb","version":""}]} def self.package_details(internet, packages) - body = "data=#{JSON.dump({ packages: packages })}" + body = URI.encode_www_form("data": JSON.dump({ packages: packages })) response = internet.post("#{ENDPOINT}/apis/launcher/1/get-package-details", FORM_ENCODED_HEADERS, body) if response.success? hash = JSON.parse(response.read, symbolize_names: true) - packages = hash[:packages].map { |pkg| Package.new(pkg) } - return packages.first if packages.size == 1 - return packages + hash[:packages].map { |pkg| Package.new(pkg) } else + pp response, body false end end diff --git a/lib/application_manager.rb b/lib/application_manager.rb index 7ed4928..1807c61 100644 --- a/lib/application_manager.rb +++ b/lib/application_manager.rb @@ -36,14 +36,14 @@ class W3DHub @tasks.push(updater) end - def import(app_id, channel, path) - puts "Import Request: #{app_id}-#{channel} -> #{path}" + def import(app_id, channel) + puts "Import Request: #{app_id}-#{channel}" # Check registry for auto-import if windows # if auto-import fails ask user for path to game exe # mark app as imported/installed - @tasks.push(Importer.new(app_id, channel, path)) + @tasks.push(Importer.new(app_id, channel)) end def settings(app_id, channel) @@ -238,6 +238,19 @@ class W3DHub end end + def imported!(task, exe_path) + application_data = { + install_directory: File.basename(exe_path), + installed_version: task.channel.current_version, + install_path: exe_path, + wine_prefix: task.wine_prefix + } + + Store.settings[:games] ||= {} + Store.settings[:games][:"#{task.app_id}_#{task.release_channel}"] = application_data + Store.settings.save_settings + end + def installed!(task) # install_dir # installed_version diff --git a/lib/application_manager/pool.rb b/lib/application_manager/pool.rb index 98b92b7..ea9eb25 100644 --- a/lib/application_manager/pool.rb +++ b/lib/application_manager/pool.rb @@ -11,7 +11,7 @@ class W3DHub end def manage_pool - while (@jobs.size.positive? || @workers.any?(&:busy?)) + while @jobs.size.positive? || @workers.any?(&:busy?) feed_pool unless @jobs.size.zero? sleep 0.1 @@ -29,9 +29,9 @@ class W3DHub @die = false @job = nil - Async do + Thread.new do until (@die) - @job.process if @job && @job.waiting? + @job.process if @job&.waiting? @job = nil sleep 0.1 end diff --git a/lib/application_manager/task.rb b/lib/application_manager/task.rb index 7e27796..027559b 100644 --- a/lib/application_manager/task.rb +++ b/lib/application_manager/task.rb @@ -43,23 +43,30 @@ class W3DHub # Start task, inside its own thread # FIXME: Ruby 3 has parallelism now: Use a Ractor to do work on a seperate core to - # prevent the UI for locking up while doing computation heavy work, i.e building + # prevent the UI from locking up while doing computation heavy work, i.e building # list of packages to download def start @task_state = :running - Async do - status = execute_task + Thread.new do + Sync do + begin + status = execute_task + rescue RuntimeError => e + status = false + @task_failure_reason = e.message[0..512] + end - # Force free some bytes - GC.compact if GC.respond_to?(:compact) - GC.start + # Force free some bytes + GC.compact if GC.respond_to?(:compact) + GC.start - @task_state = :failed unless status - @task_state = :complete unless @task_state == :failed + @task_state = :failed unless status + @task_state = :complete unless @task_state == :failed - hide_application_taskbar if @task_state == :failed - send_message_dialog(:failure, "Task #{type.inspect} failed for #{@application.name}", @task_failure_reason) if @task_state == :failed + hide_application_taskbar if @task_state == :failed + send_message_dialog(:failure, "Task #{type.inspect} failed for #{@application.name}", @task_failure_reason) if @task_state == :failed && !@fail_silently + end end end @@ -100,6 +107,10 @@ class W3DHub @task_failure_reason = reason.to_s end + def fail_silently! + @fail_silently = true + end + # Quick checks before network and computational work starts def fail_fast # tar present? @@ -211,6 +222,89 @@ class W3DHub packages end + def verify_files(manifests, packages) + @status.operations.clear + @status.label = "Downloading #{@application.name}..." + @status.value = "Verifying installed files..." + @status.progress = 0.0 + + @status.step = :verify_files + + path = Cache.install_path(@application, @channel) + accepted_files = {} + rejected_files = [] + + file_count = manifests.map { |m| m.files.count }.sum + processed_files = 0 + + manifests.each do |manifest| + manifest.files.each do |file| + safe_file_name = file.name.gsub("\\", "/") + # Fix borked data -> Data 'cause Windows don't care about capitalization + safe_file_name.sub!("data/", "Data/") unless File.exist?("#{path}/#{safe_file_name}") + + file_path = "#{path}/#{safe_file_name}" + + processed_files += 1 + @status.progress = processed_files.to_f / file_count + + next if file.removed_since + next if accepted_files.key?(safe_file_name) + + unless File.exist?(file_path) + rejected_files << { file: file, manifest_version: manifest.version } + puts "[#{manifest.version}] File missing: #{file_path}" + next + end + + digest = Digest::SHA256.new + f = File.open(file_path) + + while (chunk = f.read(32_000_000)) + digest.update(chunk) + end + + f.close + + pp file if file.checksum.nil? + + if digest.hexdigest.upcase == file.checksum.upcase + accepted_files[safe_file_name] = manifest.version + # puts "[#{manifest.version}] Verified file: #{file_path}" + else + rejected_files << { file: file, manifest_version: manifest.version } + puts "[#{manifest.version}] File failed Verification: #{file_path}" + end + end + end + + puts "#{rejected_files.count} missing or corrupt files" + + # TODO: Filter packages to only the required ones + selected_packages = [] + selected_packages_hash = {} + + rejected_files.each do |hash| + next if selected_packages_hash["#{hash[:file].package}_#{hash[:manifest_version]}"] + + package = packages.find { |pkg| pkg.name == hash[:file].package && pkg.version == hash[:manifest_version] } + + if package + selected_packages_hash["#{hash[:file].package}_#{hash[:manifest_version]}"] = true + selected_packages << package + else + raise "missing package: #{hash[:file].package}:#{hash[:manifest_version]} in fetched packages list!" + end + end + + # FIXME: Order `selected_packages` like `packages` + + # Removed packages that don't need to be fetched or processed + packages.delete_if { |package| !selected_packages.find { |pkg| pkg == package } } + + packages + end + def fetch_packages(packages) hashes = packages.map do |pkg| { @@ -306,8 +400,7 @@ class W3DHub pool.manage_pool else - puts "FAILED!" - pp package_details + fail!("Failed to fetch package details") end end @@ -406,45 +499,6 @@ class W3DHub puts "#{@app_id} has been installed." end - def verify_files(manifests, packages) - path = Cache.install_path(@application, @channel) - accepted_files = {} - rejected_files = [] - - manifests.each do |manifest| - manifest.files.each do |file| - file_path = "#{path}/#{file.name.gsub('\\', '/')}" - - unless File.exists?(file_path) - rejected_files << file - puts "[#{manifest.version}] File missing: #{file_path}" - next - end - - next if accepted_files.key?(file.name) - - digest = Digest::SHA256.new - f = File.open(file_path) - - while (chunk = f.read(32_000_000)) - digest.update(chunk) - current = Async::Task.current? - current&.yield - end - - f.close - - if digest.hexdigest.upcase == file.checksum.upcase - accepted_files[file.name] = manifest.version - puts "[#{manifest.version}] Verified file: #{file_path}" - else - rejected_files << file - puts "[#{manifest.version}] File failed Verification: #{file_path}" - end - end - end - end - ############# # Functions # ############# @@ -453,7 +507,7 @@ class W3DHub # Check for and integrity of local manifest internet = Async::HTTP::Internet.instance - package = Api.package_details(internet, [{ category: category, subcategory: subcategory, name: name, version: version }]) + package = Api.package_details(internet, [{ category: category, subcategory: subcategory, name: name, version: version }]).first if File.exist?(Cache.package_path(category, subcategory, name, version)) verified = verify_package(package) diff --git a/lib/application_manager/tasks/importer.rb b/lib/application_manager/tasks/importer.rb index 4506240..0189de7 100644 --- a/lib/application_manager/tasks/importer.rb +++ b/lib/application_manager/tasks/importer.rb @@ -1,10 +1,47 @@ class W3DHub class ApplicationManager class Importer < Task - def initialize(app_id, channel, path = nil) - super(app_id, channel) + def type + :importer + end - @path = path + def execute_task + path = ask_file + + unless File.exist?(path) && !File.directory?(path) + fail!("File #{path.inspect} does not exist or is a directory") + fail_silently! if path.nil? || path&.length&.zero? # User likely canceled the file selection + end + + return false if failed? + + Store.application_manager.imported!(self, path) + + true + end + + def ask_file(title: "Open File", filter: "*game*.exe") + if W3DHub.unix? + # search for command + cmds = %w{ zenity matedialog qarma kdialog } + + command = cmds.find do |cmd| + cmd if system("which #{cmd}") + end + + path = case File.basename(command) + when "zenity", "matedialog", "qarma" + `#{command} --file-selection --title "#{title}" --file-filter "#{filter}"` + when "kdialog" + `#{command} --title "#{title}" --getopenfilename . "#{filter}"` + else + raise "No known command found for system file selection dialog!" + end + + path.strip + else + raise NotImplementedError + end end end end diff --git a/lib/application_manager/tasks/installer.rb b/lib/application_manager/tasks/installer.rb index 35a9543..a697217 100644 --- a/lib/application_manager/tasks/installer.rb +++ b/lib/application_manager/tasks/installer.rb @@ -15,6 +15,9 @@ class W3DHub packages = build_package_list(manifests) return false if failed? + # verify_files(manifests, packages) + # return false if failed? + fetch_packages(packages) return false if failed? diff --git a/lib/application_manager/tasks/repairer.rb b/lib/application_manager/tasks/repairer.rb index 7371a3a..dd36946 100644 --- a/lib/application_manager/tasks/repairer.rb +++ b/lib/application_manager/tasks/repairer.rb @@ -1,27 +1,27 @@ class W3DHub class ApplicationManager - class Repairer < Task + class Repairer < Installer def type :repairer end - def execute_task - fail_fast - return false if failed? + # def execute_task + # fail_fast + # return false if failed? - manifests = fetch_manifests - return false if failed? + # manifests = fetch_manifests + # return false if failed? - packages = build_package_list(manifests) - return false if failed? + # packages = build_package_list(manifests) + # return false if failed? - verify_files(manifests, packages) - return false if failed? + # verify_files(manifests, packages) + # return false if failed? - # pp packages.select { |pkg| pkg.name == "misc" } + # # pp packages.select { |pkg| pkg.name == "misc" } - true - end + # true + # end end end end \ No newline at end of file diff --git a/lib/cache.rb b/lib/cache.rb index d84e5ce..393a377 100644 --- a/lib/cache.rb +++ b/lib/cache.rb @@ -48,63 +48,38 @@ class W3DHub # Download a W3D Hub package def self.fetch_package(internet, package, block) path = package_path(package.category, package.subcategory, package.name, package.version) + headers = { "Content-Type": "application/x-www-form-urlencoded", "User-Agent": Api::USER_AGENT } start_from_bytes = package.custom_partially_valid_at_bytes - puts " Start from bytes: #{start_from_bytes}" + puts " Start from bytes: #{start_from_bytes} of #{package.size}" create_directories(path) - offset = start_from_bytes - parts = [] - chunk_size = 4_000_000 - workers = 4 + file = File.open(path, start_from_bytes.positive? ? "r+b" : "wb") - file = File.open(path, offset.positive? ? "r+b" : "wb") - - amount_written = 0 - - while (offset < package.size) - byte_range_start = offset - byte_range_end = [offset + chunk_size, package.size].min - parts << (byte_range_start...byte_range_end) - - offset += chunk_size + if start_from_bytes.positive? + headers["Range"] = "bytes=#{start_from_bytes}-" + file.pos = start_from_bytes end - semaphore = Async::Semaphore.new(workers) - barrier = Async::Barrier.new(parent: semaphore) + streamer = lambda do |chunk, remaining_bytes, total_bytes| + file.write(chunk) - while !parts.empty? - barrier.async do - part = parts.shift - - range_header = [["range", "bytes=#{part.min}-#{part.max}"]] - - body = "data=#{JSON.dump({ category: package.category, subcategory: package.subcategory, name: package.name, version: package.version })}" - response = internet.post("#{Api::ENDPOINT}/apis/launcher/1/get-package", W3DHub::Api::FORM_ENCODED_HEADERS + range_header, body) - - if response.success? - chunk = response.read - written = 0 - if W3DHub.unix? - written = file.pwrite(chunk, part.min) - else - # probably not "thread safe" - file.pos = part.min - written = file.write(chunk) - end - - amount_written += written - remaining_bytes = package.size - amount_written - total_bytes = package.size - - block.call(chunk, remaining_bytes, total_bytes) - # puts " Remaining: #{((remaining_bytes.to_f / total_bytes) * 100.0).round}% (#{W3DHub::format_size(total_bytes - remaining_bytes)} / #{W3DHub::format_size(total_bytes)})" - end - end - - barrier.wait + block.call(chunk, remaining_bytes, total_bytes) + # puts " Remaining: #{((remaining_bytes.to_f / total_bytes) * 100.0).round}% (#{W3DHub::format_size(total_bytes - remaining_bytes)} / #{W3DHub::format_size(total_bytes)})" end + + # Create a new connection due to some weirdness somewhere in Excon + response = Excon.post( + "#{Api::ENDPOINT}/apis/launcher/1/get-package", + tcp_nodelay: true, + headers: headers, + body: "data=#{JSON.dump({ category: package.category, subcategory: package.subcategory, name: package.name, version: package.version })}", + chunk_size: 4_000_000, + response_block: streamer + ) + + response.status == 200 || response.status == 206 ensure file&.close end diff --git a/lib/pages/games.rb b/lib/pages/games.rb index 73d781f..9892939 100644 --- a/lib/pages/games.rb +++ b/lib/pages/games.rb @@ -89,6 +89,7 @@ class W3DHub Hash.new.tap { |hash| hash[I18n.t(:"games.game_settings")] = { icon: "gear", block: proc { Store.application_manager.settings(game.id, channel.id) } } hash[I18n.t(:"games.wine_configuration")] = { icon: "gear", block: proc { Store.application_manager.wine_configuration(game.id, channel.id) } } if W3DHub.unix? + hash[I18n.t(:"games.game_modifications")] = { icon: "gear", enabled: false, block: proc { puts "Coming Soon!" } } if game.id != "ren" hash[I18n.t(:"games.repair_installation")] = { icon: "wrench", block: proc { Store.application_manager.repair(game.id, channel.id) } } hash[I18n.t(:"games.uninstall_game")] = { icon: "trashCan", block: proc { Store.application_manager.uninstall(game.id, channel.id) } } @@ -100,7 +101,7 @@ class W3DHub flow(width: 1.0, height: 22, margin_bottom: 8) do image "#{GAME_ROOT_PATH}/media/ui_icons/#{hash[:icon]}.png", width: 0.11 if hash[:icon] image EMPTY_IMAGE, width: 0.11 unless hash[:icon] - link key, text_size: 18 do + link key, text_size: 18, enabled: hash.key?(:enabled) ? hash[:enabled] : true do hash[:block]&.call end end @@ -142,15 +143,18 @@ class W3DHub Store.application_manager.run(game.id, channel.id) end else + installing = Store.application_manager.task?(:installer, game.id, channel.id) + unless game.id == "ren" - button "#{I18n.t(:"interface.install")}", margin_left: 24, enabled: !Store.application_manager.task?(:installer, game.id, channel.id) do |button| + button "#{I18n.t(:"interface.install")}", margin_left: 24, enabled: !installing do |button| button.enabled = false + @import_button.enabled = false Store.application_manager.install(game.id, channel.id) end end - button "#{I18n.t(:"interface.import")}", margin_left: 24, enabled: false do - Store.application_manager.import(game.id, channel.id, "?") + @import_button = button "#{I18n.t(:"interface.import")}", margin_left: 24, enabled: !installing do + Store.application_manager.import(game.id, channel.id) end end end diff --git a/lib/states/boot.rb b/lib/states/boot.rb index b8158dd..c477045 100644 --- a/lib/states/boot.rb +++ b/lib/states/boot.rb @@ -2,6 +2,8 @@ class W3DHub class States class Boot < CyberarmEngine::GuiState def setup + window.show_cursor = true + theme(W3DHub::THEME) background 0xff_252525 diff --git a/lib/states/interface.rb b/lib/states/interface.rb index b20d8e8..f7d1d88 100644 --- a/lib/states/interface.rb +++ b/lib/states/interface.rb @@ -37,7 +37,7 @@ class W3DHub stack(width: 0.75, height: 1.0) do title "#{I18n.t(:"app_name")}", height: 0.5 flow(width: 1.0, height: 0.5) do - @application_taskbar_container = stack(width: 1.0, height: 1.0, margin_left: 16) do + @application_taskbar_container = stack(width: 1.0, height: 1.0, margin_left: 16, margin_right: 16) do flow(width: 1.0, height: 0.65) do @application_taskbar_label = inscription "", width: 0.60, text_wrap: :none @application_taskbar_status_label = inscription "", width: 0.40, text_align: :right, text_wrap: :none @@ -168,7 +168,7 @@ class W3DHub show_application_taskbar @application_taskbar_label.value = task.status.label - @application_taskbar_status_label.value = task.status.value + @application_taskbar_status_label.value = "#{task.status.value} (#{format("%.2f%%", task.status.progress.clamp(0.0, 1.0) * 100.0)})" @application_taskbar_progressbar.value = task.status.progress.clamp(0.0, 1.0) return unless @page.is_a?(Pages::DownloadManager) diff --git a/locales/en.yml b/locales/en.yml index 546a188..ae5d31a 100644 --- a/locales/en.yml +++ b/locales/en.yml @@ -25,6 +25,7 @@ en: games: game_settings: Game Settings wine_configuration: Wine Configuration + game_modifications: Game Modifications repair_installation: Repair Installation uninstall_game: Uninstall Game install_folder: Install Folder