Brought back Excon for package downloading as the method recommended by async-http is unreliable, added support for importing games, repairer and updater tasks are both now simple subclasses of installer, implemented verify_files for checking installed files to prune download package list (currently causes Api.package_details to fail..., so disabled for now), misc. changes.

lib/api.rb

@@ -2,7 +2,8 @@ class W3DHub
   class Api
     USER_AGENT = "Cyberarm's Linux Friendly W3D Hub Launcher v#{W3DHub::VERSION}".freeze
     DEFAULT_HEADERS = [
-      ["User-Agent", USER_AGENT]
+      ["User-Agent", USER_AGENT],
+      ["Accept", "application/json"]
     ].freeze
     FORM_ENCODED_HEADERS = (
       DEFAULT_HEADERS + [["Content-Type", "application/x-www-form-urlencoded"]]
@@ -126,15 +127,14 @@ class W3DHub
     # /apis/launcher/1/get-package-details
     # client requests package details: data={"packages":[{"category":"games","name":"apb.ico","subcategory":"apb","version":""}]}
     def self.package_details(internet, packages)
-      body = "data=#{JSON.dump({ packages: packages })}"
+      body = URI.encode_www_form("data": JSON.dump({ packages: packages }))
       response = internet.post("#{ENDPOINT}/apis/launcher/1/get-package-details", FORM_ENCODED_HEADERS, body)
 
       if response.success?
         hash = JSON.parse(response.read, symbolize_names: true)
-        packages = hash[:packages].map { |pkg| Package.new(pkg) }
+        hash[:packages].map { |pkg| Package.new(pkg) }
-        return packages.first if packages.size == 1
-        return packages
       else
+        pp response, body
         false
       end
     end

lib/application_manager.rb

@@ -36,14 +36,14 @@ class W3DHub
       @tasks.push(updater)
     end
 
-    def import(app_id, channel, path)
+    def import(app_id, channel)
-      puts "Import Request: #{app_id}-#{channel} -> #{path}"
+      puts "Import Request: #{app_id}-#{channel}"
 
       # Check registry for auto-import if windows
       # if auto-import fails ask user for path to game exe
       # mark app as imported/installed
 
-      @tasks.push(Importer.new(app_id, channel, path))
+      @tasks.push(Importer.new(app_id, channel))
     end
 
     def settings(app_id, channel)
@@ -238,6 +238,19 @@ class W3DHub
       end
     end
 
+    def imported!(task, exe_path)
+      application_data = {
+        install_directory: File.basename(exe_path),
+        installed_version: task.channel.current_version,
+        install_path: exe_path,
+        wine_prefix: task.wine_prefix
+      }
+
+      Store.settings[:games] ||= {}
+      Store.settings[:games][:"#{task.app_id}_#{task.release_channel}"] = application_data
+      Store.settings.save_settings
+    end
+
     def installed!(task)
       # install_dir
       # installed_version

lib/application_manager/pool.rb

@@ -11,7 +11,7 @@ class W3DHub
       end
 
       def manage_pool
-        while (@jobs.size.positive? || @workers.any?(&:busy?))
+        while @jobs.size.positive? || @workers.any?(&:busy?)
           feed_pool unless @jobs.size.zero?
 
           sleep 0.1
@@ -29,9 +29,9 @@ class W3DHub
           @die = false
           @job = nil
 
-          Async do
+          Thread.new do
             until (@die)
-              @job.process if @job && @job.waiting?
+              @job.process if @job&.waiting?
               @job = nil
               sleep 0.1
             end

lib/application_manager/task.rb

@@ -43,13 +43,19 @@ class W3DHub
 
       # Start task, inside its own thread
       # FIXME: Ruby 3 has parallelism now: Use a Ractor to do work on a seperate core to
-      #        prevent the UI for locking up while doing computation heavy work, i.e building
+      #        prevent the UI from locking up while doing computation heavy work, i.e building
       #        list of packages to download
       def start
         @task_state = :running
 
-        Async do
+        Thread.new do
+          Sync do
+            begin
               status = execute_task
+            rescue RuntimeError => e
+              status = false
+              @task_failure_reason = e.message[0..512]
+            end
 
             # Force free some bytes
             GC.compact if GC.respond_to?(:compact)
@@ -59,7 +65,8 @@ class W3DHub
             @task_state = :complete unless @task_state == :failed
 
             hide_application_taskbar if @task_state == :failed
-          send_message_dialog(:failure, "Task #{type.inspect} failed for #{@application.name}", @task_failure_reason) if @task_state == :failed
+            send_message_dialog(:failure, "Task #{type.inspect} failed for #{@application.name}", @task_failure_reason) if @task_state == :failed && !@fail_silently
+          end
         end
       end
 
@@ -100,6 +107,10 @@ class W3DHub
         @task_failure_reason = reason.to_s
       end
 
+      def fail_silently!
+        @fail_silently = true
+      end
+
       # Quick checks before network and computational work starts
       def fail_fast
         # tar present?
@@ -211,6 +222,89 @@ class W3DHub
         packages
       end
 
+      def verify_files(manifests, packages)
+        @status.operations.clear
+        @status.label = "Downloading #{@application.name}..."
+        @status.value = "Verifying installed files..."
+        @status.progress = 0.0
+
+        @status.step = :verify_files
+
+        path = Cache.install_path(@application, @channel)
+        accepted_files = {}
+        rejected_files = []
+
+        file_count = manifests.map { |m| m.files.count }.sum
+        processed_files = 0
+
+        manifests.each do |manifest|
+          manifest.files.each do |file|
+            safe_file_name = file.name.gsub("\\", "/")
+            # Fix borked data -> Data 'cause Windows don't care about capitalization
+            safe_file_name.sub!("data/", "Data/") unless File.exist?("#{path}/#{safe_file_name}")
+
+            file_path = "#{path}/#{safe_file_name}"
+
+            processed_files += 1
+            @status.progress = processed_files.to_f / file_count
+
+            next if file.removed_since
+            next if accepted_files.key?(safe_file_name)
+
+            unless File.exist?(file_path)
+              rejected_files << { file: file, manifest_version: manifest.version }
+              puts "[#{manifest.version}] File missing: #{file_path}"
+              next
+            end
+
+            digest = Digest::SHA256.new
+            f = File.open(file_path)
+
+            while (chunk = f.read(32_000_000))
+              digest.update(chunk)
+            end
+
+            f.close
+
+            pp file if file.checksum.nil?
+
+            if digest.hexdigest.upcase == file.checksum.upcase
+              accepted_files[safe_file_name] = manifest.version
+              # puts "[#{manifest.version}] Verified file: #{file_path}"
+            else
+              rejected_files << { file: file, manifest_version: manifest.version }
+              puts "[#{manifest.version}] File failed Verification: #{file_path}"
+            end
+          end
+        end
+
+        puts "#{rejected_files.count} missing or corrupt files"
+
+        # TODO: Filter packages to only the required ones
+        selected_packages = []
+        selected_packages_hash = {}
+
+        rejected_files.each do |hash|
+          next if selected_packages_hash["#{hash[:file].package}_#{hash[:manifest_version]}"]
+
+          package = packages.find { |pkg| pkg.name == hash[:file].package && pkg.version == hash[:manifest_version] }
+
+          if package
+            selected_packages_hash["#{hash[:file].package}_#{hash[:manifest_version]}"] = true
+            selected_packages << package
+          else
+            raise "missing package: #{hash[:file].package}:#{hash[:manifest_version]} in fetched packages list!"
+          end
+        end
+
+        # FIXME: Order `selected_packages` like `packages`
+
+        # Removed packages that don't need to be fetched or processed
+        packages.delete_if { |package| !selected_packages.find { |pkg| pkg == package } }
+
+        packages
+      end
+
       def fetch_packages(packages)
         hashes = packages.map do |pkg|
           {
@@ -306,8 +400,7 @@ class W3DHub
 
           pool.manage_pool
         else
-          puts "FAILED!"
+          fail!("Failed to fetch package details")
-          pp package_details
         end
       end
 
@@ -406,45 +499,6 @@ class W3DHub
         puts "#{@app_id} has been installed."
       end
 
-      def verify_files(manifests, packages)
-        path = Cache.install_path(@application, @channel)
-        accepted_files = {}
-        rejected_files = []
-
-        manifests.each do |manifest|
-          manifest.files.each do |file|
-            file_path = "#{path}/#{file.name.gsub('\\', '/')}"
-
-            unless File.exists?(file_path)
-              rejected_files << file
-              puts "[#{manifest.version}] File missing: #{file_path}"
-              next
-            end
-
-            next if accepted_files.key?(file.name)
-
-            digest = Digest::SHA256.new
-            f = File.open(file_path)
-
-            while (chunk = f.read(32_000_000))
-              digest.update(chunk)
-              current = Async::Task.current?
-              current&.yield
-            end
-
-            f.close
-
-            if digest.hexdigest.upcase == file.checksum.upcase
-              accepted_files[file.name] = manifest.version
-              puts "[#{manifest.version}] Verified file: #{file_path}"
-            else
-              rejected_files << file
-              puts "[#{manifest.version}] File failed Verification: #{file_path}"
-            end
-          end
-        end
-      end
-
       #############
       # Functions #
       #############
@@ -453,7 +507,7 @@ class W3DHub
         # Check for and integrity of local manifest
         internet = Async::HTTP::Internet.instance
 
-        package = Api.package_details(internet, [{ category: category, subcategory: subcategory, name: name, version: version }])
+        package = Api.package_details(internet, [{ category: category, subcategory: subcategory, name: name, version: version }]).first
 
         if File.exist?(Cache.package_path(category, subcategory, name, version))
           verified = verify_package(package)

lib/application_manager/tasks/importer.rb

@@ -1,10 +1,47 @@
 class W3DHub
   class ApplicationManager
     class Importer < Task
-      def initialize(app_id, channel, path = nil)
+      def type
-        super(app_id, channel)
+        :importer
+      end
 
-        @path = path
+      def execute_task
+        path = ask_file
+
+        unless File.exist?(path) && !File.directory?(path)
+          fail!("File #{path.inspect} does not exist or is a directory")
+          fail_silently! if path.nil? || path&.length&.zero? # User likely canceled the file selection
+        end
+
+        return false if failed?
+
+        Store.application_manager.imported!(self, path)
+
+        true
+      end
+
+      def ask_file(title: "Open File", filter: "*game*.exe")
+        if W3DHub.unix?
+          # search for command
+          cmds = %w{ zenity matedialog qarma kdialog }
+
+          command = cmds.find do |cmd|
+            cmd if system("which #{cmd}")
+          end
+
+          path = case File.basename(command)
+          when "zenity", "matedialog", "qarma"
+            `#{command} --file-selection --title "#{title}" --file-filter "#{filter}"`
+          when "kdialog"
+            `#{command} --title "#{title}" --getopenfilename . "#{filter}"`
+          else
+            raise "No known command found for system file selection dialog!"
+          end
+
+          path.strip
+        else
+          raise NotImplementedError
+        end
       end
     end
   end

lib/application_manager/tasks/installer.rb

@@ -15,6 +15,9 @@ class W3DHub
         packages = build_package_list(manifests)
         return false if failed?
 
+        # verify_files(manifests, packages)
+        # return false if failed?
+
         fetch_packages(packages)
         return false if failed?
 

lib/application_manager/tasks/repairer.rb

@@ -1,27 +1,27 @@
 class W3DHub
   class ApplicationManager
-    class Repairer < Task
+    class Repairer < Installer
       def type
         :repairer
       end
 
-      def execute_task
+      # def execute_task
-        fail_fast
+      #   fail_fast
-        return false if failed?
+      #   return false if failed?
 
-        manifests = fetch_manifests
+      #   manifests = fetch_manifests
-        return false if failed?
+      #   return false if failed?
 
-        packages = build_package_list(manifests)
+      #   packages = build_package_list(manifests)
-        return false if failed?
+      #   return false if failed?
 
-        verify_files(manifests, packages)
+      #   verify_files(manifests, packages)
-        return false if failed?
+      #   return false if failed?
 
-        # pp packages.select { |pkg| pkg.name == "misc" }
+      #   # pp packages.select { |pkg| pkg.name == "misc" }
 
-        true
+      #   true
-      end
+      # end
     end
   end
 end

lib/cache.rb

@@ -48,63 +48,38 @@ class W3DHub
     # Download a W3D Hub package
     def self.fetch_package(internet, package, block)
       path = package_path(package.category, package.subcategory, package.name, package.version)
+      headers = { "Content-Type": "application/x-www-form-urlencoded", "User-Agent": Api::USER_AGENT }
       start_from_bytes = package.custom_partially_valid_at_bytes
 
-      puts "    Start from bytes: #{start_from_bytes}"
+      puts "    Start from bytes: #{start_from_bytes} of #{package.size}"
 
       create_directories(path)
 
-      offset = start_from_bytes
+      file = File.open(path, start_from_bytes.positive? ? "r+b" : "wb")
-      parts = []
-      chunk_size = 4_000_000
-      workers = 4
 
-      file = File.open(path, offset.positive? ? "r+b" : "wb")
+      if start_from_bytes.positive?
-
+        headers["Range"] = "bytes=#{start_from_bytes}-"
-      amount_written = 0
+        file.pos = start_from_bytes
-
-      while (offset < package.size)
-        byte_range_start = offset
-        byte_range_end   = [offset + chunk_size, package.size].min
-        parts << (byte_range_start...byte_range_end)
-
-        offset += chunk_size
       end
 
-      semaphore = Async::Semaphore.new(workers)
+      streamer = lambda do |chunk, remaining_bytes, total_bytes|
-      barrier   = Async::Barrier.new(parent: semaphore)
+        file.write(chunk)
-
-      while !parts.empty?
-        barrier.async do
-          part = parts.shift
-
-          range_header = [["range", "bytes=#{part.min}-#{part.max}"]]
-
-          body = "data=#{JSON.dump({ category: package.category, subcategory: package.subcategory, name: package.name, version: package.version })}"
-          response = internet.post("#{Api::ENDPOINT}/apis/launcher/1/get-package", W3DHub::Api::FORM_ENCODED_HEADERS + range_header, body)
-
-          if response.success?
-            chunk = response.read
-            written = 0
-            if W3DHub.unix?
-              written = file.pwrite(chunk, part.min)
-            else
-              # probably not "thread safe"
-              file.pos = part.min
-              written = file.write(chunk)
-            end
-
-            amount_written += written
-            remaining_bytes = package.size - amount_written
-            total_bytes = package.size
 
         block.call(chunk, remaining_bytes, total_bytes)
         # puts "    Remaining: #{((remaining_bytes.to_f / total_bytes) * 100.0).round}% (#{W3DHub::format_size(total_bytes - remaining_bytes)} / #{W3DHub::format_size(total_bytes)})"
       end
-        end
 
-        barrier.wait
+      # Create a new connection due to some weirdness somewhere in Excon
-      end
+      response = Excon.post(
+        "#{Api::ENDPOINT}/apis/launcher/1/get-package",
+        tcp_nodelay: true,
+        headers: headers,
+        body: "data=#{JSON.dump({ category: package.category, subcategory: package.subcategory, name: package.name, version: package.version })}",
+        chunk_size: 4_000_000,
+        response_block: streamer
+      )
+
+      response.status == 200 || response.status == 206
     ensure
       file&.close
     end

lib/pages/games.rb

@@ -89,6 +89,7 @@ class W3DHub
                 Hash.new.tap { |hash|
                   hash[I18n.t(:"games.game_settings")] = { icon: "gear", block: proc { Store.application_manager.settings(game.id, channel.id) } }
                   hash[I18n.t(:"games.wine_configuration")] = { icon: "gear", block: proc { Store.application_manager.wine_configuration(game.id, channel.id) } } if W3DHub.unix?
+                  hash[I18n.t(:"games.game_modifications")] = { icon: "gear", enabled: false, block: proc { puts "Coming Soon!" } }
                   if game.id != "ren"
                     hash[I18n.t(:"games.repair_installation")] = { icon: "wrench", block: proc { Store.application_manager.repair(game.id, channel.id) } }
                     hash[I18n.t(:"games.uninstall_game")] = { icon: "trashCan", block: proc { Store.application_manager.uninstall(game.id, channel.id) } }
@@ -100,7 +101,7 @@ class W3DHub
                   flow(width: 1.0, height: 22, margin_bottom: 8) do
                     image "#{GAME_ROOT_PATH}/media/ui_icons/#{hash[:icon]}.png", width: 0.11 if hash[:icon]
                     image EMPTY_IMAGE, width: 0.11 unless hash[:icon]
-                    link key, text_size: 18 do
+                    link key, text_size: 18, enabled: hash.key?(:enabled) ? hash[:enabled] : true do
                       hash[:block]&.call
                     end
                   end
@@ -142,15 +143,18 @@ class W3DHub
                 Store.application_manager.run(game.id, channel.id)
               end
             else
+              installing = Store.application_manager.task?(:installer, game.id, channel.id)
+
               unless game.id == "ren"
-                button "<b>#{I18n.t(:"interface.install")}</b>", margin_left: 24, enabled: !Store.application_manager.task?(:installer, game.id, channel.id) do |button|
+                button "<b>#{I18n.t(:"interface.install")}</b>", margin_left: 24, enabled: !installing do |button|
                   button.enabled = false
+                  @import_button.enabled = false
                   Store.application_manager.install(game.id, channel.id)
                 end
               end
 
-              button "<b>#{I18n.t(:"interface.import")}</b>", margin_left: 24, enabled: false do
+              @import_button = button "<b>#{I18n.t(:"interface.import")}</b>", margin_left: 24, enabled: !installing do
-                Store.application_manager.import(game.id, channel.id, "?")
+                Store.application_manager.import(game.id, channel.id)
               end
             end
           end

lib/states/boot.rb

@@ -2,6 +2,8 @@ class W3DHub
   class States
     class Boot < CyberarmEngine::GuiState
       def setup
+        window.show_cursor = true
+
         theme(W3DHub::THEME)
 
         background 0xff_252525

lib/states/interface.rb

@@ -37,7 +37,7 @@ class W3DHub
                 stack(width: 0.75, height: 1.0) do
                   title "<b>#{I18n.t(:"app_name")}</b>", height: 0.5
                   flow(width: 1.0, height: 0.5) do
-                    @application_taskbar_container = stack(width: 1.0, height: 1.0, margin_left: 16) do
+                    @application_taskbar_container = stack(width: 1.0, height: 1.0, margin_left: 16, margin_right: 16) do
                       flow(width: 1.0, height: 0.65) do
                         @application_taskbar_label = inscription "", width: 0.60, text_wrap: :none
                         @application_taskbar_status_label = inscription "", width: 0.40, text_align: :right, text_wrap: :none
@@ -168,7 +168,7 @@ class W3DHub
         show_application_taskbar
 
         @application_taskbar_label.value = task.status.label
-        @application_taskbar_status_label.value = task.status.value
+        @application_taskbar_status_label.value = "#{task.status.value} (#{format("%.2f%%", task.status.progress.clamp(0.0, 1.0) * 100.0)})"
         @application_taskbar_progressbar.value = task.status.progress.clamp(0.0, 1.0)
 
         return unless @page.is_a?(Pages::DownloadManager)

locales/en.yml

@@ -25,6 +25,7 @@ en:
   games:
     game_settings: Game Settings
     wine_configuration: Wine Configuration
+    game_modifications: Game Modifications
     repair_installation: Repair Installation
     uninstall_game: Uninstall Game
     install_folder: Install Folder