Fixed unable to download ECW due to spaces in the file names, added checksum caching for Task to make updating much faster for most games (IA with its mostly loose files doesn't gain anything and might even be slower... Consider not performing any verification for non-repair tasks?)

lib/pages/games.rb

@@ -47,7 +47,7 @@ module W3DHubLauncher
       def blur
         @parent.hide_battleview_panel
 
-        CyberarmEngine::EventBus.unsubscribe("application_changed")
+        CyberarmEngine::EventBus.unsubscribe("application_changed", self)
       end
 
       def populate_game_content_container

lib/worker/api/legacy_manifest_package.rb

@@ -19,6 +19,12 @@ module W3DHubLauncher
           @sha256_checksum = hash["checksum"]
           @checksum_chunks = hash["checksum-chunks"]
           @download_url = hash["download_url"]
+
+          # Fix crashing on spaces
+          if @download_url
+            uri_path = @download_url.split("/").last
+            @download_url = @download_url.sub(uri_path, URI.encode_uri_component(uri_path))
+          end
         end
 
         def error?

lib/worker/api/verification_data.rb

@@ -0,0 +1,107 @@
+module W3DHubLauncher
+  class Worker
+    class Api
+
+      # Cached verifcation data for application and pending package files
+      class VerificationData
+        SCHEMA = 0
+
+        def initialize(path)
+          @path = path
+
+          data = {}
+          if File.exist?(@path)
+            begin
+              data = JSON.parse(File.read(@path))
+            rescue JSON::ParserError
+              data = {}
+            end
+          end
+          data = {} unless data.is_a?(Hash)
+
+          @schema = data["schema"] || SCHEMA
+          @data = {}
+          data["data"]&.each do |key, value|
+            @data[key] = Data.new(value)
+          end
+        end
+
+        def key(file_name, file_version: "", package: false)
+          file_name = file_name.gsub("\\", "/").downcase
+
+          if package
+            "package:{file_name}:#{file_version}"
+          else
+            file_name
+          end
+        end
+
+        def data(key)
+          @data[key]
+        end
+
+        def unchanged?(key, checksum:, file_path:)
+          data = data(key)
+          return nil unless data
+
+          return data if data.checksum.upcase.strip == checksum.upcase.strip &&
+                         data.modification_time == File.mtime(file_path).to_i &&
+                         data.file_size == File.size(file_path)
+
+          nil
+        end
+
+        def set(key, checksum:, file_path:)
+          data = Data.new(
+            {
+              "checksum" => checksum.upcase.strip,
+              "modification_time" => File.mtime(file_path).to_i,
+              "file_size" => File.size(file_path)
+            }
+          )
+
+          @data[key] = data
+          save
+
+          data
+        end
+
+        def delete(key)
+          data = @data.delete(key)
+          save
+
+          data
+        end
+
+        def save
+          File.write(@path, JSON.pretty_generate(self))
+        end
+
+        def to_json(options = {})
+          {
+            schema: SCHEMA,
+            data: @data
+          }.to_json(options)
+        end
+
+        class Data
+          attr_reader :checksum, :modification_time, :file_size
+
+          def initialize(data)
+            @checksum = data["checksum"]
+            @modification_time = data["modification_time"]
+            @file_size = data["file_size"]
+          end
+
+          def to_json(options = {})
+            {
+              checksum: @checksum,
+              modification_time: @modification_time,
+              file_size: @file_size
+            }.to_json(options)
+          end
+        end
+      end
+    end
+  end
+end

lib/worker/task.rb

@@ -19,6 +19,8 @@ module W3DHubLauncher
 
       @status = Worker::Request::STATUS_PENDING
 
+      @verification_data = Worker::Api::VerificationData.new("#{CACHE_PATH}/verification_data-#{@application.id}-#{@channel.id}.json")
+
       # caching mechinism for file path normalization
       @file_index = {}
       # manifests!
@@ -32,6 +34,10 @@ module W3DHubLauncher
       # packages to verify and/or download
       @packages = []
 
+      # whether task should perform a per file checksum or just read it from the cache and
+      # check file modification time and size
+      @force_verification = false
+
       setup
     end
 
@@ -136,7 +142,7 @@ module W3DHubLauncher
 
       # Process manifest game files in NEWEST to OLDEST order so that we don't erroneously flag
       #   valid files as invalid due to an OLDER version of the file being checked FIRST.
-      @manifest_files.reverse.each do |file|
+      @manifest_files.reverse.each_with_index do |file, i|
         break unless File.directory?(@installation_directory)
 
         # skip irrelevant older versions of files
@@ -147,8 +153,15 @@ module W3DHubLauncher
         next unless File.exist?(file_path) # && !File.directory?(file_path)
 
         puts "verifying #{file_path}..."
-        checksum = cached_checksums[file_path] || Digest::SHA256.file(file_path).hexdigest.upcase
+        key = @verification_data.key(file.name)
-        cached_checksums[file_path] ||= checksum
+        pp [key, @force_verification]
+        verification_data = @force_verification ? nil : @verification_data.unchanged?(key, checksum: file.checksum, file_path: file_path)
+
+        # Use cached checksum if the file appears unchanged, otherwise recalculate it
+        checksum = verification_data&.checksum || Digest::SHA256.file(file_path).hexdigest.upcase
+
+        # Set or update cached verification data if files don't match
+        @verification_data.set(key, checksum: checksum, file_path: file_path) unless verification_data
 
         if checksum == file.checksum
           @required_manifest_files.delete(file)
@@ -159,6 +172,8 @@ module W3DHubLauncher
         else
           puts "  failed verification:#{file.version}"
         end
+
+        update_status(label: "Verifying files...", fraction: i.to_f / @manifest_files.size)
       end
     end
 
@@ -249,7 +264,7 @@ module W3DHubLauncher
         if manifest_file.patch?
           apply_patch(manifest_file, package)
         else
-          unzip(package_path)
+          unzip(manifest_file, package_path)
         end
 
         processed_packages[package_path] = manifest_file
@@ -265,6 +280,8 @@ module W3DHubLauncher
         if File.exist?(file_path) && !File.directory?(file_path)
           puts "Removing file: #{file_path}"
           File.delete(file_path)
+          key = @verification_data.key(manifest_file.name)
+          @verification_data.delete(key)
 
           remove_from_file_index(file_path)
         end
@@ -285,7 +302,7 @@ module W3DHubLauncher
       end
     end
 
-    # updated, and moved applications will overwrite existing application data in settings
+    # updated, repaired, and moved applications will overwrite existing application data in settings
     def mark_application_installed
       app = Worker::Api::Settings::Application.create(
         id: @application.id,
@@ -469,7 +486,7 @@ module W3DHubLauncher
         package_path = package_cache_path(package)
 
         puts "unpacking patch..."
-        unzip(package_path, f.path)
+        unzip(false, package_path, output_path: f.path)
 
         puts "reading patch data.."
         patch_mix = W3DHubLauncher::WWMix.new(path: f.path)
@@ -504,12 +521,16 @@ module W3DHubLauncher
           raise temp_mix.error_reason unless temp_mix.save
 
           puts "copying file..."
-          FileUtils.cp(temp.path, normalize_path(manifest_file.name))
+          file_path = normalize_path(manifest_file.name)
+          FileUtils.cp(temp.path, file_path)
+
+          key = @verification_data.key(manifest_file.name)
+          @verification_data.set(key, checksum: manifest_file.checksum, file_path: file_path)
         end
       end
     end
 
-    def unzip(zip_path, output_path = nil)
+    def unzip(manifest_file, zip_path, output_path: nil)
       stream = Zip::InputStream.new(File.open(zip_path))
 
       while(entry = stream.get_next_entry)
@@ -529,6 +550,23 @@ module W3DHubLauncher
           end
         end
 
+
+        # the manifest file we receive MIGHT NOT be for the current zip file entry
+        # so look it up to be safe
+        if manifest_file
+          manifest = @manifests[manifest_file.version]
+          if manifest
+            entry_manifest_file = manifest.files.find do |f|
+              f.name.gsub("\\", "/").downcase == entry.name.gsub("\\", "/").downcase
+            end
+
+            if entry_manifest_file
+              key = @verification_data.key(entry.name)
+              @verification_data.set(key, checksum: entry_manifest_file.checksum, file_path: file_path)
+            end
+          end
+        end
+
         add_to_file_index(file_path)
       end
     end

lib/worker/tasks/repair_application.rb

@@ -4,6 +4,9 @@ module W3DHubLauncher
       def setup
         super
 
+        # User requested a repair, make sure everything is as it should be!
+        @force_verification = true
+
         @task_verb = "Repairing".freeze
       end
     end

w3d_hub_linux_launcher.rb

@@ -60,6 +60,7 @@ require_relative "lib/worker/api/server_event"
 require_relative "lib/worker/api/test_event"
 require_relative "lib/worker/api/legacy_manifest"
 require_relative "lib/worker/api/legacy_manifest_package"
+require_relative "lib/worker/api/verification_data"
 require_relative "lib/worker/request"
 require_relative "lib/worker/w3dhub_api"
 require_relative "lib/worker/task"