Import format_size and format_size_number from old launcher, improved Task progress reporting, use aggressively cached file verification for Task unless we're running a repair (speed up updating)

2026-11-03 16:44:05 +00:00 · 2026-10-17 09:24:30 -05:00
parent 13a782aabf
commit c0dedac4a4
4 changed files with 97 additions and 19 deletions
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -57,7 +57,7 @@ GEM
    io-stream (0.14.0)
      openssl (>= 3.3)
    ircparser (1.0.0)
-    json (3.0.2)
+    json (3.1.0)
    logger (1.7.0)
    openssl (4.0.2)
    protocol-hpack (1.5.1)
@@ -140,7 +140,7 @@ CHECKSUMS
  io-event (1.22.1) sha256=0dd054cefe4a29f1b8286604f332c14f3208d75f54902961e91e94d45e5e1600
  io-stream (0.14.0) sha256=07d9b13ea2066ccbbc761fff8d69ad5a31e47561d31e02cc271f3ed9c1a8f1cd
  ircparser (1.0.0) sha256=0d4598f691172e2ad078207c50ba62c0d3428ae841836b0a682e6cd01d62878d
-  json (3.0.2) sha256=8e6d7e7b11384c21230430cef90b71f14849a34a1f4452796670f7c981bd19df
+  json (3.1.0) sha256=3ce7ccf8fde58960177a3a756c31f292a4c1e1dedee884547096a5b0604e3bd8
  logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203
  openssl (4.0.2) sha256=1037ad2868ae58df9ad917891c0c0f9815a1172f6846d4bcdd508e4c2ee747c2
  protocol-hpack (1.5.1) sha256=6feca238b8078da1cd295677d6f306c6001af92d75fe0643d33e6956cbc3ad91
--- a/lib/common.rb
+++ b/lib/common.rb
@@ -6,4 +6,21 @@ module W3DHubLauncher
      "/etc/ssl/ca-bundle.pem"              # Some other distros
    ].find { |path| File.exist?(path) }
  end
+
+  def self.format_size(bytes)
+    case bytes
+    when 0..1023 # Bytes
+      "#{bytes} B"
+    when 1024..1_048_575 # KiloBytes
+      "#{format_size_number(bytes / 1024.0)} KB"
+    when 1_048_576..1_073_741_999 # MegaBytes
+      "#{format_size_number(bytes / 1_048_576.0)} MB"
+    else # GigaBytes
+      "#{format_size_number(bytes / 1_073_742_000.0)} GB"
+    end
+  end
+
+  def self.format_size_number(i)
+    format("%0.2f", i)
+  end
 end
--- a/lib/worker/task.rb
+++ b/lib/worker/task.rb
@@ -3,6 +3,12 @@ module W3DHubLauncher
    # ignore *.meta and paths.ini files
    IGNORED_FILES = [/\A.+\.meta\z/i, /\A.+\/paths\.ini\z/i].freeze

+    STAGE_FETCH_MANIFEST = 1
+    STAGE_VERIFY_FILES = 2
+    STAGE_DOWNLOAD_FILES = 3
+    STAGE_INSTALL_FILES = 4
+    STAGE_MAX = STAGE_INSTALL_FILES
+
    IndexedFile = Data.define(:type, :path)

    attr_reader :request_id, :application, :channel, :installed_version, :target_version, :status
@@ -37,6 +43,10 @@ module W3DHubLauncher
      # check file modification time and size
      @force_verification = false

+      # Status progress reporting stage
+      @stage = 0
+      @stage_count = STAGE_MAX # manifests and package list building, verifying, downloading, installing
+
      setup
    end

@@ -81,7 +91,7 @@ module W3DHubLauncher
    end

    def fetch_manifests(version = @target_version)
-      update_status(label: "Fetching manifests...", fraction: 0.0)
+      update_stage_progress(label: "Fetching manifests...", fraction: 0.0)

      while (manifest = fetch_manifest(version))
        @manifests[version] = manifest
@@ -93,7 +103,7 @@ module W3DHubLauncher
    end

    def build_package_list
-      update_status(label: "Building package list...", fraction: 0.0)
+      update_stage_progress(label: "Building package list...", fraction: 0.0)

      channel_manifests = []
      version = @target_version
@@ -108,7 +118,9 @@ module W3DHubLauncher

      files = []
      deleted_files = []
-      channel_manifests.reverse.each do |manifest|
+      channel_manifests.reverse.each_with_index do |manifest, i|
+        update_stage_progress(label: "Building package list...", fraction: (i.to_f + 1) / channel_manifests.size)
+
        manifest.files.each do |file|
          if file.removed?
            files.delete_if { |f| f.name.casecmp?(file.name) }
@@ -130,10 +142,13 @@ module W3DHubLauncher

      @manifest_files = files
      @deleted_manifest_files = deleted_files
+
+      update_stage_progress(label: "Building package list...", fraction: 1.0)
+      @stage += 1
    end

    def verify_files
-      update_status(label: "Verifying files...", fraction: 0.0)
+      update_stage_progress(label: "Verifying files...", fraction: 0.0)

      @required_manifest_files = @manifest_files.clone

@@ -153,14 +168,22 @@ module W3DHubLauncher

        puts "verifying #{file_path}..."
        key = @verification_data.key(file.name)
-        pp [key, @force_verification]
        verification_data = @force_verification ? nil : @verification_data.unchanged?(key, checksum: file.checksum, file_path: file_path)

-        # Use cached checksum if the file appears unchanged, otherwise recalculate it
-        checksum = verification_data&.checksum || Digest::SHA256.file(file_path).hexdigest.upcase
+        # pp [key, @force_verification, verification_data, cached_checksums[file_path], file.checksum]
+        # pp @verification_data.data(key)

-        # Set or update cached verification data if files don't match
-        @verification_data.set(key, checksum: checksum, file_path: file_path) unless verification_data
+        checksum = verification_data&.checksum || cached_checksums[file_path]
+
+        # File exists, verification data for an older version exists, skip generationing file checksum
+        if @force_verification || @verification_data.data(key).nil?
+          # Use cached checksum if the file appears unchanged, otherwise recalculate it
+          checksum = verification_data&.checksum || cached_checksums[file_path] || Digest::SHA256.file(file_path).hexdigest.upcase
+
+          # Set or update cached verification data if files don't match
+          @verification_data.set(key, checksum: checksum, file_path: file_path) unless verification_data
+          cached_checksums[file_path] = checksum
+        end

        if checksum == file.checksum
          @required_manifest_files.delete(file)
@@ -172,12 +195,16 @@ module W3DHubLauncher
          puts "  failed verification:#{file.version}"
        end

-        update_status(label: "Verifying files...", fraction: i.to_f / @manifest_files.size)
+        update_stage_progress(label: "Verifying files (#{i + 1} / #{@manifest_files.size})", fraction: (i.to_f + 1) / @manifest_files.size)
      end
+
+      update_stage_progress(label: "Verified files.", fraction: 1.0)
+
+      @stage += 1
    end

    def fetch_packages
-      update_status(label: "Downloading packages...", fraction: 0.0)
+      update_stage_progress(label: "Downloading packages...", fraction: 0.0)

      required_packages = {}
      @required_manifest_files.each do |file|
@@ -215,7 +242,12 @@ module W3DHubLauncher

      @packages = manifest_packages

-      manifest_packages.each do |pkg|
+      # FIXME: We should include packages in verification step so once we reach the downloading stage
+      #        we don't halt everything to verify package files and have a stable download size report
+      total_data_to_download_bytes = manifest_packages.sum { |pkg| pkg.file_size }
+      total_data_downloaded_bytes = 0
+
+      manifest_packages.each_with_index do |pkg, i|
        file_path = package_cache_path(pkg)
        unless File.directory?(File.dirname(file_path))
          puts "creating directory: #{File.dirname(file_path)}"
@@ -227,16 +259,34 @@ module W3DHubLauncher
        if state.is_a?(Integer)
          puts "partially valid at: #{state} bytes (#{file_path})" # 20971520
          partially_valid_at = state
+          total_data_to_download_bytes -= partially_valid_at
        else
          if state == true # completely verified, skip download!
            puts "skipping #{file_path}"
+            total_data_to_download_bytes -= pkg.file_size
+            update_stage_progress(label: "Downloading packages...", fraction: (i.to_f + 1) / manifest_packages.size)
            next
          end
        end

+        last_sample_time_ms = -10_000
+        sample_downloaded_bytes = 0
+
        result = if pkg.download_url
          puts "downloading #{pkg.download_url} to #{file_path}"
-          @worker.w3dhub_api.download(pkg.download_url, path: file_path, headers: @worker.w3dhub_api.headers(range: partially_valid_at))
+          @worker.w3dhub_api.download(pkg.download_url, path: file_path, headers: @worker.w3dhub_api.headers(range: partially_valid_at)) do |downloaded_bytes, total_downloaded_bytes, content_length|
+            sample_downloaded_bytes += downloaded_bytes
+            total_data_downloaded_bytes += downloaded_bytes
+
+            if Gosu.milliseconds - last_sample_time_ms >= 1_000
+              last_sample_time_ms = Gosu.milliseconds
+
+              fraction = (total_downloaded_bytes + partially_valid_at) / pkg.file_size.to_f
+
+              update_stage_progress(label: "Downloading #{W3DHubLauncher.format_size(total_data_downloaded_bytes)} / #{W3DHubLauncher.format_size(total_data_to_download_bytes)} [#{W3DHubLauncher.format_size(sample_downloaded_bytes)}/s]", fraction: (i.to_f + fraction) / manifest_packages.size)
+              sample_downloaded_bytes = 0
+            end
+          end
        else
          # TODO xD
          @worker.w3dhub_api.fetch_package("TODO")
@@ -244,15 +294,19 @@ module W3DHubLauncher

        abort_task!("Failed to download required package: #{pkg.name}:#{pkg.version} (#{result.error})") unless result.okay?
      end
+
+      update_stage_progress(label: "Downloaded packages.", fraction: 1.0)
+
+      @stage += 1
    end

    def install_packages
-      update_status(label: "Installing...", fraction: 0.0)
+      update_stage_progress(label: "Installing...", fraction: 0.0)

      create_directory(@installation_directory)

      processed_packages = {}
-      @required_manifest_files.each do |manifest_file|
+      @required_manifest_files.each_with_index do |manifest_file, i|
        package = @packages.find do |pkg|
          pkg.name.casecmp?("#{manifest_file.package}.zip") && manifest_file.version == pkg.version
        end
@@ -267,11 +321,13 @@ module W3DHubLauncher
        end

        processed_packages[package_path] = manifest_file
+
+        update_stage_progress(label: "Installing (#{i + 1} / #{@required_manifest_files.size})", fraction: (i.to_f + 1)/ @required_manifest_files.size)
      end
    end

    def remove_deleted_files
-      update_status(label: "Removing files...", fraction: 0.0)
+      update_stage_progress(label: "Removing files...", fraction: 1.0)

      @deleted_manifest_files.each do |manifest_file|
        file_path = normalize_path(manifest_file.name)
@@ -334,6 +390,12 @@ module W3DHubLauncher
      message_requester(data: { status: { title: "#{@task_verb} #{@application.name} (#{@channel.name})", label: label, fraction: fraction } } )
    end

+    def update_stage_progress(label:, fraction:)
+      fraction = (@stage / @stage_count.to_f) + ((1.0 / @stage_count) * fraction)
+
+      message_requester(data: { status: { title: "#{@task_verb} #{@application.name} (#{@channel.name})", label: label, fraction: fraction } } )
+    end
+
    def application_target_installation_directory
      target_application = @worker.settings.applications.find { |app| app.id == @application.id && app.channel == @channel.id }
      if target_application
--- a/lib/worker/w3dhub_api.rb
+++ b/lib/worker/w3dhub_api.rb
@@ -38,7 +38,6 @@ module W3DHubLauncher
      Sync do |task|
        task.with_timeout(API_TIMEOUT) do
          Async::HTTP::Internet.send(method, url, headers, body) do |response|
-            pp [method, url, headers, body, response]
            if response.success?
              result.data = response.read
            else