diff --git a/Gemfile.lock b/Gemfile.lock index f433342..bb12c3a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -57,7 +57,7 @@ GEM io-stream (0.14.0) openssl (>= 3.3) ircparser (1.0.0) - json (3.0.2) + json (3.1.0) logger (1.7.0) openssl (4.0.2) protocol-hpack (1.5.1) @@ -140,7 +140,7 @@ CHECKSUMS io-event (1.22.1) sha256=0dd054cefe4a29f1b8286604f332c14f3208d75f54902961e91e94d45e5e1600 io-stream (0.14.0) sha256=07d9b13ea2066ccbbc761fff8d69ad5a31e47561d31e02cc271f3ed9c1a8f1cd ircparser (1.0.0) sha256=0d4598f691172e2ad078207c50ba62c0d3428ae841836b0a682e6cd01d62878d - json (3.0.2) sha256=8e6d7e7b11384c21230430cef90b71f14849a34a1f4452796670f7c981bd19df + json (3.1.0) sha256=3ce7ccf8fde58960177a3a756c31f292a4c1e1dedee884547096a5b0604e3bd8 logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203 openssl (4.0.2) sha256=1037ad2868ae58df9ad917891c0c0f9815a1172f6846d4bcdd508e4c2ee747c2 protocol-hpack (1.5.1) sha256=6feca238b8078da1cd295677d6f306c6001af92d75fe0643d33e6956cbc3ad91 diff --git a/lib/common.rb b/lib/common.rb index 0b2a35a..d9be16f 100644 --- a/lib/common.rb +++ b/lib/common.rb @@ -6,4 +6,21 @@ module W3DHubLauncher "/etc/ssl/ca-bundle.pem" # Some other distros ].find { |path| File.exist?(path) } end + + def self.format_size(bytes) + case bytes + when 0..1023 # Bytes + "#{bytes} B" + when 1024..1_048_575 # KiloBytes + "#{format_size_number(bytes / 1024.0)} KB" + when 1_048_576..1_073_741_999 # MegaBytes + "#{format_size_number(bytes / 1_048_576.0)} MB" + else # GigaBytes + "#{format_size_number(bytes / 1_073_742_000.0)} GB" + end + end + + def self.format_size_number(i) + format("%0.2f", i) + end end diff --git a/lib/worker/task.rb b/lib/worker/task.rb index 8812148..32876d5 100644 --- a/lib/worker/task.rb +++ b/lib/worker/task.rb @@ -3,6 +3,12 @@ module W3DHubLauncher # ignore *.meta and paths.ini files IGNORED_FILES = [/\A.+\.meta\z/i, /\A.+\/paths\.ini\z/i].freeze + STAGE_FETCH_MANIFEST = 1 + STAGE_VERIFY_FILES = 2 + STAGE_DOWNLOAD_FILES = 3 + STAGE_INSTALL_FILES = 4 + STAGE_MAX = STAGE_INSTALL_FILES + IndexedFile = Data.define(:type, :path) attr_reader :request_id, :application, :channel, :installed_version, :target_version, :status @@ -37,6 +43,10 @@ module W3DHubLauncher # check file modification time and size @force_verification = false + # Status progress reporting stage + @stage = 0 + @stage_count = STAGE_MAX # manifests and package list building, verifying, downloading, installing + setup end @@ -81,7 +91,7 @@ module W3DHubLauncher end def fetch_manifests(version = @target_version) - update_status(label: "Fetching manifests...", fraction: 0.0) + update_stage_progress(label: "Fetching manifests...", fraction: 0.0) while (manifest = fetch_manifest(version)) @manifests[version] = manifest @@ -93,7 +103,7 @@ module W3DHubLauncher end def build_package_list - update_status(label: "Building package list...", fraction: 0.0) + update_stage_progress(label: "Building package list...", fraction: 0.0) channel_manifests = [] version = @target_version @@ -108,7 +118,9 @@ module W3DHubLauncher files = [] deleted_files = [] - channel_manifests.reverse.each do |manifest| + channel_manifests.reverse.each_with_index do |manifest, i| + update_stage_progress(label: "Building package list...", fraction: (i.to_f + 1) / channel_manifests.size) + manifest.files.each do |file| if file.removed? files.delete_if { |f| f.name.casecmp?(file.name) } @@ -130,10 +142,13 @@ module W3DHubLauncher @manifest_files = files @deleted_manifest_files = deleted_files + + update_stage_progress(label: "Building package list...", fraction: 1.0) + @stage += 1 end def verify_files - update_status(label: "Verifying files...", fraction: 0.0) + update_stage_progress(label: "Verifying files...", fraction: 0.0) @required_manifest_files = @manifest_files.clone @@ -153,14 +168,22 @@ module W3DHubLauncher puts "verifying #{file_path}..." key = @verification_data.key(file.name) - pp [key, @force_verification] verification_data = @force_verification ? nil : @verification_data.unchanged?(key, checksum: file.checksum, file_path: file_path) - # Use cached checksum if the file appears unchanged, otherwise recalculate it - checksum = verification_data&.checksum || Digest::SHA256.file(file_path).hexdigest.upcase + # pp [key, @force_verification, verification_data, cached_checksums[file_path], file.checksum] + # pp @verification_data.data(key) - # Set or update cached verification data if files don't match - @verification_data.set(key, checksum: checksum, file_path: file_path) unless verification_data + checksum = verification_data&.checksum || cached_checksums[file_path] + + # File exists, verification data for an older version exists, skip generationing file checksum + if @force_verification || @verification_data.data(key).nil? + # Use cached checksum if the file appears unchanged, otherwise recalculate it + checksum = verification_data&.checksum || cached_checksums[file_path] || Digest::SHA256.file(file_path).hexdigest.upcase + + # Set or update cached verification data if files don't match + @verification_data.set(key, checksum: checksum, file_path: file_path) unless verification_data + cached_checksums[file_path] = checksum + end if checksum == file.checksum @required_manifest_files.delete(file) @@ -172,12 +195,16 @@ module W3DHubLauncher puts " failed verification:#{file.version}" end - update_status(label: "Verifying files...", fraction: i.to_f / @manifest_files.size) + update_stage_progress(label: "Verifying files (#{i + 1} / #{@manifest_files.size})", fraction: (i.to_f + 1) / @manifest_files.size) end + + update_stage_progress(label: "Verified files.", fraction: 1.0) + + @stage += 1 end def fetch_packages - update_status(label: "Downloading packages...", fraction: 0.0) + update_stage_progress(label: "Downloading packages...", fraction: 0.0) required_packages = {} @required_manifest_files.each do |file| @@ -215,7 +242,12 @@ module W3DHubLauncher @packages = manifest_packages - manifest_packages.each do |pkg| + # FIXME: We should include packages in verification step so once we reach the downloading stage + # we don't halt everything to verify package files and have a stable download size report + total_data_to_download_bytes = manifest_packages.sum { |pkg| pkg.file_size } + total_data_downloaded_bytes = 0 + + manifest_packages.each_with_index do |pkg, i| file_path = package_cache_path(pkg) unless File.directory?(File.dirname(file_path)) puts "creating directory: #{File.dirname(file_path)}" @@ -227,16 +259,34 @@ module W3DHubLauncher if state.is_a?(Integer) puts "partially valid at: #{state} bytes (#{file_path})" # 20971520 partially_valid_at = state + total_data_to_download_bytes -= partially_valid_at else if state == true # completely verified, skip download! puts "skipping #{file_path}" + total_data_to_download_bytes -= pkg.file_size + update_stage_progress(label: "Downloading packages...", fraction: (i.to_f + 1) / manifest_packages.size) next end end + last_sample_time_ms = -10_000 + sample_downloaded_bytes = 0 + result = if pkg.download_url puts "downloading #{pkg.download_url} to #{file_path}" - @worker.w3dhub_api.download(pkg.download_url, path: file_path, headers: @worker.w3dhub_api.headers(range: partially_valid_at)) + @worker.w3dhub_api.download(pkg.download_url, path: file_path, headers: @worker.w3dhub_api.headers(range: partially_valid_at)) do |downloaded_bytes, total_downloaded_bytes, content_length| + sample_downloaded_bytes += downloaded_bytes + total_data_downloaded_bytes += downloaded_bytes + + if Gosu.milliseconds - last_sample_time_ms >= 1_000 + last_sample_time_ms = Gosu.milliseconds + + fraction = (total_downloaded_bytes + partially_valid_at) / pkg.file_size.to_f + + update_stage_progress(label: "Downloading #{W3DHubLauncher.format_size(total_data_downloaded_bytes)} / #{W3DHubLauncher.format_size(total_data_to_download_bytes)} [#{W3DHubLauncher.format_size(sample_downloaded_bytes)}/s]", fraction: (i.to_f + fraction) / manifest_packages.size) + sample_downloaded_bytes = 0 + end + end else # TODO xD @worker.w3dhub_api.fetch_package("TODO") @@ -244,15 +294,19 @@ module W3DHubLauncher abort_task!("Failed to download required package: #{pkg.name}:#{pkg.version} (#{result.error})") unless result.okay? end + + update_stage_progress(label: "Downloaded packages.", fraction: 1.0) + + @stage += 1 end def install_packages - update_status(label: "Installing...", fraction: 0.0) + update_stage_progress(label: "Installing...", fraction: 0.0) create_directory(@installation_directory) processed_packages = {} - @required_manifest_files.each do |manifest_file| + @required_manifest_files.each_with_index do |manifest_file, i| package = @packages.find do |pkg| pkg.name.casecmp?("#{manifest_file.package}.zip") && manifest_file.version == pkg.version end @@ -267,11 +321,13 @@ module W3DHubLauncher end processed_packages[package_path] = manifest_file + + update_stage_progress(label: "Installing (#{i + 1} / #{@required_manifest_files.size})", fraction: (i.to_f + 1)/ @required_manifest_files.size) end end def remove_deleted_files - update_status(label: "Removing files...", fraction: 0.0) + update_stage_progress(label: "Removing files...", fraction: 1.0) @deleted_manifest_files.each do |manifest_file| file_path = normalize_path(manifest_file.name) @@ -334,6 +390,12 @@ module W3DHubLauncher message_requester(data: { status: { title: "#{@task_verb} #{@application.name} (#{@channel.name})", label: label, fraction: fraction } } ) end + def update_stage_progress(label:, fraction:) + fraction = (@stage / @stage_count.to_f) + ((1.0 / @stage_count) * fraction) + + message_requester(data: { status: { title: "#{@task_verb} #{@application.name} (#{@channel.name})", label: label, fraction: fraction } } ) + end + def application_target_installation_directory target_application = @worker.settings.applications.find { |app| app.id == @application.id && app.channel == @channel.id } if target_application diff --git a/lib/worker/w3dhub_api.rb b/lib/worker/w3dhub_api.rb index 9b6c32e..f6d2c5d 100644 --- a/lib/worker/w3dhub_api.rb +++ b/lib/worker/w3dhub_api.rb @@ -38,7 +38,6 @@ module W3DHubLauncher Sync do |task| task.with_timeout(API_TIMEOUT) do Async::HTTP::Internet.send(method, url, headers, body) do |response| - pp [method, url, headers, body, response] if response.success? result.data = response.read else