From 7adb23d71f2aff4732e61d5606bbe7f41746a4c7 Mon Sep 17 00:00:00 2001 From: Cyberarm Date: Mon, 14 Sep 2026 15:40:21 -0500 Subject: [PATCH] Manifest package verification now works and partial package downloads (resumed downloads) --- lib/worker/api/legacy_manifest_package.rb | 16 ++++++++-------- lib/worker/task.rb | 19 ++++++++++++++----- lib/worker/w3dhub_api.rb | 8 ++++++-- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/lib/worker/api/legacy_manifest_package.rb b/lib/worker/api/legacy_manifest_package.rb index 92fa47a..88bf519 100644 --- a/lib/worker/api/legacy_manifest_package.rb +++ b/lib/worker/api/legacy_manifest_package.rb @@ -31,32 +31,32 @@ module W3DHubLauncher # checksum whole file and file chunks until a mismatch occurs or the whole file is verified. def verify_file(filename) + return false unless File.exist?(filename) && !File.directory?(filename) + file_size = File.size(filename) - checksum_chunks = {} - overall_checksum = "" overall_digest = Digest::SHA256.new chunk_digest = Digest::SHA256.new File.open(filename, "rb") do |f| f.pos = 0 offset = 0 + last_valid_offset = 0 while (chunk = f.read(@checksum_chunk_size)) - overall_digest << chunk - checksum_chunks[offset] = chunk_digest.update(chunk).hexdigest.upcase + # return last valid chunk on invalid chunk + return last_valid_offset unless @checksum_chunks[offset.to_s] == chunk_digest.update(chunk).hexdigest.upcase + last_valid_offset = offset offset += @checksum_chunk_size chunk_digest.reset end end - overall_checksum = overall_digest.hexdigest.upcase - - # FIXME: Make this a nice Data struct object - [overall_checksum, checksum_chunks, file_size] + # return boolean after completely digesting file + @sha256_checksum == overall_digest.hexdigest.upcase end end end diff --git a/lib/worker/task.rb b/lib/worker/task.rb index 844d6a6..b6a6f5f 100644 --- a/lib/worker/task.rb +++ b/lib/worker/task.rb @@ -164,24 +164,33 @@ module W3DHubLauncher end manifest_packages.each do |pkg| - # FIXME: verify local packages to prevent overdownloading! - # pkg.verify_file(normalize_path(pkg.name)) - file_path = package_cache_path(pkg) unless File.directory?(File.dirname(file_path)) puts "creating directory: #{File.dirname(file_path)}" FileUtils.mkdir_p(File.dirname(file_path)) end + partially_valid_at = 0 + state = pkg.verify_file(file_path) + if state.is_a?(Integer) + puts "partially valid at: #{state} bytes (#{file_path})" # 20971520 + partially_valid_at = state + else + if state == true # completely verified, skip download! + puts "skipping #{file_path}" + next + end + end + result = if pkg.download_url puts "downloading #{pkg.download_url} to #{file_path}" - @worker.w3dhub_api.download(pkg.download_url, path: file_path) + @worker.w3dhub_api.download(pkg.download_url, path: file_path, headers: @worker.w3dhub_api.headers(range: partially_valid_at)) else # TODO xD @worker.w3dhub_api.fetch_package("TODO") end - abort_task!("Failed to download required package: #{pkg.name}:#{pkg.version}") unless result.okay? + abort_task!("Failed to download required package: #{pkg.name}:#{pkg.version} (#{result.error})") unless result.okay? end end diff --git a/lib/worker/w3dhub_api.rb b/lib/worker/w3dhub_api.rb index d280bdf..603e615 100644 --- a/lib/worker/w3dhub_api.rb +++ b/lib/worker/w3dhub_api.rb @@ -16,7 +16,7 @@ module W3DHubLauncher @http_clients = {} end - def headers(form_encoded: false) + def headers(form_encoded: false, range: nil) array = [ ["user-agent", W3DHubLauncher::USER_AGENT], ["accept", "application/json"], @@ -24,6 +24,7 @@ module W3DHubLauncher array << ["content-type", "application/x-www-form-urlencoded"] if form_encoded array << ["authorization", "Bearer #{@access_token}"] if @access_token + array << ["range", "bytes=#{range}-"] if range # pp array @@ -66,7 +67,10 @@ module W3DHubLauncher content_length = response.headers["content-length"] || 0 total_downloaded_bytes = 0 - File.open(path, "wb") do |file| + range = headers.find { |key, value| key == "range" }&.last&.split("=")&.last&.split("-")&.first + File.open(path, range ? "r+b" : "wb") do |file| + file.pos = Integer(range) if range + response.each do |chunk| file.write(chunk) downloaded_bytes = chunk.length