From 662e5c2b0c7592f129815d0eef65c89c3f4840a7 Mon Sep 17 00:00:00 2001 From: Cyberarm Date: Wed, 17 Nov 2021 22:34:15 -0600 Subject: [PATCH] Redid file checking to use the provided chunked checksums, added (broken) support for partial package downloads (resumed package fails verification), misc. supporting changes. --- lib/api.rb | 4 +- lib/api/package.rb | 15 ++++++- lib/application_manager.rb | 4 +- lib/application_manager/task.rb | 51 +++++++++++++++------- lib/application_manager/tasks/installer.rb | 2 +- lib/cache.rb | 18 +++++--- lib/common.rb | 2 +- lib/pages/download_manager.rb | 22 ++++++---- 8 files changed, 81 insertions(+), 37 deletions(-) diff --git a/lib/api.rb b/lib/api.rb index 182cfbd..3323a88 100644 --- a/lib/api.rb +++ b/lib/api.rb @@ -164,8 +164,8 @@ class W3DHub # client requests package: data={"category":"games","name":"ECW_Asteroids.zip","subcategory":"ecw","version":"1.0.0.0"} # # server responds with download bytes, probably supports chunked download and resume - def self.package(category, subcategory, name, version, &block) - Cache.fetch_package(W3DHUB_API_CONNECTION, category, subcategory, name, version, block) + def self.package(package, &block) + Cache.fetch_package(package, block) end #! === Server List API === !# diff --git a/lib/api/package.rb b/lib/api/package.rb index 8d7221d..2b0fafa 100644 --- a/lib/api/package.rb +++ b/lib/api/package.rb @@ -1,7 +1,8 @@ class W3DHub class Api class Package - attr_reader :category, :subcategory, :name, :version, :size, :checksum, :checksum_chunk_size, :checksum_chunks + attr_reader :category, :subcategory, :name, :version, :size, :checksum, :checksum_chunk_size, :checksum_chunks, + :custom_partially_valid_at_bytes def initialize(hash) @data = hash @@ -14,7 +15,17 @@ class W3DHub @size = @data[:size] @checksum = @data[:checksum] @checksum_chunk_size = @data[:"checksum-chunk-size"] - @checksum_chunks = @data[:"checksum-chunks"]&.map { |c| Chunk.new(c) } + @checksum_chunks = @data[:"checksum-chunks"] + + @custom_partially_valid_at_bytes = 0 + end + + def chunk(key) + @checksum_chunks[:"#{key}"] + end + + def partially_valid_at_bytes=(i) + @custom_partially_valid_at_bytes = i end class Chunk diff --git a/lib/application_manager.rb b/lib/application_manager.rb index d88c4a7..1092c4a 100644 --- a/lib/application_manager.rb +++ b/lib/application_manager.rb @@ -22,7 +22,7 @@ class W3DHub installer = Installer.new(app_id, channel) @tasks.push(installer) - installer.start + # installer.start end def import(app_id, channel, path) @@ -95,7 +95,7 @@ class W3DHub end def current_task - @tasks.find { |t| t.state == :running } + @tasks.first#find { |t| t.state == :running } end end end diff --git a/lib/application_manager/task.rb b/lib/application_manager/task.rb index c918fa2..fd5f0b0 100644 --- a/lib/application_manager/task.rb +++ b/lib/application_manager/task.rb @@ -174,19 +174,21 @@ class W3DHub if package_details @packages_to_download = [] + update_application_taskbar("Downloading #{@application.name}...", "Verifying local packages...", 0.0) + package_details.each do |pkg| - unless verify_package(pkg, pkg.category, pkg.subcategory, pkg.name, pkg.version) + unless verify_package(pkg) @packages_to_download << pkg end end - @total_bytes_to_download = @packages_to_download.sum { |pkg| pkg.size } + @total_bytes_to_download = @packages_to_download.sum { |pkg| pkg.size - pkg.custom_partially_valid_at_bytes } @bytes_downloaded = 0 @packages_to_download.each do |pkg| package_bytes_downloaded = 0 - package_fetch(pkg.category, pkg.subcategory, pkg.name, pkg.version) do |chunk, remaining_bytes, total_bytes| + package_fetch(pkg) do |chunk, remaining_bytes, total_bytes| @bytes_downloaded += chunk.to_s.length package_bytes_downloaded += chunk.to_s.length @@ -235,21 +237,21 @@ class W3DHub # Check for and integrity of local manifest if File.exist?(Cache.package_path(category, subcategory, name, version)) package = Api.package_details([{ category: category, subcategory: subcategory, name: name, version: version }]) - verified = verify_package(package, category, subcategory, name, version) + verified = verify_package(package) # download manifest if not valid - package_fetch(category, subcategory, name, version) unless verified + package_fetch(package) unless verified true if verified else # download manifest if not cached - package_fetch(category, subcategory, name, version) + package_fetch(package) end end - def package_fetch(category, subcategory, name, version, &block) - puts "Downloading: #{category}:#{subcategory}:#{name}-#{version}" + def package_fetch(package, &block) + puts "Downloading: #{package.category}:#{package.subcategory}:#{package.name}-#{package.version}" - Api.package(category, subcategory, name, version) do |chunk, remaining_bytes, total_bytes| + Api.package(package) do |chunk, remaining_bytes, total_bytes| # Store progress somewhere # Kernel.puts "#{name}-#{version}: #{(remaining_bytes.to_f / total_bytes).round}%" @@ -257,22 +259,41 @@ class W3DHub end end - def verify_package(package, category, subcategory, name, version, &block) - puts "Verifying: #{category}:#{subcategory}:#{name}-#{version}" + def verify_package(package, &block) + puts "Verifying: #{package.category}:#{package.subcategory}:#{package.name}-#{package.version}" digest = Digest::SHA256.new - path = Cache.package_path(category, subcategory, name, version) + path = Cache.package_path(package.category, package.subcategory, package.name, package.version) return false unless File.exists?(path) file_size = File.size(path) puts " File size: #{file_size}" - chunk_size = 128_000_000 if file_size >= 32_000_000 - chunk_size ||= 32_000_000 + chunk_size = package.checksum_chunk_size File.open(path) do |f| - while (chunk = f.read(32_000_000)) + package.checksum_chunks.each do |chunk_start, checksum| + chunk_start = Integer(chunk_start.to_s) + + read_length = chunk_size + read_length = file_size - chunk_start if chunk_start + chunk_size > file_size + + break if file_size - chunk_start < 0 + + f.seek(chunk_start) + + chunk = f.read(read_length) digest.update(chunk) + + if Digest::SHA256.new.hexdigest(chunk).upcase == checksum.upcase + valid_at = chunk_start + read_length + puts " Passed chunk: #{chunk_start}" + # package.partially_valid_at_bytes = valid_at + package.partially_valid_at_bytes = chunk_start + else + puts " FAILED chunk: #{chunk_start}" + break + end end end diff --git a/lib/application_manager/tasks/installer.rb b/lib/application_manager/tasks/installer.rb index 1a013aa..1e1bfa8 100644 --- a/lib/application_manager/tasks/installer.rb +++ b/lib/application_manager/tasks/installer.rb @@ -6,7 +6,7 @@ class W3DHub manifests = fetch_manifests return false if failed? - # update_application_taskbar("Downloading #{@application.name}...", "Building package list...", 0.0) + update_application_taskbar("Downloading #{@application.name}...", "Building package list...", 0.0) packages = build_package_list(manifests) return false if failed? diff --git a/lib/cache.rb b/lib/cache.rb index cb7c9b7..5877b23 100644 --- a/lib/cache.rb +++ b/lib/cache.rb @@ -40,12 +40,20 @@ class W3DHub end # Download a W3D Hub package - def self.fetch_package(socket, category, subcategory, name, version, block) - path = package_path(category, subcategory, name, version) + def self.fetch_package(package, block) + path = package_path(package.category, package.subcategory, package.name, package.version) + headers = { "Content-Type": "application/x-www-form-urlencoded" } + start_from_bytes = package.custom_partially_valid_at_bytes + + puts " Start from bytes: #{start_from_bytes}" create_directories(path) file = File.open(path, "wb") + if (start_from_bytes > 0) + headers["Range"] = "bytes=#{start_from_bytes}-#{package.size}" + file.seek(start_from_bytes) + end streamer = lambda do |chunk, remaining_bytes, total_bytes| file.write(chunk) @@ -57,15 +65,15 @@ class W3DHub # Create a new connection due to some weirdness somewhere in Excon response = Excon.post( "#{Api::ENDPOINT}/apis/launcher/1/get-package", - headers: Api::DEFAULT_HEADERS.merge({ "Content-Type": "application/x-www-form-urlencoded" }), - body: "data=#{JSON.dump({ category: category, subcategory: subcategory, name: name, version: version })}", + headers: Api::DEFAULT_HEADERS.merge(headers), + body: "data=#{JSON.dump({ category: package.category, subcategory: package.subcategory, name: package.name, version: package.version })}", chunk_size: 4_000_000, response_block: streamer ) file.close - response.status == 200 + response.status == 200 || response.status == 206 end end end diff --git a/lib/common.rb b/lib/common.rb index 3674bd7..7e3bb93 100644 --- a/lib/common.rb +++ b/lib/common.rb @@ -13,6 +13,6 @@ class W3DHub end def self.format_size_number(i) - format("%0.1f", i) + format("%0.2f", i) end end diff --git a/lib/pages/download_manager.rb b/lib/pages/download_manager.rb index ded16bc..178a0ec 100644 --- a/lib/pages/download_manager.rb +++ b/lib/pages/download_manager.rb @@ -5,20 +5,27 @@ class W3DHub def setup @download_package_info ||= {} + @task = window.application_manager.current_task + + return unless @task body.clear do stack(width: 1.0, height: 1.0) do # TODO: Show correct application details here flow(width: 1.0, height: 0.1, padding: 8) do - background 0xff_252550 + background @task.application.color + flow(width: 0.70, height: 1.0) do - image "#{GAME_ROOT_PATH}/media/icons/apb.png", height: 1.0 + image "#{GAME_ROOT_PATH}/media/icons/#{@task.app_id}.png", height: 1.0 + stack(margin_left: 8) do - tagline "Red Alert: A Path Beyond" - inscription "Version: 0.3.23 (release)" + $bug_1 = tagline "#{@task.application.name}" + $bug_2 = inscription "Version: #{@task.channel.current_version} (#{@task.channel.id})" end end + puts "OKAY" + flow(width: 0.30, height: 1.0) do stack(width: 0.499, height: 1.0) do para "Download Speed", width: 1.0, text_align: :center @@ -35,11 +42,8 @@ class W3DHub # Operations @downloads_container = stack(width: 1.0, height: 0.9, padding: 8, scroll: true) do # TODO: Show actual list of downloads - task = window.application_manager.current_task - pp task - - task&.packages_to_download&.each_with_index do |pkg, i| + @task&.packages_to_download&.each_with_index do |pkg, i| stack(width: 1.0, height: 24, padding: 8) do background 0xff_333333 if i.odd? @@ -48,7 +52,7 @@ class W3DHub @download_package_info["#{pkg.checksum}_status"] = inscription "Pending...", width: 0.3, text_align: :right, text_wrap: :none, tag: "#{pkg.checksum}_status" end - @download_package_info["#{pkg.checksum}_progress"] = progress fraction: rand(0.25..0.8), height: 2, width: 1.0, tag: "#{pkg.checksum}_progress" + @download_package_info["#{pkg.checksum}_progress"] = progress fraction: 0.0, height: 2, width: 1.0, tag: "#{pkg.checksum}_progress" end end end