From 970711097247b8d80a742b9da302218c32928646 Mon Sep 17 00:00:00 2001 From: Cyberarm Date: Thu, 23 Dec 2021 10:56:28 -0600 Subject: [PATCH] Added parallel downloads setting (not in gui options yet), added a thread pool for downloading multiple packages at once- quite helpful for TSR :grin:, fixed download resuming (used wrong file mode...) --- lib/application_manager/pool.rb | 89 +++++++++++++++++++++++++++++++++ lib/application_manager/task.rb | 29 +++++++---- lib/cache.rb | 6 +-- lib/settings.rb | 1 + w3d_hub_linux_launcher.rb | 1 + 5 files changed, 112 insertions(+), 14 deletions(-) create mode 100644 lib/application_manager/pool.rb diff --git a/lib/application_manager/pool.rb b/lib/application_manager/pool.rb new file mode 100644 index 0000000..ab4c59f --- /dev/null +++ b/lib/application_manager/pool.rb @@ -0,0 +1,89 @@ +class W3DHub + class ApplicationManager + class Pool + def initialize(workers:) + @workers = workers.times.collect { Worker.new } + @jobs = [] + end + + def add_job(job) + @jobs << job + end + + def manage_pool + while (@jobs.size.positive? || @workers.any?(&:busy?)) + feed_pool unless @jobs.size.zero? + + sleep 0.1 + end + end + + def feed_pool + @workers.select(&:available?).each do |worker| + worker.feed(@jobs.shift) + end + end + + class Worker + def initialize + @die = false + @job = nil + + Thread.new do + until (@die) + @job.process if @job && @job.waiting? + @job = nil + sleep 0.1 + end + end + end + + def feed(job) + raise "Worker already processing a job!" if @job&.processing? + + @job = job + end + + def die! + @die = true + end + + def available? + @job.nil? + end + + def busy? + !available? + end + end + + class Job + def initialize(block) + @block = block + + @state = :waiting + end + + def waiting? + @state == :waiting + end + + def processing? + @state == :processing + end + + def complete? + @state == :complete + end + + def process + @state = :processing + + @block.call + + @state == :complete + end + end + end + end +end \ No newline at end of file diff --git a/lib/application_manager/task.rb b/lib/application_manager/task.rb index 85fc381..40ff63d 100644 --- a/lib/application_manager/task.rb +++ b/lib/application_manager/task.rb @@ -281,23 +281,29 @@ class W3DHub @total_bytes_to_download = @packages_to_download.sum { |pkg| pkg.size - pkg.custom_partially_valid_at_bytes } @bytes_downloaded = 0 + pool = Pool.new(workers: Store.settings[:parallel_downloads]) + @packages_to_download.each do |pkg| - package_bytes_downloaded = pkg.custom_partially_valid_at_bytes + pool.add_job Pool::Job.new( proc { + package_bytes_downloaded = pkg.custom_partially_valid_at_bytes - package_fetch(pkg) do |chunk, remaining_bytes, total_bytes| - @bytes_downloaded += chunk.to_s.length - package_bytes_downloaded += chunk.to_s.length + package_fetch(pkg) do |chunk, remaining_bytes, total_bytes| + @bytes_downloaded += chunk.to_s.length + package_bytes_downloaded += chunk.to_s.length - @status.value = "#{W3DHub.format_size(@bytes_downloaded)} / #{W3DHub.format_size(@total_bytes_to_download)}" - @status.progress = @bytes_downloaded.to_f / @total_bytes_to_download + @status.value = "#{W3DHub.format_size(@bytes_downloaded)} / #{W3DHub.format_size(@total_bytes_to_download)}" + @status.progress = @bytes_downloaded.to_f / @total_bytes_to_download - operation = @status.operations[:"#{pkg.checksum}"] - operation.value = "#{W3DHub.format_size(package_bytes_downloaded)} / #{W3DHub.format_size(pkg.size)}" - operation.progress = package_bytes_downloaded.to_f / total_bytes + operation = @status.operations[:"#{pkg.checksum}"] + operation.value = "#{W3DHub.format_size(package_bytes_downloaded)} / #{W3DHub.format_size(pkg.size)}" + operation.progress = package_bytes_downloaded.to_f / pkg.size # total_bytes - update_interface_task_status - end + update_interface_task_status + end + }) end + + pool.manage_pool else puts "FAILED!" pp package_details @@ -318,6 +324,7 @@ class W3DHub @status.progress = 0.0 packages.each do |pkg| + # FIXME: can't add a new key into hash during iteration (RuntimeError) @status.operations[:"#{pkg.checksum}"] = Status::Operation.new( label: pkg.name, value: "Pending...", diff --git a/lib/cache.rb b/lib/cache.rb index fbd0199..72473fb 100644 --- a/lib/cache.rb +++ b/lib/cache.rb @@ -55,9 +55,9 @@ class W3DHub create_directories(path) - file = File.open(path, "ab") - if (start_from_bytes > 0) - headers["Range"] = "bytes=#{start_from_bytes}-#{package.size}" + file = File.open(path, start_from_bytes.positive? ? "r+b" : "wb") + if (start_from_bytes.positive?) + headers["Range"] = "bytes=#{start_from_bytes}-" file.pos = start_from_bytes end diff --git a/lib/settings.rb b/lib/settings.rb index b1732db..dd8f8e9 100644 --- a/lib/settings.rb +++ b/lib/settings.rb @@ -5,6 +5,7 @@ class W3DHub language: Gosu.user_languages.first.split("_").first, app_install_dir: default_app_install_dir, package_cache_dir: default_package_cache_dir, + parallel_downloads: 4, wine_command: "wine", create_wine_prefixes: true, allow_diagnostic_reports: false, diff --git a/w3d_hub_linux_launcher.rb b/w3d_hub_linux_launcher.rb index 25f6426..59084f0 100644 --- a/w3d_hub_linux_launcher.rb +++ b/w3d_hub_linux_launcher.rb @@ -36,6 +36,7 @@ require_relative "lib/mixer" require_relative "lib/application_manager" require_relative "lib/application_manager/manifest" require_relative "lib/application_manager/status" +require_relative "lib/application_manager/pool" require_relative "lib/application_manager/task" require_relative "lib/application_manager/tasks/installer" require_relative "lib/application_manager/tasks/updater"