Brought back Excon for package downloading as the method recommended by async-http is unreliable, added support for importing games, repairer and updater tasks are both now simple subclasses of installer, implemented verify_files for checking installed files to prune download package list (currently causes Api.package_details to fail..., so disabled for now), misc. changes.

lib/cache.rb

@@ -48,63 +48,38 @@ class W3DHub
     # Download a W3D Hub package
     def self.fetch_package(internet, package, block)
       path = package_path(package.category, package.subcategory, package.name, package.version)
+      headers = { "Content-Type": "application/x-www-form-urlencoded", "User-Agent": Api::USER_AGENT }
       start_from_bytes = package.custom_partially_valid_at_bytes
 
-      puts "    Start from bytes: #{start_from_bytes}"
+      puts "    Start from bytes: #{start_from_bytes} of #{package.size}"
 
       create_directories(path)
 
-      offset = start_from_bytes
-      parts = []
-      chunk_size = 4_000_000
-      workers = 4
+      file = File.open(path, start_from_bytes.positive? ? "r+b" : "wb")
 
-      file = File.open(path, offset.positive? ? "r+b" : "wb")
-
-      amount_written = 0
-
-      while (offset < package.size)
-        byte_range_start = offset
-        byte_range_end   = [offset + chunk_size, package.size].min
-        parts << (byte_range_start...byte_range_end)
-
-        offset += chunk_size
+      if start_from_bytes.positive?
+        headers["Range"] = "bytes=#{start_from_bytes}-"
+        file.pos = start_from_bytes
       end
 
-      semaphore = Async::Semaphore.new(workers)
-      barrier   = Async::Barrier.new(parent: semaphore)
+      streamer = lambda do |chunk, remaining_bytes, total_bytes|
+        file.write(chunk)
 
-      while !parts.empty?
-        barrier.async do
-          part = parts.shift
-
-          range_header = [["range", "bytes=#{part.min}-#{part.max}"]]
-
-          body = "data=#{JSON.dump({ category: package.category, subcategory: package.subcategory, name: package.name, version: package.version })}"
-          response = internet.post("#{Api::ENDPOINT}/apis/launcher/1/get-package", W3DHub::Api::FORM_ENCODED_HEADERS + range_header, body)
-
-          if response.success?
-            chunk = response.read
-            written = 0
-            if W3DHub.unix?
-              written = file.pwrite(chunk, part.min)
-            else
-              # probably not "thread safe"
-              file.pos = part.min
-              written = file.write(chunk)
-            end
-
-            amount_written += written
-            remaining_bytes = package.size - amount_written
-            total_bytes = package.size
-
-            block.call(chunk, remaining_bytes, total_bytes)
-            # puts "    Remaining: #{((remaining_bytes.to_f / total_bytes) * 100.0).round}% (#{W3DHub::format_size(total_bytes - remaining_bytes)} / #{W3DHub::format_size(total_bytes)})"
-          end
-        end
-
-        barrier.wait
+        block.call(chunk, remaining_bytes, total_bytes)
+        # puts "    Remaining: #{((remaining_bytes.to_f / total_bytes) * 100.0).round}% (#{W3DHub::format_size(total_bytes - remaining_bytes)} / #{W3DHub::format_size(total_bytes)})"
       end
+
+      # Create a new connection due to some weirdness somewhere in Excon
+      response = Excon.post(
+        "#{Api::ENDPOINT}/apis/launcher/1/get-package",
+        tcp_nodelay: true,
+        headers: headers,
+        body: "data=#{JSON.dump({ category: package.category, subcategory: package.subcategory, name: package.name, version: package.version })}",
+        chunk_size: 4_000_000,
+        response_block: streamer
+      )
+
+      response.status == 200 || response.status == 206
     ensure
       file&.close
     end