Manifest package verification now works and partial package downloads (resumed downloads)

lib/worker/api/legacy_manifest_package.rb

@@ -31,32 +31,32 @@ module W3DHubLauncher
 
         # checksum whole file and file chunks until a mismatch occurs or the whole file is verified.
         def verify_file(filename)
+          return false unless File.exist?(filename) && !File.directory?(filename)
+
           file_size = File.size(filename)
 
-          checksum_chunks = {}
-          overall_checksum = ""
           overall_digest = Digest::SHA256.new
           chunk_digest = Digest::SHA256.new
 
           File.open(filename, "rb") do |f|
             f.pos = 0
             offset = 0
+            last_valid_offset = 0
 
             while (chunk = f.read(@checksum_chunk_size))
-
               overall_digest << chunk
 
-              checksum_chunks[offset] = chunk_digest.update(chunk).hexdigest.upcase
+              # return last valid chunk on invalid chunk
+              return last_valid_offset unless @checksum_chunks[offset.to_s] == chunk_digest.update(chunk).hexdigest.upcase
 
+              last_valid_offset = offset
               offset += @checksum_chunk_size
               chunk_digest.reset
             end
           end
 
-          overall_checksum = overall_digest.hexdigest.upcase
+          # return boolean after completely digesting file
-
+          @sha256_checksum == overall_digest.hexdigest.upcase
-          # FIXME: Make this a nice Data struct object
-          [overall_checksum, checksum_chunks, file_size]
         end
       end
     end

lib/worker/task.rb

@@ -164,24 +164,33 @@ module W3DHubLauncher
       end
 
       manifest_packages.each do |pkg|
-        # FIXME: verify local packages to prevent overdownloading!
-        # pkg.verify_file(normalize_path(pkg.name))
-
         file_path = package_cache_path(pkg)
         unless File.directory?(File.dirname(file_path))
           puts "creating directory: #{File.dirname(file_path)}"
           FileUtils.mkdir_p(File.dirname(file_path))
         end
 
+        partially_valid_at = 0
+        state = pkg.verify_file(file_path)
+        if state.is_a?(Integer)
+          puts "partially valid at: #{state} bytes (#{file_path})" # 20971520
+          partially_valid_at = state
+        else
+          if state == true # completely verified, skip download!
+            puts "skipping #{file_path}"
+            next
+          end
+        end
+
         result = if pkg.download_url
           puts "downloading #{pkg.download_url} to #{file_path}"
-          @worker.w3dhub_api.download(pkg.download_url, path: file_path)
+          @worker.w3dhub_api.download(pkg.download_url, path: file_path, headers: @worker.w3dhub_api.headers(range: partially_valid_at))
         else
           # TODO xD
           @worker.w3dhub_api.fetch_package("TODO")
         end
 
-        abort_task!("Failed to download required package: #{pkg.name}:#{pkg.version}") unless result.okay?
+        abort_task!("Failed to download required package: #{pkg.name}:#{pkg.version} (#{result.error})") unless result.okay?
       end
     end
 

lib/worker/w3dhub_api.rb

@@ -16,7 +16,7 @@ module W3DHubLauncher
       @http_clients = {}
     end
 
-    def headers(form_encoded: false)
+    def headers(form_encoded: false, range: nil)
       array = [
         ["user-agent", W3DHubLauncher::USER_AGENT],
         ["accept", "application/json"],
@@ -24,6 +24,7 @@ module W3DHubLauncher
 
       array << ["content-type", "application/x-www-form-urlencoded"] if form_encoded
       array << ["authorization", "Bearer #{@access_token}"] if @access_token
+      array << ["range", "bytes=#{range}-"] if range
 
       # pp array
 
@@ -66,7 +67,10 @@ module W3DHubLauncher
             content_length = response.headers["content-length"] || 0
 
             total_downloaded_bytes = 0
-            File.open(path, "wb") do |file|
+            range = headers.find { |key, value| key == "range" }&.last&.split("=")&.last&.split("-")&.first
+            File.open(path, range ? "r+b" : "wb") do |file|
+              file.pos = Integer(range) if range
+
               response.each do |chunk|
                 file.write(chunk)
                 downloaded_bytes = chunk.length