Write paths.ini, fix a few issues

lib/worker/api/application.rb

@@ -68,7 +68,7 @@ module W3DHubLauncher
           def extended_data(key, default)
             v = @extended_data&.find { |d| d.name == key }&.value
             if v.nil?
-              v = @_app.extended_data(key, value)
+              v = @_app.extended_data(key, default)
 
               return default if v.nil?
             end

lib/worker/api/legacy_manifest_package.rb

@@ -31,32 +31,32 @@ module W3DHubLauncher
 
         # checksum whole file and file chunks until a mismatch occurs or the whole file is verified.
         def verify_file(filename)
+          return false unless File.exist?(filename) && !File.directory?(filename)
+
           file_size = File.size(filename)
 
-          checksum_chunks = {}
-          overall_checksum = ""
           overall_digest = Digest::SHA256.new
           chunk_digest = Digest::SHA256.new
 
           File.open(filename, "rb") do |f|
             f.pos = 0
             offset = 0
+            last_valid_offset = 0
 
             while (chunk = f.read(@checksum_chunk_size))
-
               overall_digest << chunk
 
-              checksum_chunks[offset] = chunk_digest.update(chunk).hexdigest.upcase
+              # return last valid chunk on invalid chunk
+              return last_valid_offset unless @checksum_chunks[offset.to_s] == chunk_digest.update(chunk).hexdigest.upcase
 
+              last_valid_offset = offset
               offset += @checksum_chunk_size
               chunk_digest.reset
             end
           end
 
-          overall_checksum = overall_digest.hexdigest.upcase
+          # return boolean after completely digesting file
-
+          @sha256_checksum == overall_digest.hexdigest.upcase
-          # FIXME: Make this a nice Data struct object
-          [overall_checksum, checksum_chunks, file_size]
         end
       end
     end

lib/worker/task.rb

@@ -1,5 +1,8 @@
 module W3DHubLauncher
   class Task
+    # ignore *.meta and paths.ini files
+    IGNORED_FILES = [/\A.+\.meta\z/i, /\A.+\/paths\.ini\z/i].freeze
+
     IndexedFile = Data.define(:type, :path)
 
     attr_reader :request_id, :application, :channel, :installed_version, :target_version
@@ -52,9 +55,7 @@ module W3DHubLauncher
       return unless File.directory?(@installation_directory)
 
       Dir.glob("#{@installation_directory}/**/**").each do |path|
-        path.gsub!("\\", "/")
+        add_to_file_index(path)
-
-        @file_index[path.downcase] = IndexedFile.new(File.directory?(path) ? :directory : :file, path)
       end
     end
 
@@ -94,6 +95,9 @@ module W3DHubLauncher
           # remove files that are total replacements, not patches
           files.delete_if { |f| f.name.casecmp?(file.name) } unless file.patch?
 
+          # exclude ignored files from consideration
+          files.delete_if { |f| ignored_file?(f.name) }
+
           # add file to file list
           files << file
         end
@@ -151,6 +155,8 @@ module W3DHubLauncher
         end
       end.flatten
 
+      return if packages.empty?
+
       result = @worker.w3dhub_api.fetch_package_details(packages)
 
       unless result.okay?
@@ -163,39 +169,110 @@ module W3DHubLauncher
         abort_task!("Failed to retrieve packages details for: #{failed_packages.map { |pkg| "#{pkg.name}:#{pkg.version}: #{pkg.error}"}.join(', ') }")
       end
 
-      manifest_packages.each do |pkg|
+      @packages = manifest_packages
-        # FIXME: verify local packages to prevent overdownloading!
-        # pkg.verify_file(normalize_path(pkg.name))
 
+      manifest_packages.each do |pkg|
         file_path = package_cache_path(pkg)
         unless File.directory?(File.dirname(file_path))
           puts "creating directory: #{File.dirname(file_path)}"
-          FileUtils.mkdir_p(File.dirname(file_path))
+          create_directory(File.dirname(file_path))
+        end
+
+        partially_valid_at = 0
+        state = pkg.verify_file(file_path)
+        if state.is_a?(Integer)
+          puts "partially valid at: #{state} bytes (#{file_path})" # 20971520
+          partially_valid_at = state
+        else
+          if state == true # completely verified, skip download!
+            puts "skipping #{file_path}"
+            next
+          end
         end
 
         result = if pkg.download_url
           puts "downloading #{pkg.download_url} to #{file_path}"
-          @worker.w3dhub_api.download(pkg.download_url, path: file_path)
+          @worker.w3dhub_api.download(pkg.download_url, path: file_path, headers: @worker.w3dhub_api.headers(range: partially_valid_at))
         else
           # TODO xD
           @worker.w3dhub_api.fetch_package("TODO")
         end
 
-        abort_task!("Failed to download required package: #{pkg.name}:#{pkg.version}") unless result.okay?
+        abort_task!("Failed to download required package: #{pkg.name}:#{pkg.version} (#{result.error})") unless result.okay?
       end
     end
 
     def install_packages
+      create_directory(@installation_directory)
+
+      processed_packages = {}
+      @required_manifest_files.each do |manifest_file|
+        package = @packages.find do |pkg|
+          pkg.name.casecmp?("#{manifest_file.package}.zip") && manifest_file.version == pkg.version
+        end
+        package_path = package_cache_path(package)
+
+        next if processed_packages[package_path]
+
+        if manifest_file.patch?
+        else
+          stream = Zip::InputStream.new(File.open(package_path))
+
+          while(entry = stream.get_next_entry)
+            file_path = normalize_path(entry.name)
+
+            next if ignored_file?(file_path)
+
+            pp file_path
+
+            create_directory(File.dirname(file_path))
+
+            File.open(file_path, "wb") do |f|
+              entry_stream = entry.get_input_stream
+
+              while(chunk = entry_stream.read(4_194_304))
+                f.write(chunk)
+              end
+            end
+
+            add_to_file_index(file_path)
+          end
+        end
+
+        processed_packages[package_path] = manifest_file
+      end
     end
 
     def remove_deleted_files
+      @deleted_manifest_files.each do |manifest_file|
+        file_path = normalize_path(manifest_file.name)
+
+        if File.exist?(file_path) && !File.directory?(file_path)
+          puts "Removing file: #{file_path}"
+          File.delete(file_path)
+
+          remove_from_file_index(file_path)
+        end
+      end
     end
 
     def write_paths_ini
+      File.open(normalize_path("data/paths.ini"), "w") do |file|
+        file.puts("[paths]")
+        file.puts("RegBase=W3D Hub")
+        file.puts("RegClient=#{@application.category}\\#{@application.id}-#{@channel.id}")
+        file.puts("RegFDS=#{@application.category}\\#{@application.id}-#{@channel.id}-server")
+        file.puts("FileBase=W3D Hub");
+        file.puts("FileClient=#{@application.category}\\#{@application.id}-#{@channel.id}")
+        file.puts("FileFDS=#{@application.category}\\#{@application.id}-#{@channel.id}-server")
+
+        file.puts("UseRenFolder=#{@channel.extended_data("usesRenFolder", false)}")
+      end
     end
 
     # updated, and moved applications will overwrite existing application data in settings
     def mark_application_installed
+      puts "APPLICATION: #{@application.name} #{@application.id}:#{@channel.id}:#{@target_version} installed."
     end
 
     def mark_application_uninstalled
@@ -216,12 +293,58 @@ module W3DHubLauncher
       directory = @worker.settings.preferences.launcher_package_cache_directory
 
       if package.version?
-        format("%s/%s/%s", directory, package.version.to_s, package.name)
+        format("%s/%s/%s/%s", directory, @application.id, package.version.to_s, package.name)
       else
-        format("%s/%s", directory, package.name)
+        format("%s/%s/%s", directory, @application.id, package.name)
       end
     end
 
+    def ignored_file?(path)
+      IGNORED_FILES.any? { |regex| path.match?(regex) }
+    end
+
+    def create_directory(path)
+      path.gsub!("\\", "/")
+
+      # directory doesn't exists, create it!
+      unless File.directory?(path)
+        FileUtils.mkdir_p(path)
+
+        add_directory_to_file_index(path)
+      end
+    end
+
+    # add EXISTING file or directory to index
+    def add_to_file_index(path)
+      path.gsub!("\\", "/")
+
+      @file_index[path.downcase] = IndexedFile.new(File.directory?(path) ? :directory : :file, path)
+    end
+
+    # add EXISTING directory to file index, walking up the directory tree until existing directory entries are found
+    # NOTE: this will **NOT** "scan" the directory for new directories or files
+    def add_directory_to_file_index(path)
+      path.gsub!("\\", "/")
+
+      segments = path.split("/")
+      sub_path = path
+      until (file_index = @file_index[sub_path.downcase])
+        @file_index[sub_path.downcase]
+
+        segments.pop
+
+        break if segments.empty?
+
+        sub_path = segments.join("/")
+      end
+    end
+
+    def remove_from_file_index(path)
+      path.gsub!("\\", "/")
+
+      @file_index.delete(path.downcase)
+    end
+
     # And behold, the backslashes were slain and the path deemed sane!
     def normalize_path(base_path)
       base_path = base_path.gsub("\\", "/")
@@ -276,24 +399,29 @@ module W3DHubLauncher
       return result unless result.okay?
 
       response = JSON.parse(result.data)
-      package_details = response["packages"]&.map { |item| Worker::Api::LegacyManifestPackage.new(item) }&.first
+      manifest_package = response["packages"]&.map { |item| Worker::Api::LegacyManifestPackage.new(item) }&.first
 
-      if package_details.nil? || package_details.error?
+      if manifest_package.nil? || manifest_package.error?
-        pp package_details
+        pp manifest_package
         abort_task!("Failed to fetch package details for manifest #{version}")
       end
 
       # FIXME: check if locally downloaded manifest is still valid, if present.
-      result = if package_details.download_url
+      file_path = package_cache_path(manifest_package)
-        @worker.w3dhub_api.download(package_details.download_url, path: "manifest-#{version}.xml")
+
+      create_directory(File.dirname(file_path))
+
+      result = if manifest_package.download_url
+        @worker.w3dhub_api.download(manifest_package.download_url, path: file_path)
       else
         # TODO xD
         @worker.w3dhub_api.fetch_package("TODO")
       end
 
       if result.okay?
-        return Worker::Api::LegacyManifest.new("manifest-#{version}.xml")
+        return Worker::Api::LegacyManifest.new(package_cache_path(manifest_package))
       else
+        pp result
         abort_task!("Failed to fetch manifest #{version}")
       end
     end

lib/worker/tasks/install_application.rb

@@ -28,7 +28,7 @@ module W3DHubLauncher
         # unpack packages and patch files to reach target version
         install_packages
 
-        # delete files that are no longer part of the application (and whose presence my break the application)
+        # delete files that are no longer part of the application (and whose presence may break the application)
         remove_deleted_files
 
         # tell the application to behive

lib/worker/w3dhub_api.rb

@@ -16,7 +16,7 @@ module W3DHubLauncher
       @http_clients = {}
     end
 
-    def headers(form_encoded: false)
+    def headers(form_encoded: false, range: nil)
       array = [
         ["user-agent", W3DHubLauncher::USER_AGENT],
         ["accept", "application/json"],
@@ -24,6 +24,7 @@ module W3DHubLauncher
 
       array << ["content-type", "application/x-www-form-urlencoded"] if form_encoded
       array << ["authorization", "Bearer #{@access_token}"] if @access_token
+      array << ["range", "bytes=#{range}-"] if range
 
       # pp array
 
@@ -66,7 +67,10 @@ module W3DHubLauncher
             content_length = response.headers["content-length"] || 0
 
             total_downloaded_bytes = 0
-            File.open(path, "wb") do |file|
+            range = Integer(headers.find { |key, value| key == "range" }&.last&.split("=")&.last&.split("-")&.first || 0)
+            File.open(path, range.positive? ? "r+b" : "wb") do |file|
+              file.pos = range if range.positive?
+
               response.each do |chunk|
                 file.write(chunk)
                 downloaded_bytes = chunk.length

w3d_hub_linux_launcher.rb

@@ -11,6 +11,7 @@ require "logger"
 require "digest"
 require "pathname"
 require "fileutils"
+require "tempfile"
 
 require "async"
 require "async/http/internet/instance"