diff --git a/README.md b/README.md index 3e2a4cb..bab04c5 100644 --- a/README.md +++ b/README.md @@ -11,4 +11,4 @@ Only requires OpenGL, Ruby, and a few gems. `ruby w3d_hub_linux_launcher.rb` ## Contributing -Contributors welcome, especially if anyone can lend a hand at reducing patching memory usage. +Contributors welcome. diff --git a/lib/application_manager/task.rb b/lib/application_manager/task.rb index 4f334c4..9727790 100644 --- a/lib/application_manager/task.rb +++ b/lib/application_manager/task.rb @@ -739,34 +739,51 @@ class W3DHub temp_file_path = normalize_path(manifest_file.name, temp_path) logger.info(LOG_TAG) { " Loading #{temp_file_path}.patch..." } - patch_mix = W3DHub::Mixer::Reader.new(file_path: "#{temp_file_path}.patch", ignore_crc_mismatches: false) - patch_info = JSON.parse(patch_mix.package.files.find { |f| f.name.casecmp?(".w3dhub.patch") || f.name.casecmp?(".bhppatch") }.data, symbolize_names: true) + patch_mix = W3DHub::WWMix.new(path: "#{temp_file_path}.patch") + unless patch_mix.load + raise patch_mix.error_reason + end + patch_entry = patch_mix.entries.find { |e| e.name.casecmp?(".w3dhub.patch") || e.name.casecmp?(".bhppatch") } + patch_entry.read + + patch_info = JSON.parse(patch_entry.blob, symbolize_names: true) logger.info(LOG_TAG) { " Loading #{file_path}..." } - target_mix = W3DHub::Mixer::Reader.new(file_path: "#{file_path}", ignore_crc_mismatches: false) + target_mix = W3DHub::WWMix.new(path: "#{file_path}") + unless target_mix.load + raise target_mix.error_reason + end logger.info(LOG_TAG) { " Removing files..." } if patch_info[:removedFiles].size.positive? patch_info[:removedFiles].each do |file| logger.debug(LOG_TAG) { " #{file}" } - target_mix.package.files.delete_if { |f| f.name.casecmp?(file) } + target_mix.entries.delete_if { |e| e.name.casecmp?(file) } end logger.info(LOG_TAG) { " Adding/Updating files..." } if patch_info[:updatedFiles].size.positive? patch_info[:updatedFiles].each do |file| logger.debug(LOG_TAG) { " #{file}" } - patch = patch_mix.package.files.find { |f| f.name.casecmp?(file) } - target = target_mix.package.files.find { |f| f.name.casecmp?(file) } + patch = patch_mix.entries.find { |e| e.name.casecmp?(file) } + target = target_mix.entries.find { |e| e.name.casecmp?(file) } if target - target_mix.package.files[target_mix.package.files.index(target)] = patch + target_mix.entries[target_mix.entries.index(target)] = patch else - target_mix.package.files << patch + target_mix.entries << patch end end logger.info(LOG_TAG) { " Writing updated #{file_path}..." } if patch_info[:updatedFiles].size.positive? - W3DHub::Mixer::Writer.new(file_path: "#{file_path}", package: target_mix.package, memory_buffer: true, encrypted: target_mix.encrypted?) + temp_mix_path = "#{temp_path}/#{File.basename(file_path)}" + temp_mix = W3DHub::WWMix.new(path: temp_mix_path) + target_mix.entries.each { |e| temp_mix.add_entry(entry: e) } + unless temp_mix.save + raise temp_mix.error_reason + end + + # Overwrite target mix with temp mix + FileUtils.mv(temp_mix_path, file_path) FileUtils.remove_dir(temp_path) diff --git a/lib/mixer.rb b/lib/mixer.rb deleted file mode 100644 index 86d4399..0000000 --- a/lib/mixer.rb +++ /dev/null @@ -1,386 +0,0 @@ -require "digest" -require "stringio" - -class W3DHub - - # https://github.com/TheUnstoppable/MixLibrary used for reference - class Mixer - DEFAULT_BUFFER_SIZE = 32_000_000 - MIX1_HEADER = 0x3158494D - MIX2_HEADER = 0x3258494D - - class MixParserException < RuntimeError; end - class MixFormatException < RuntimeError; end - - class MemoryBuffer - def initialize(file_path:, mode:, buffer_size:, encoding: Encoding::ASCII_8BIT) - @mode = mode - - @file = File.open(file_path, mode == :read ? "rb" : "wb") - @file.pos = 0 - @file_size = File.size(file_path) - - @buffer_size = buffer_size - @chunk = 0 - @last_chunk = 0 - @max_chunks = @file_size / @buffer_size - @last_cached_chunk = nil - - @encoding = encoding - - @last_buffer_pos = 0 - @buffer = @mode == :read ? StringIO.new(@file.read(@buffer_size)) : StringIO.new - @buffer.set_encoding(encoding) - - # Cache frequently accessed chunks to reduce disk hits - @cache = {} - end - - def pos - @chunk * @buffer_size + @buffer.pos - end - - def pos=(offset) - last_chunk = @chunk - @chunk = offset / @buffer_size - - raise "No backsies! #{offset} (#{@chunk}/#{last_chunk})" if @mode == :write && @chunk < last_chunk - - fetch_chunk(@chunk) if @mode == :read - - @buffer.pos = offset % @buffer_size - end - - # string of bytes - def write(bytes) - length = bytes.length - - # Crossing buffer boundry - if @buffer.pos + length > @buffer_size - - edge_size = @buffer_size - @buffer.pos - buffer_edge = bytes[0...edge_size] - - bytes_to_write = bytes.length - buffer_edge.length - chunks_to_write = (bytes_to_write / @buffer_size.to_f).ceil - bytes_written = buffer_edge.length - - @buffer.write(buffer_edge) - flush_chunk - - chunks_to_write.times do |i| - i += 1 - - @buffer.write(bytes[bytes_written...bytes_written + @buffer_size]) - bytes_written += @buffer_size - - flush_chunk if string.length == @buffer_size - end - else - @buffer.write(bytes) - end - - bytes - end - - def write_header(data_offset:, name_offset:) - flush_chunk - - @file.pos = 4 - write_i32(data_offset) - write_i32(name_offset) - - @file.pos = 0 - end - - def write_i32(int) - @file.write([int].pack("l")) - end - - def read(bytes = nil) - raise ArgumentError, "Cannot read whole file" if bytes.nil? - raise ArgumentError, "Cannot under read buffer" if bytes.negative? - - # Long read, need to fetch next chunk while reading, mostly defeats this class...? - if @buffer.pos + bytes > buffered - buff = string[@buffer.pos..buffered] - - bytes_to_read = bytes - buff.length - chunks_to_read = (bytes_to_read / @buffer_size.to_f).ceil - - chunks_to_read.times do |i| - i += 1 - - fetch_chunk(@chunk + 1) - - if i == chunks_to_read # read partial - already_read_bytes = (chunks_to_read - 1) * @buffer_size - bytes_more_to_read = bytes_to_read - already_read_bytes - - buff << @buffer.read(bytes_more_to_read) - else - buff << @buffer.read - end - end - - buff - else - fetch_chunk(@chunk) if @last_chunk != @chunk - - @buffer.read(bytes) - end - end - - def readbyte - fetch_chunk(@chunk + 1) if @buffer.pos + 1 > buffered - - @buffer.readbyte - end - - def fetch_chunk(chunk) - raise ArgumentError, "Cannot fetch chunk #{chunk}, only #{@max_chunks} exist!" if chunk > @max_chunks - @last_chunk = @chunk - @chunk = chunk - @last_buffer_pos = @buffer.pos - - cached = @cache[chunk] - - if cached - @buffer.string = cached - else - @file.pos = chunk * @buffer_size - buff = @buffer.string = @file.read(@buffer_size) - - # Cache the active chunk (implementation bounces from @file_data_chunk and back to this for each 'file' processed) - if @chunk != @file_data_chunk && @chunk != @last_cached_chunk - @cache.delete(@last_cached_chunk) unless @last_cached_chunk == @file_data_chunk - @cache[@chunk] = buff - @last_cached_chunk = @chunk - end - - buff - end - end - - # This is accessed quite often, keep it around - def cache_file_data_chunk! - @file_data_chunk = @chunk - - last_buffer_pos = @buffer.pos - @buffer.pos = 0 - @cache[@chunk] = @buffer.read - @buffer.pos = last_buffer_pos - end - - def flush_chunk - @last_chunk = @chunk - @chunk += 1 - - @file.pos = @last_chunk * @buffer_size - @file.write(string) - - @buffer.string = "".force_encoding(@encoding) - end - - def string - @buffer.string - end - - def buffered - @buffer.string.length - end - - def close - @file&.close - end - end - - class Reader - attr_reader :package - - def initialize(file_path:, ignore_crc_mismatches: false, metadata_only: false, buffer_size: DEFAULT_BUFFER_SIZE) - @package = Package.new - - @buffer = MemoryBuffer.new(file_path: file_path, mode: :read, buffer_size: buffer_size) - - @buffer.pos = 0 - - @encrypted = false - - # Valid header - if (mime = read_i32) && (mime == MIX1_HEADER || mime == MIX2_HEADER) - @encrypted = mime == MIX2_HEADER - - file_data_offset = read_i32 - file_names_offset = read_i32 - - @buffer.pos = file_names_offset - file_count = read_i32 - - file_count.times do - @package.files << Package::File.new(name: read_string) - end - - @buffer.pos = file_data_offset - @buffer.cache_file_data_chunk! - - _file_count = read_i32 - - file_count.times do |i| - file = @package.files[i] - - file.mix_crc = read_u32.to_s(16).rjust(8, "0") - file.content_offset = read_u32 - file.content_length = read_u32 - - if !ignore_crc_mismatches && file.mix_crc != file.file_crc - raise MixParserException, "CRC mismatch for #{file.name}. #{file.mix_crc.inspect} != #{file.file_crc.inspect}" - end - - pos = @buffer.pos - @buffer.pos = file.content_offset - file.data = @buffer.read(file.content_length) unless metadata_only - @buffer.pos = pos - end - else - raise MixParserException, "Invalid MIX file: Expected \"#{MIX1_HEADER}\" or \"#{MIX2_HEADER}\", got \"0x#{mime.to_s(16).upcase}\"\n(#{file_path})" - end - - ensure - @buffer&.close - @buffer = nil # let GC collect - end - - def read_i32 - @buffer.read(4).unpack1("l") - end - - def read_u32 - @buffer.read(4).unpack1("L") - end - - def read_string - buffer = "" - - length = @buffer.readbyte - - length.times do - buffer << @buffer.readbyte - end - - buffer.strip - end - - def encrypted? - @encrypted - end - end - - class Writer - attr_reader :package - - def initialize(file_path:, package:, memory_buffer: false, buffer_size: DEFAULT_BUFFER_SIZE, encrypted: false) - @package = package - - @buffer = MemoryBuffer.new(file_path: file_path, mode: :write, buffer_size: buffer_size) - @buffer.pos = 0 - - @encrypted = encrypted - - @buffer.write(encrypted? ? "MIX2" : "MIX1") - - files = @package.files.sort { |a, b| a.file_crc <=> b.file_crc } - - @buffer.pos = 16 - - files.each do |file| - file.content_offset = @buffer.pos - file.content_length = file.data.length - @buffer.write(file.data) - - @buffer.pos += -@buffer.pos & 7 - end - - file_data_offset = @buffer.pos - write_i32(files.count) - - files.each do |file| - write_u32(file.file_crc.to_i(16)) - write_u32(file.content_offset) - write_u32(file.content_length) - end - - file_name_offset = @buffer.pos - write_i32(files.count) - - files.each do |file| - write_byte(file.name.length + 1) - @buffer.write("#{file.name}\0") - end - - @buffer.write_header(data_offset: file_data_offset, name_offset: file_name_offset) - ensure - @buffer&.close - end - - def write_i32(int) - @buffer.write([int].pack("l")) - end - - def write_u32(uint) - @buffer.write([uint].pack("L")) - end - - def write_byte(byte) - @buffer.write([byte].pack("c")) - end - - def encrypted? - @encrypted - end - end - - # Eager loads patch file and streams target file metadata (doen't load target file data or generate CRCs) - # after target file metadata is loaded, create a temp file and merge patched files into list then - # build ordered file list and stream patched files and target file chunks into temp file, - # after that is done, replace target file with temp file - class Patcher - def initialize(patch_files:, target_file:, temp_file:, buffer_size: DEFAULT_BUFFER_SIZE) - @patch_files = patch_files.to_a.map { |f| Reader.new(file_path: f) } - @target_file = File.open(target_file) - @temp_file = File.open(temp_file, "a+b") - @buffer_size = buffer_size - end - end - - class Package - attr_reader :files - - def initialize(files: []) - @files = files - end - - class File - attr_accessor :name, :mix_crc, :content_offset, :content_length, :data - - def initialize(name:, mix_crc: nil, content_offset: nil, content_length: nil, data: nil) - @name = name - @mix_crc = mix_crc - @content_offset = content_offset - @content_length = content_length - @data = data - end - - def file_crc - return "e6fe46b8" if @name.downcase == ".w3dhub.patch" - - Digest::CRC32.hexdigest(@name.upcase) - end - - def data_crc - Digest::CRC32.hexdigest(@data) - end - end - end - end -end \ No newline at end of file diff --git a/w3d_hub_linux_launcher.rb b/w3d_hub_linux_launcher.rb index ac73b04..0dfbc88 100644 --- a/w3d_hub_linux_launcher.rb +++ b/w3d_hub_linux_launcher.rb @@ -105,7 +105,7 @@ require_relative "lib/store" require_relative "lib/window" require_relative "lib/cache" require_relative "lib/settings" -require_relative "lib/mixer" +require_relative "lib/ww_mix" require_relative "lib/ico" require_relative "lib/broadcast_server" require_relative "lib/hardware_survey"