diff --git a/Abstractions/Extractors/IFS/Avslz.cs b/Abstractions/Extractors/IFS/Avslz.cs new file mode 100644 index 0000000..a9b5b72 --- /dev/null +++ b/Abstractions/Extractors/IFS/Avslz.cs @@ -0,0 +1,68 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace Abstractions.Extractors.IFS; + +/// The simple LZ variant ("AVSLZ") used to compress some IFS texture payloads. +internal static class Avslz +{ + internal static byte[] Decompress(byte[] input) + { + var output = new List(); + var offset = 0; + while (offset < input.Length) + { + var flag = input[offset++]; + for (var bit = 0; bit < 8; bit++) + { + if (((flag >> bit) & 1) != 0) + { + if (offset >= input.Length) throw new InvalidDataException("Truncated AVSLZ literal"); + output.Add(input[offset++]); + continue; + } + if (offset + 1 >= input.Length) throw new InvalidDataException("Truncated AVSLZ reference"); + var word = (input[offset] << 8) | input[offset + 1]; + offset += 2; + var position = word >> 4; + var length = (word & 15) + 3; + if (position == 0) return [.. output]; + if (position > output.Count) + { + var zeros = Math.Min(position - output.Count, length); + for (var index = 0; index < zeros; index++) output.Add(0); + length -= zeros; + } + for (var index = 0; index < length; index++) output.Add(output[output.Count - position]); + } + } + throw new InvalidDataException("AVSLZ stream has no terminator"); + } + + /// Encodes raw bytes as an all-literal AVSLZ stream (no back-references), terminated correctly. + internal static byte[] CompressLiterals(byte[] input) + { + var completeGroups = input.Length / 8; + var remainder = input.Length % 8; + // completeGroups * (1 flag byte + 8 literal bytes) + a terminating flag byte + any trailing literals + // + a 2-byte zero terminator reference. + var compressed = new byte[completeGroups * 9 + (remainder != 0 ? remainder + 3 : 3)]; + var source = 0; + var target = 0; + for (var group = 0; group < completeGroups; group++) + { + compressed[target++] = 0xFF; + Buffer.BlockCopy(input, source, compressed, target, 8); + source += 8; + target += 8; + } + compressed[target++] = (byte)(remainder != 0 ? (1 << remainder) - 1 : 0); + if (remainder == 0) return compressed; + Buffer.BlockCopy(input, source, compressed, target, remainder); + target += remainder; + // Trailing 2 bytes are the zero back-reference terminator (position 0); the array is + // already zero-initialized, so no explicit write is needed here. + return compressed; + } +} diff --git a/Abstractions/Extractors/IFS/Dxt.cs b/Abstractions/Extractors/IFS/Dxt.cs new file mode 100644 index 0000000..08e79a1 --- /dev/null +++ b/Abstractions/Extractors/IFS/Dxt.cs @@ -0,0 +1,93 @@ +using System; +using System.Buffers.Binary; +using System.Collections.Generic; + +namespace Abstractions.Extractors.IFS; + +/// S3TC block decompression (DXT1 / DXT5) for IFS textures. +internal static class Dxt +{ + private static (byte R, byte G, byte B) Color565(int value) => ( + (byte)Math.Round((value >> 11 & 31) * 255.0 / 31), + (byte)Math.Round((value >> 5 & 63) * 255.0 / 63), + (byte)Math.Round((value & 31) * 255.0 / 31)); + + private static (byte R, byte G, byte B, byte A)[] DecodeColors(ReadOnlySpan data, int offset, bool forceFour) + { + int first = BinaryPrimitives.ReadUInt16LittleEndian(data.Slice(offset, 2)); + int second = BinaryPrimitives.ReadUInt16LittleEndian(data.Slice(offset + 2, 2)); + var a = Color565(first); + var b = Color565(second); + var colors = new (byte R, byte G, byte B, byte A)[4]; + colors[0] = (a.R, a.G, a.B, 255); + colors[1] = (b.R, b.G, b.B, 255); + if (first > second || forceFour) + { + colors[2] = ((byte)Math.Round((2 * a.R + b.R) / 3.0), (byte)Math.Round((2 * a.G + b.G) / 3.0), (byte)Math.Round((2 * a.B + b.B) / 3.0), 255); + colors[3] = ((byte)Math.Round((a.R + 2 * b.R) / 3.0), (byte)Math.Round((a.G + 2 * b.G) / 3.0), (byte)Math.Round((a.B + 2 * b.B) / 3.0), 255); + } + else + { + colors[2] = ((byte)Math.Round((a.R + b.R) / 2.0), (byte)Math.Round((a.G + b.G) / 2.0), (byte)Math.Round((a.B + b.B) / 2.0), 255); + colors[3] = (0, 0, 0, 0); + } + return colors; + } + + internal static byte[] Decode(byte[] raw, int width, int height, bool dxt5) + { + var data = (byte[])raw.Clone(); + for (var index = 0; index + 1 < data.Length; index += 2) + (data[index], data[index + 1]) = (data[index + 1], data[index]); + + var rgba = new byte[width * height * 4]; + var blockSize = dxt5 ? 16 : 8; + var offset = 0; + for (var by = 0; by < height; by += 4) + { + for (var bx = 0; bx < width; bx += 4) + { + if (offset + blockSize > data.Length) return rgba; + var alpha = new int[16]; + Array.Fill(alpha, 255); + var colorOffset = offset; + if (dxt5) + { + byte a0 = data[offset], a1 = data[offset + 1]; + var palette = new List { a0, a1 }; + if (a0 > a1) + for (var i = 1; i <= 6; i++) palette.Add((int)Math.Round(((7 - i) * a0 + i * a1) / 7.0)); + else + { + for (var i = 1; i <= 4; i++) palette.Add((int)Math.Round(((5 - i) * a0 + i * a1) / 5.0)); + palette.Add(0); + palette.Add(255); + } + ulong bits = 0; + for (var i = 0; i < 6; i++) bits |= (ulong)data[offset + 2 + i] << (i * 8); + for (var i = 0; i < 16; i++) alpha[i] = palette[(int)(bits >> (i * 3) & 7)]; + colorOffset += 8; + } + var colors = DecodeColors(data, colorOffset, dxt5); + var indices = BinaryPrimitives.ReadUInt32LittleEndian(data.AsSpan(colorOffset + 4, 4)); + for (var py = 0; py < 4; py++) + { + for (var px = 0; px < 4; px++) + { + int x = bx + px, y = by + py; + if (x >= width || y >= height) continue; + var pixel = py * 4 + px; + var color = colors[(int)(indices >> (pixel * 2) & 3)]; + var target = (y * width + x) * 4; + rgba[target] = color.R; + rgba[target + 1] = color.G; + rgba[target + 2] = color.B; + rgba[target + 3] = dxt5 ? (byte)alpha[pixel] : color.A; + } + } + offset += blockSize; + } + } + return rgba; + } +} diff --git a/Abstractions/Extractors/IFS/IfsBatchExtractor.cs b/Abstractions/Extractors/IFS/IfsBatchExtractor.cs new file mode 100644 index 0000000..1445692 --- /dev/null +++ b/Abstractions/Extractors/IFS/IfsBatchExtractor.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace Abstractions.Extractors.IFS; + +/// Progress reported after each archive finishes during a batch extract. +public sealed record BatchExtractProgress(int Processed, int TotalFiles, int ItemsExtracted, int Skipped, int FailedCount); + +/// One archive that failed to extract during a batch run. +public sealed record BatchExtractFailure(string File, string Error); + +/// Outcome of a batch extract across every archive under a source directory. +public sealed record BatchExtractResult(int TotalFiles, int TotalItemsExtracted, int Skipped, IReadOnlyList Failures) +{ + public int Succeeded => TotalFiles - Failures.Count - Skipped; +} + +/// +/// Reusable, logic for extracting every .ifs archive under a directory tree. +/// +public static class IfsBatchExtractor +{ + public static string[] FindIfsFiles(string sourceRootFull) => + [ + .. Directory.EnumerateFiles(sourceRootFull, "*", SearchOption.AllDirectories) + .Where(file => file.EndsWith(".ifs", StringComparison.OrdinalIgnoreCase)) + ]; + + public static BatchExtractResult ExtractFiles( + string sourceRootFull, + IReadOnlyList files, + string outputDirectory, + string? filter, + bool includeTextures, + bool includeRaw, + bool skipExisting, + Action? onProgress = null) + { + var processed = 0; + var totalItems = 0; + var skipped = 0; + var failures = new ConcurrentBag(); + + Parallel.ForEach(files, file => + { + try + { + // Mirror the archive's own directory under the output root, so e.g. sd/popn1/anime.ifs + // and tex/6/rose_6a.ifs land in separate places even if two archives share a base name. + var relativeDirectory = Path.GetDirectoryName(Path.GetRelativePath(sourceRootFull, file)); + var fileOutputRoot = string.IsNullOrEmpty(relativeDirectory) + ? outputDirectory + : Path.Combine(outputDirectory, relativeDirectory); + + if (skipExisting && Directory.Exists(IfsFile.GetOutputDirectory(file, fileOutputRoot))) + { + Interlocked.Increment(ref skipped); + return; + } + + var outputs = IfsFile.ExtractAll(file, fileOutputRoot, filter, includeTextures, includeRaw); + Interlocked.Add(ref totalItems, outputs.Count); + } + catch (Exception ex) + { + failures.Add(new BatchExtractFailure(file, ex.Message)); + } + finally + { + var done = Interlocked.Increment(ref processed); + onProgress?.Invoke(new BatchExtractProgress(done, files.Count, totalItems, skipped, failures.Count)); + } + }); + + return new BatchExtractResult( + files.Count, + totalItems, + skipped, + [.. failures.OrderBy(failure => failure.File, StringComparer.Ordinal)]); + } +} diff --git a/Abstractions/Extractors/IFS/IfsFile.cs b/Abstractions/Extractors/IFS/IfsFile.cs new file mode 100644 index 0000000..8971624 --- /dev/null +++ b/Abstractions/Extractors/IFS/IfsFile.cs @@ -0,0 +1,342 @@ +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.IO; +using System.IO.Enumeration; +using System.Linq; +using System.Security.Cryptography; +using System.Text; + +namespace Abstractions.Extractors.IFS; + +/// Extracts and replaces textures inside AVS/IFS archives. +public static class IfsFile +{ + private const uint Magic = 0x6CAD8F89; + + private static byte[] Concat(params byte[][] parts) + { + var result = new byte[parts.Sum(part => part.Length)]; + var offset = 0; + foreach (var part in parts) + { + part.CopyTo(result, offset); + offset += part.Length; + } + return result; + } + + private static List ValidateIntegrity(byte[] input, byte[] manifestData, KNode manifest, int manifestEnd) + { + var manifestHash = MD5.HashData(manifestData); + if (!input.AsSpan(20, 16).SequenceEqual(manifestHash)) + throw new InvalidDataException("IFS manifest checksum is invalid."); + + var info = KBin.FindChild(manifest, "_info_"); + var dataSizeNode = info is not null ? KBin.FindChild(info, "size") : null; + var dataHashNode = info is not null ? KBin.FindChild(info, "md5") : null; + if (dataSizeNode is null || dataHashNode is null || dataHashNode.Values.Length != 16) + throw new InvalidDataException("IFS integrity fields were not found."); + + var data = input.AsSpan(manifestEnd).ToArray(); + if ((int)dataSizeNode.Values[0] != data.Length) + throw new InvalidDataException("IFS data size is invalid."); + var expectedDataHash = dataHashNode.Values.Select(v => (byte)v).ToArray(); + if (!MD5.HashData(data).AsSpan().SequenceEqual(expectedDataHash)) + throw new InvalidDataException("IFS data checksum is invalid."); + + var fileNodes = KBin.Walk(manifest).Where(node => node.Values.Length == 3 && node.ValueOffset is not null).ToList(); + foreach (var node in fileNodes) + { + var offset = node.Values[0]; + var size = node.Values[1]; + if (offset != Math.Floor(offset) || size != Math.Floor(size) || offset < 0 || size < 0 || offset + size > data.Length) + throw new InvalidDataException($"IFS entry {KBin.FixedName(node.Name)} has invalid bounds."); + } + return fileNodes; + } + + private static void WriteNodeNumber(byte[] buffer, KNode node, int index, int value) + { + if (!KBin.Formats.TryGetValue(node.Type, out var format) || node.ValueOffset is null || format.Float) + throw new InvalidDataException($"Unsupported IFS manifest value {node.Name}"); + var offset = node.ValueOffset.Value + index * format.Size; + if (format.Size == 1) + buffer[offset] = unchecked((byte)value); + else if (format.Size == 2) + { + if (format.Signed) BinaryPrimitives.WriteInt16BigEndian(buffer.AsSpan(offset, 2), (short)value); + else BinaryPrimitives.WriteUInt16BigEndian(buffer.AsSpan(offset, 2), (ushort)value); + } + else if (format.Size == 4) + { + if (format.Signed) BinaryPrimitives.WriteInt32BigEndian(buffer.AsSpan(offset, 4), value); + else BinaryPrimitives.WriteUInt32BigEndian(buffer.AsSpan(offset, 4), (uint)value); + } + else + throw new InvalidDataException($"Unsupported IFS manifest number size {format.Size}"); + node.Values[index] = value; + } + + private static string HashName(string imageName) => + Convert.ToHexStringLower(MD5.HashData(Encoding.UTF8.GetBytes(imageName))); + + private static string BuildRelativePath(KNode manifest, KNode node) + { + var segments = new List(); + for (var current = node; current is not null && current != manifest; current = current.Parent) + segments.Add(KBin.FixedName(current.Name)); + segments.Reverse(); + return Path.Combine(segments.ToArray()); + } + + private static bool MatchesFilter(string? pattern, string candidate) + { + if (string.IsNullOrEmpty(pattern)) return true; + var normalized = candidate.Replace('\\', '/'); + return FileSystemName.MatchesSimpleExpression(pattern, Path.GetFileName(normalized), ignoreCase: true) + || FileSystemName.MatchesSimpleExpression(pattern, normalized, ignoreCase: true); + } + + /// Where writes a given archive's output, so callers can check it up front (e.g. to skip already-extracted archives). + public static string GetOutputDirectory(string source, string outputRoot) => + Path.Combine(outputRoot, $"{Path.GetFileNameWithoutExtension(source)}_ifs"); + + /// + /// Extracts every texture from as decoded PNGs, and dumps every other raw + /// entry in the archive (audio, AFP/BSI sprite metadata, or anything else the manifest describes) verbatim, + /// all into <outputRoot>/<name>_ifs/, mirroring the archive's own folder structure. + /// + /// + /// Patch archives (those referencing a base archive via _super_) are not supported and will + /// cause the extraction to be refused rather than risk extracting incorrect data. + /// + /// Path to the IFS archive file to extract. + /// Root directory where the <name>_ifs/ output folder will be created. + /// Only extract entries whose name matches this glob (e.g. "gr*"); null extracts everything. + /// Whether to decode and write texture images as PNGs. + /// Whether to dump every other entry (audio, sprite metadata, etc.) as a raw file. + /// A read-only list of full paths to every file that was written. + public static IReadOnlyList ExtractAll(string source, string outputRoot, + string? namePattern = null, bool includeTextures = true, bool includeRaw = true) + { + var input = File.ReadAllBytes(source); + if (input.Length < 36 || BinaryPrimitives.ReadUInt32BigEndian(input.AsSpan(0, 4)) != Magic || BinaryPrimitives.ReadUInt16BigEndian(input.AsSpan(4, 2)) < 2) + throw new InvalidDataException("Unsupported or invalid IFS file"); + var manifestEnd = (int)BinaryPrimitives.ReadUInt32BigEndian(input.AsSpan(16, 4)); + if (manifestEnd <= 36 || manifestEnd > input.Length) throw new InvalidDataException("Invalid IFS manifest size"); + var manifestData = input.AsSpan(36, manifestEnd - 36).ToArray(); + var manifest = KBin.Read(manifestData); + ValidateIntegrity(input, manifestData, manifest, manifestEnd); + + var outputDirectory = GetOutputDirectory(source, outputRoot); + Directory.CreateDirectory(outputDirectory); + var outputDirectoryFull = Path.GetFullPath(outputDirectory); + + var outputs = new List(); + var consumedOffsets = new HashSet(); + + var tex = KBin.FindChild(manifest, "tex"); + if (tex is not null) + { + var files = new Dictionary(); + foreach (var entry in tex.Children) + if (entry.Values.Length >= 2) + files[KBin.FixedName(entry.Name)] = ((int)entry.Values[0], (int)entry.Values[1]); + + var textureListName = files.Keys.FirstOrDefault(name => name.EndsWith(".xml", StringComparison.Ordinal)); + if (textureListName is null) throw new InvalidDataException("IFS texture list was not found"); + var (xmlOffset, xmlSize) = files[textureListName]; + consumedOffsets.Add(xmlOffset); + var textureListData = input.AsSpan(manifestEnd + xmlOffset, xmlSize).ToArray(); + var textureList = KBin.Read(textureListData); + var compress = textureList.Attrs.GetValueOrDefault("compress", ""); + + foreach (var texture in textureList.Children) + { + var format = texture.Attrs.GetValueOrDefault("format"); + foreach (var image in texture.Children.Where(node => node.Name == "image")) + { + var imageName = image.Attrs.GetValueOrDefault("name"); + var imgrectNode = KBin.FindChild(image, "imgrect"); + if (string.IsNullOrEmpty(imageName) || imgrectNode is null || imgrectNode.Values.Length < 4) continue; + + var manifestName = HashName(imageName); + if (!files.TryGetValue(manifestName, out var entry) && !files.TryGetValue($"_{manifestName}", out entry)) continue; + // Mark this offset as belonging to the texture pipeline regardless of includeTextures/namePattern, + // so a skipped-but-identified texture never falls through into the generic raw dump below. + consumedOffsets.Add(entry.Offset); + if (!includeTextures || !MatchesFilter(namePattern, imageName)) continue; + + if (imageName.Contains('/') || imageName.Contains('\\') || imageName.Contains('\0')) + throw new InvalidDataException("IFS texture name contains an invalid path."); + + var data = input.AsSpan(manifestEnd + entry.Offset, entry.Size).ToArray(); + if (compress == "avslz") + { + if (data.Length < 8) throw new InvalidDataException($"Invalid AVSLZ texture {imageName}"); + var uncompressed = BinaryPrimitives.ReadUInt32BigEndian(data.AsSpan(0, 4)); + var compressedLength = BinaryPrimitives.ReadUInt32BigEndian(data.AsSpan(4, 4)); + if (uncompressed > 64 * 1024 * 1024) throw new InvalidDataException($"AVSLZ texture {imageName} is too large."); + if (data.Length == compressedLength + 8) + { + data = Avslz.Decompress(data[8..]); + if (data.Length != uncompressed) throw new InvalidDataException($"AVSLZ size mismatch for {imageName}"); + } + else + { + data = Concat(data[8..], data[..8]); + } + } + + var width = (int)Math.Floor((imgrectNode.Values[1] - imgrectNode.Values[0]) / 2); + var height = (int)Math.Floor((imgrectNode.Values[3] - imgrectNode.Values[2]) / 2); + if (width <= 0 || height <= 0 || width > 8192 || height > 8192 || (long)width * height > 16 * 1024 * 1024) + throw new InvalidDataException($"IFS texture {imageName} has invalid dimensions."); + + var output = Path.GetFullPath(Path.Combine(outputDirectory, $"{imageName}.png")); + if (!output.StartsWith(outputDirectoryFull + Path.DirectorySeparatorChar, StringComparison.Ordinal)) + throw new InvalidDataException("IFS texture output path is invalid."); + + File.WriteAllBytes(output, Png.Encode(width, height, PixelCodec.Decode(format, data, width, height))); + outputs.Add(output); + } + } + } + + // Dump every remaining raw entry the manifest describes -- audio (.2dx/.bin), AFP/BSI sprite + // metadata, unrecognized image entries, and anything else -- so nothing is silently skipped. + if (includeRaw) + { + foreach (var node in KBin.Walk(manifest).Where(n => n.Values.Length == 3 && n.ValueOffset is not null)) + { + var offset = (int)node.Values[0]; + var size = (int)node.Values[1]; + if (consumedOffsets.Contains(offset)) continue; + if (offset < 0 || size < 0 || offset + size > input.Length - manifestEnd) continue; + + var relativePath = BuildRelativePath(manifest, node); + if (string.IsNullOrEmpty(relativePath) || !MatchesFilter(namePattern, relativePath)) continue; + var output = Path.GetFullPath(Path.Combine(outputDirectory, relativePath)); + if (!output.StartsWith(outputDirectoryFull + Path.DirectorySeparatorChar, StringComparison.Ordinal)) continue; + + Directory.CreateDirectory(Path.GetDirectoryName(output)!); + File.WriteAllBytes(output, input.AsSpan(manifestEnd + offset, size).ToArray()); + outputs.Add(output); + } + } + + return outputs; + } + + /// Replaces one texture's pixels in , writing the patched archive to . Returns a re-encoded PNG preview of the pixels actually written. + public static byte[] ReplaceTexture(string source, string destination, string imageName, int width, int height, byte[] png) + { + var sourcePath = Path.GetFullPath(source); + var destinationPath = Path.GetFullPath(destination); + var sameFile = OperatingSystem.IsWindows() + ? string.Equals(sourcePath, destinationPath, StringComparison.OrdinalIgnoreCase) + : sourcePath == destinationPath; + if (sameFile) throw new InvalidOperationException("Refusing to overwrite the source IFS directly."); + + var (_, _, rgba) = Png.Decode(png, width, height); + var input = File.ReadAllBytes(source); + if (input.Length < 36 || BinaryPrimitives.ReadUInt32BigEndian(input.AsSpan(0, 4)) != Magic || BinaryPrimitives.ReadUInt16BigEndian(input.AsSpan(4, 2)) < 2) + throw new InvalidDataException("Unsupported or invalid IFS file."); + var manifestEnd = (int)BinaryPrimitives.ReadUInt32BigEndian(input.AsSpan(16, 4)); + if (manifestEnd <= 36 || manifestEnd > input.Length) throw new InvalidDataException("Invalid IFS manifest size."); + var manifestData = input.AsSpan(36, manifestEnd - 36).ToArray(); + var manifest = KBin.Read(manifestData); + var fileNodes = ValidateIntegrity(input, manifestData, manifest, manifestEnd); + + var tex = KBin.FindChild(manifest, "tex"); + if (tex is null) throw new InvalidDataException("IFS texture folder was not found."); + + var files = new Dictionary(); + foreach (var node in tex.Children) + if (node.Values.Length >= 2) + files[KBin.FixedName(node.Name)] = node; + + var textureListName = files.Keys.FirstOrDefault(name => name.EndsWith(".xml", StringComparison.Ordinal)); + if (textureListName is null) throw new InvalidDataException("IFS texture list was not found."); + var textureListNode = files[textureListName]; + var textureListOffset = (int)textureListNode.Values[0]; + var textureListSize = (int)textureListNode.Values[1]; + var textureList = KBin.Read([.. input.AsSpan(manifestEnd + textureListOffset, textureListSize)]); + + var format = ""; + var compress = textureList.Attrs.GetValueOrDefault("compress", ""); + int actualWidth = 0, actualHeight = 0; + foreach (var texture in textureList.Children) + { + var image = texture.Children.FirstOrDefault(node => node.Name == "image" && node.Attrs.GetValueOrDefault("name") == imageName); + if (image is null) continue; + var imgrectNode = KBin.FindChild(image, "imgrect"); + if (imgrectNode is null || imgrectNode.Values.Length < 4) throw new InvalidDataException("IFS texture dimensions were not found."); + format = texture.Attrs.GetValueOrDefault("format", ""); + actualWidth = (int)Math.Floor((imgrectNode.Values[1] - imgrectNode.Values[0]) / 2); + actualHeight = (int)Math.Floor((imgrectNode.Values[3] - imgrectNode.Values[2]) / 2); + break; + } + if (actualWidth != width || actualHeight != height) + throw new InvalidDataException($"IFS texture size is {actualWidth} x {actualHeight}, not {width} x {height}."); + if (format != "argb8888rev") + throw new InvalidDataException($"Unsupported writable IFS texture format {(string.IsNullOrEmpty(format) ? "unknown" : format)}."); + + var packedName = HashName(imageName); + if (!files.TryGetValue(packedName, out var targetNode) && !files.TryGetValue($"_{packedName}", out targetNode)) + throw new InvalidDataException($"IFS texture {imageName} was not found."); + var targetOffset = (int)targetNode.Values[0]; + var targetSize = (int)targetNode.Values[1]; + + var raw = PixelCodec.EncodeArgb8888Rev(rgba); + var packed = raw; + if (compress == "avslz") + { + var compressedLiterals = Avslz.CompressLiterals(raw); + var header = new byte[8]; + BinaryPrimitives.WriteUInt32BigEndian(header.AsSpan(0, 4), (uint)raw.Length); + BinaryPrimitives.WriteUInt32BigEndian(header.AsSpan(4, 4), (uint)compressedLiterals.Length); + packed = Concat(header, compressedLiterals); + } + else if (!string.IsNullOrEmpty(compress)) + { + throw new InvalidDataException($"Unsupported IFS texture compression {compress}."); + } + + var originalData = input.AsSpan(manifestEnd).ToArray(); + var nextOffset = fileNodes + .Select(node => (int)node.Values[0]) + .Where(offset => offset > targetOffset) + .DefaultIfEmpty(KBin.Align16(targetOffset + targetSize)) + .Min(); + if (targetOffset < 0 || targetSize < 0 || nextOffset < targetOffset + targetSize || nextOffset > originalData.Length) + throw new InvalidDataException("IFS texture entry has invalid bounds."); + + var padding = new byte[KBin.Align16(packed.Length) - packed.Length]; + var replacement = Concat(packed, padding); + var oldSpan = nextOffset - targetOffset; + var delta = replacement.Length - oldSpan; + var data = Concat(originalData[..targetOffset], replacement, originalData[nextOffset..]); + + WriteNodeNumber(manifestData, targetNode, 1, packed.Length); + foreach (var node in fileNodes) + { + var offset = (int)node.Values[0]; + if (node != targetNode && offset >= nextOffset) WriteNodeNumber(manifestData, node, 0, offset + delta); + } + + var info = KBin.FindChild(manifest, "_info_"); + var dataSizeNode = info is not null ? KBin.FindChild(info, "size") : null; + var dataHashNode = info is not null ? KBin.FindChild(info, "md5") : null; + if (dataSizeNode is null || dataHashNode is null || dataHashNode.ValueOffset is null || dataHashNode.Values.Length != 16) + throw new InvalidDataException("IFS integrity fields were not found."); + WriteNodeNumber(manifestData, dataSizeNode, 0, data.Length); + MD5.HashData(data).CopyTo(manifestData.AsSpan(dataHashNode.ValueOffset.Value)); + + var header2 = input.AsSpan(0, 36).ToArray(); + MD5.HashData(manifestData).CopyTo(header2.AsSpan(20)); + File.WriteAllBytes(destination, Concat(header2, manifestData, data)); + return Png.Encode(width, height, rgba); + } +} diff --git a/Abstractions/Extractors/IFS/KBin.cs b/Abstractions/Extractors/IFS/KBin.cs new file mode 100644 index 0000000..2867992 --- /dev/null +++ b/Abstractions/Extractors/IFS/KBin.cs @@ -0,0 +1,248 @@ +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Numerics; +using System.Text; + +namespace Abstractions.Extractors.IFS; + +/// A node in the IFS "kbin" binary XML tree. +internal sealed class KNode +{ + public required string Name { get; init; } + public required int Type { get; init; } + public Dictionary Attrs { get; } = new(); + public double[] Values { get; set; } = []; + public int? ValueOffset { get; set; } + public List Children { get; } = []; + public KNode? Parent { get; init; } +} + +internal readonly record struct KFormat(int Size, int Count, bool Signed = false, bool Float = false); + +/// Reads the compressed/uncompressed "kbin" binary XML format used throughout IFS archives. +internal static class KBin +{ + private const string SixBit = "0123456789:ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz"; + + internal static readonly Dictionary Formats = new() + { + [1] = new KFormat(0, 0), + [2] = new KFormat(1, 1, Signed: true), + [3] = new KFormat(1, 1), + [4] = new KFormat(2, 1, Signed: true), + [5] = new KFormat(2, 1), + [6] = new KFormat(4, 1, Signed: true), + [7] = new KFormat(4, 1), + [8] = new KFormat(8, 1, Signed: true), + [9] = new KFormat(8, 1), + [10] = new KFormat(1, -1), + [11] = new KFormat(1, -1), + [12] = new KFormat(4, 1), + [13] = new KFormat(4, 1), + [14] = new KFormat(4, 1, Float: true), + [15] = new KFormat(8, 1, Float: true), + [16] = new KFormat(1, 2, Signed: true), + [17] = new KFormat(1, 2), + [18] = new KFormat(2, 2, Signed: true), + [19] = new KFormat(2, 2), + [20] = new KFormat(4, 2, Signed: true), + [21] = new KFormat(4, 2), + [22] = new KFormat(8, 2, Signed: true), + [23] = new KFormat(8, 2), + [24] = new KFormat(4, 2, Float: true), + [25] = new KFormat(8, 2, Float: true), + [26] = new KFormat(1, 3, Signed: true), + [27] = new KFormat(1, 3), + [28] = new KFormat(2, 3, Signed: true), + [29] = new KFormat(2, 3), + [30] = new KFormat(4, 3, Signed: true), + [31] = new KFormat(4, 3), + [32] = new KFormat(8, 3, Signed: true), + [33] = new KFormat(8, 3), + [34] = new KFormat(4, 3, Float: true), + [35] = new KFormat(8, 3, Float: true), + [36] = new KFormat(1, 4, Signed: true), + [37] = new KFormat(1, 4), + [38] = new KFormat(2, 4, Signed: true), + [39] = new KFormat(2, 4), + [40] = new KFormat(4, 4, Signed: true), + [41] = new KFormat(4, 4), + [42] = new KFormat(8, 4, Signed: true), + [43] = new KFormat(8, 4), + [44] = new KFormat(4, 4, Float: true), + [45] = new KFormat(8, 4, Float: true), + [48] = new KFormat(1, 16, Signed: true), + [49] = new KFormat(1, 16), + [50] = new KFormat(2, 8, Signed: true), + [51] = new KFormat(2, 8), + [52] = new KFormat(1, 1, Signed: true), + [53] = new KFormat(1, 2, Signed: true), + [54] = new KFormat(1, 3, Signed: true), + [55] = new KFormat(1, 4, Signed: true), + [56] = new KFormat(1, 16, Signed: true), + }; + + private static int Align4(int value) => (value + 3) & ~3; + + internal static int Align16(int value) => (value + 15) & ~15; + + internal static string FixedName(string name) + { + var result = name.Replace("_E", ".").Replace("__", "_"); + if (result.Length >= 2 && result[0] == '_' && char.IsAsciiDigit(result[1])) + result = result[1..]; + return result; + } + + internal static KNode? FindChild(KNode node, string name) => + node.Children.FirstOrDefault(child => FixedName(child.Name) == name); + + internal static IEnumerable Walk(KNode node) + { + yield return node; + foreach (var child in node.Children) + foreach (var descendant in Walk(child)) + yield return descendant; + } + + internal static KNode Read(byte[] input) + { + if (input.Length < 12 || input[0] != 0xA0 || (input[1] != 0x42 && input[1] != 0x45)) + throw new InvalidDataException("Invalid binary XML"); + var compressed = input[1] == 0x42; + var nodeOffset = 8; + var nodeEnd = (int)BinaryPrimitives.ReadUInt32BigEndian(input.AsSpan(4, 4)) + 8; + var dataOffset = nodeEnd + 4; + var byteOffset = nodeEnd; + var wordOffset = nodeEnd; + var root = new KNode { Name = "$root", Type = 1 }; + var current = root; + + string ReadName() + { + if (!compressed) + { + var length = (input[nodeOffset++] & ~0x40) + 1; + var value = Encoding.Latin1.GetString(input, nodeOffset, length); + nodeOffset += length; + return value; + } + int len = input[nodeOffset++]; + var byteLength = (len * 6 + 7) / 8; + var bits = BigInteger.Zero; + for (var index = 0; index < byteLength; index++) + bits = (bits << 8) | input[nodeOffset++]; + var padding = (8 - len * 6 % 8) % 8; + bits >>= padding; + var chars = new char[len]; + for (var index = len - 1; index >= 0; index--) + { + chars[index] = SixBit[(int)(bits & 63)]; + bits >>= 6; + } + return new string(chars); + } + + (double[] Values, int Offset) ReadValues(KFormat format, int count, bool array) + { + int offset; + if (array || format.Size * count > 2) + { + offset = dataOffset; + var result = new double[count]; + for (var index = 0; index < count; index++) + result[index] = ReadNumber(offset + index * format.Size, format); + dataOffset = Align4(offset + count * format.Size); + return (result, offset); + } + if (format.Size == 1) + { + if (byteOffset % 4 == 0) byteOffset = dataOffset; + offset = byteOffset; + byteOffset += count; + } + else + { + if (wordOffset % 4 == 0) wordOffset = dataOffset; + offset = wordOffset; + wordOffset += format.Size * count; + } + var values = new double[count]; + for (var index = 0; index < count; index++) + values[index] = ReadNumber(offset + index * format.Size, format); + var trailing = Math.Max(byteOffset, wordOffset); + if (dataOffset < trailing) dataOffset = Align4(trailing); + return (values, offset); + } + + string ReadString() + { + var size = BinaryPrimitives.ReadInt32BigEndian(input.AsSpan(dataOffset, 4)); + var start = dataOffset + 4; + dataOffset = Align4(start + size); + var length = Math.Max(0, size - 1); + return Encoding.UTF8.GetString(input, start, length).TrimEnd('\0'); + } + + while (nodeOffset < nodeEnd) + { + while (nodeOffset < nodeEnd && input[nodeOffset] == 0) nodeOffset++; + if (nodeOffset >= nodeEnd) break; + int rawType = input[nodeOffset++]; + var array = (rawType & 64) != 0; + var type = rawType & ~64; + if (type == 190) { if (current.Parent is not null) current = current.Parent; continue; } + if (type == 191) break; + var name = ReadName(); + if (type == 46) { current.Attrs[name] = ReadString(); continue; } + if (!Formats.TryGetValue(type, out var format)) + throw new InvalidDataException($"Unsupported binary XML node type {type}"); + var node = new KNode { Name = name, Type = type, Parent = current }; + current.Children.Add(node); + current = node; + if (type == 1) continue; + var count = format.Count; + var isArray = array; + if (count == -1) + { + count = (int)BinaryPrimitives.ReadUInt32BigEndian(input.AsSpan(dataOffset, 4)); + dataOffset += 4; + isArray = true; + } + else if (array) + { + var scale = BinaryPrimitives.ReadUInt32BigEndian(input.AsSpan(dataOffset, 4)); + count = (int)(count * (scale / (double)(format.Size * format.Count))); + dataOffset += 4; + } + var decoded = ReadValues(format, count, isArray); + node.Values = decoded.Values; + node.ValueOffset = decoded.Offset; + } + return root.Children.Count != 1 ? throw new InvalidDataException("Binary XML has no root node") : root.Children[0]; + + double ReadNumber(int offset, KFormat format) + { + if (format.Float) + return format.Size == 4 + ? BinaryPrimitives.ReadSingleBigEndian(input.AsSpan(offset, 4)) + : BinaryPrimitives.ReadDoubleBigEndian(input.AsSpan(offset, 8)); + return format.Size switch + { + 1 => format.Signed ? unchecked((sbyte)input[offset]) : input[offset], + 2 => format.Signed + ? BinaryPrimitives.ReadInt16BigEndian(input.AsSpan(offset, 2)) + : BinaryPrimitives.ReadUInt16BigEndian(input.AsSpan(offset, 2)), + 4 => format.Signed + ? BinaryPrimitives.ReadInt32BigEndian(input.AsSpan(offset, 4)) + : BinaryPrimitives.ReadUInt32BigEndian(input.AsSpan(offset, 4)), + _ => format.Signed + ? BinaryPrimitives.ReadInt64BigEndian(input.AsSpan(offset, 8)) + : BinaryPrimitives.ReadUInt64BigEndian(input.AsSpan(offset, 8)) + }; + } + } +} diff --git a/Abstractions/Extractors/IFS/PixelCodec.cs b/Abstractions/Extractors/IFS/PixelCodec.cs new file mode 100644 index 0000000..081ffc4 --- /dev/null +++ b/Abstractions/Extractors/IFS/PixelCodec.cs @@ -0,0 +1,62 @@ +using System; +using System.Buffers.Binary; + +namespace Abstractions.Extractors.IFS; + +/// Decodes raw IFS texture payloads into RGBA, and re-encodes RGBA back into the writable format. +internal static class PixelCodec +{ + internal static byte[] Decode(string? format, byte[] data, int width, int height) + { + var pixels = width * height; + switch (format) + { + case "argb8888rev": + { + var rgba = new byte[pixels * 4]; + for (var index = 0; index < pixels; index++) + { + var source = index * 4; + rgba[source] = source + 2 < data.Length ? data[source + 2] : (byte)0; + rgba[source + 1] = source + 1 < data.Length ? data[source + 1] : (byte)0; + rgba[source + 2] = source < data.Length ? data[source] : (byte)0; + rgba[source + 3] = source + 3 < data.Length ? data[source + 3] : (byte)0; + } + return rgba; + } + case "argb4444": + { + var rgba = new byte[pixels * 4]; + for (var index = 0; index < pixels; index++) + { + var wordIndex = index * 2; + var word = wordIndex + 1 < data.Length + ? BinaryPrimitives.ReadUInt16BigEndian(data.AsSpan(wordIndex, 2)) + : (ushort)0; + var target = index * 4; + rgba[target] = (byte)((word & 15) * 17); + rgba[target + 1] = (byte)((word >> 8 & 15) * 17); + rgba[target + 2] = (byte)((word >> 12 & 15) * 17); + rgba[target + 3] = (byte)((word >> 4 & 15) * 17); + } + return rgba; + } + case "dxt1": return Dxt.Decode(data, width, height, dxt5: false); + case "dxt5": return Dxt.Decode(data, width, height, dxt5: true); + default: throw new NotSupportedException($"Unsupported IFS texture format {format ?? "undefined"}"); + } + } + + internal static byte[] EncodeArgb8888Rev(byte[] rgba) + { + var output = new byte[rgba.Length]; + for (var offset = 0; offset < rgba.Length; offset += 4) + { + output[offset] = rgba[offset + 2]; + output[offset + 1] = rgba[offset + 1]; + output[offset + 2] = rgba[offset]; + output[offset + 3] = rgba[offset + 3]; + } + return output; + } +} diff --git a/Abstractions/Extractors/IFS/Png.cs b/Abstractions/Extractors/IFS/Png.cs new file mode 100644 index 0000000..5e56c73 --- /dev/null +++ b/Abstractions/Extractors/IFS/Png.cs @@ -0,0 +1,178 @@ +using System; +using System.Buffers.Binary; +using System.IO; +using System.IO.Compression; +using System.Text; + +namespace Abstractions.Extractors.IFS; + +/// Minimal PNG encode/decode for 8-bit RGBA, non-interlaced images (matches what the IFS format stores). +internal static class Png +{ + private static readonly byte[] Signature = [137, 80, 78, 71, 13, 10, 26, 10]; + private static readonly uint[] CrcTable = BuildCrcTable(); + + private static uint[] BuildCrcTable() + { + var table = new uint[256]; + for (uint n = 0; n < 256; n++) + { + var c = n; + for (var k = 0; k < 8; k++) + c = (c & 1) != 0 ? 0xEDB88320 ^ (c >> 1) : c >> 1; + table[n] = c; + } + return table; + } + + private static uint Crc32Update(uint crc, ReadOnlySpan data) + { + foreach (var b in data) + crc = CrcTable[(crc ^ b) & 0xFF] ^ (crc >> 8); + return crc; + } + + private static byte[] Chunk(string name, byte[] data) + { + var type = Encoding.ASCII.GetBytes(name); + using var stream = new MemoryStream(8 + data.Length + 4); + Span lengthBuffer = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32BigEndian(lengthBuffer, (uint)data.Length); + stream.Write(lengthBuffer); + stream.Write(type); + stream.Write(data); + var crc = Crc32Update(Crc32Update(0xFFFFFFFF, type), data) ^ 0xFFFFFFFF; + Span crcBuffer = stackalloc byte[4]; + BinaryPrimitives.WriteUInt32BigEndian(crcBuffer, crc); + stream.Write(crcBuffer); + return stream.ToArray(); + } + + private static byte[] ZlibDeflate(byte[] data) + { + using var output = new MemoryStream(); + using (var zlib = new ZLibStream(output, CompressionLevel.Optimal, leaveOpen: true)) + zlib.Write(data); + return output.ToArray(); + } + + private static byte[] ZlibInflate(byte[] data, int expectedLength) + { + using var input = new MemoryStream(data); + using var zlib = new ZLibStream(input, CompressionMode.Decompress); + using var output = new MemoryStream(expectedLength); + zlib.CopyTo(output); + return output.ToArray(); + } + + internal static byte[] Encode(int width, int height, byte[] rgba) + { + var stride = width * 4; + var scanLines = new byte[height * (stride + 1)]; + for (var y = 0; y < height; y++) + Buffer.BlockCopy(rgba, y * stride, scanLines, y * (stride + 1) + 1, stride); + + var ihdr = new byte[13]; + BinaryPrimitives.WriteUInt32BigEndian(ihdr.AsSpan(0, 4), (uint)width); + BinaryPrimitives.WriteUInt32BigEndian(ihdr.AsSpan(4, 4), (uint)height); + ihdr[8] = 8; // bit depth + ihdr[9] = 6; // color type: RGBA + + using var output = new MemoryStream(); + output.Write(Signature); + output.Write(Chunk("IHDR", ihdr)); + output.Write(Chunk("IDAT", ZlibDeflate(scanLines))); + output.Write(Chunk("IEND", [])); + return output.ToArray(); + } + + internal static (int Width, int Height, byte[] Rgba) Decode(byte[] input, int expectedWidth, int expectedHeight) + { + if (input.Length < 33 || !input.AsSpan(0, 8).SequenceEqual(Signature)) + throw new InvalidDataException("The cropped image is not a valid PNG."); + + int width = 0, height = 0; + var ended = false; + using var idatStream = new MemoryStream(); + var offset = 8; + while (offset + 12 <= input.Length) + { + var size = (int)BinaryPrimitives.ReadUInt32BigEndian(input.AsSpan(offset, 4)); + var type = Encoding.ASCII.GetString(input, offset + 4, 4); + var dataStart = offset + 8; + var dataEnd = dataStart + size; + if (dataEnd + 4 > input.Length) throw new InvalidDataException("The cropped PNG is truncated."); + var chunkData = input.AsSpan(dataStart, size); + var expectedCrc = BinaryPrimitives.ReadUInt32BigEndian(input.AsSpan(dataEnd, 4)); + var crc = Crc32Update(Crc32Update(0xFFFFFFFF, Encoding.ASCII.GetBytes(type)), chunkData) ^ 0xFFFFFFFF; + if (crc != expectedCrc) throw new InvalidDataException("The cropped PNG checksum is invalid."); + + switch (type) + { + case "IHDR" when size != 13 || width != 0 || height != 0: + throw new InvalidDataException("The cropped PNG header is invalid."); + case "IHDR": + { + width = (int)BinaryPrimitives.ReadUInt32BigEndian(chunkData[..4]); + height = (int)BinaryPrimitives.ReadUInt32BigEndian(chunkData.Slice(4, 4)); + if (width != expectedWidth || height != expectedHeight) + throw new InvalidDataException("The cropped image has an invalid pixel size."); + if (chunkData[8] != 8 || chunkData[9] != 6 || chunkData[10] != 0 || chunkData[11] != 0 || chunkData[12] != 0) + throw new InvalidDataException("The cropped PNG must use 8-bit RGBA pixels without interlacing."); + break; + } + case "IDAT" when width == 0 || height == 0: + throw new InvalidDataException("The cropped PNG chunk order is invalid."); + case "IDAT": + idatStream.Write(chunkData); + break; + case "IEND" when size != 0 || dataEnd + 4 != input.Length: + throw new InvalidDataException("The cropped PNG ending is invalid."); + case "IEND": + ended = true; + break; + } + offset = dataEnd + 4; + if (type == "IEND") break; + } + if (!ended || width <= 0 || height <= 0 || idatStream.Length == 0) + throw new InvalidDataException("The cropped PNG has no image data."); + + var stride = width * 4; + var expectedSize = height * (stride + 1); + var filtered = ZlibInflate(idatStream.ToArray(), expectedSize); + if (filtered.Length != expectedSize) throw new InvalidDataException("The cropped PNG pixel data is invalid."); + + var rgba = new byte[width * height * 4]; + for (var y = 0; y < height; y++) + { + var filter = filtered[y * (stride + 1)]; + var source = y * (stride + 1) + 1; + var target = y * stride; + for (var x = 0; x < stride; x++) + { + var value = filtered[source + x]; + var left = x >= 4 ? rgba[target + x - 4] : (byte)0; + var above = y > 0 ? rgba[target + x - stride] : (byte)0; + var upperLeft = y > 0 && x >= 4 ? rgba[target + x - stride - 4] : (byte)0; + rgba[target + x] = filter switch + { + 0 => value, + 1 => (byte)(value + left), + 2 => (byte)(value + above), + 3 => (byte)(value + (left + above) / 2), + 4 => (byte)(value + Paeth(left, above, upperLeft)), + _ => throw new InvalidDataException($"Unsupported PNG row filter {filter}."), + }; + } + } + return (width, height, rgba); + } + + private static byte Paeth(byte a, byte b, byte c) + { + var estimate = a + b - c; + int da = Math.Abs(estimate - a), db = Math.Abs(estimate - b), dc = Math.Abs(estimate - c); + return da <= db && da <= dc ? a : db <= dc ? b : c; + } +}