diff --git a/README.md b/README.md index 05265ce..60befb2 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,12 @@ SEGA arcade image toolkit ## features -- APP/OPT/APM3 decryption -- NTFS/exFAT support -- VHD support (fixed, dynamic, differencing) -- stream directly from encrypted image (no temp files) +- support for APP/PACK and OPT (including APM3 type) +- support for NTFS and exFAT +- support for VHD (fixed, dynamic, differencing) +- stream directly from encrypted image (no temporary files) - preserved timestamps -- AES-NI accelerated with software fallback +- AES-NI hardware accelerated ## build @@ -31,11 +31,24 @@ flags: - `-n` decrypt only, skip extraction - `-w` write intermediate .ntfs/.exfat files - `-p file` parent for differencing VHD +- `-k` keep all versions (each delta gets its own folder instead of stacking) - `-s` silent - `-v` verbose - `-vn` version -drag and drop works on windows +### deltas (differencing VHD) + +base + update files are stacked into a single output directory named after the latest delta. files are automatically sorted by actual order. + +``` +unsegareborn -o out -p BASE.opt/app DELTA1.opt/app DELTA2.opt/app +``` + +result: `out/DELTA2_*/` with merged content (base + all updates applied in order) + +### drag and drop + +drag and drop works on windows. multiple files are handled automatically. the earliest file on drag selection order is used as parent, the rest are applied as updates in order. no flags needed. ## keys @@ -47,9 +60,11 @@ to build from source: format: ```c -{"SDEZ", {0xd1,0x36,...}, {0xc4,0x84,...}, true}, +{"SDEZ", {0xd1,0x36,...}}, ``` +iv values are derived automatically from the key and ciphertext + ## platforms | platform | method | @@ -62,10 +77,11 @@ format: ## release -push version tag: +build generates a version tag automatically (`YYYYMMDDNN`, revision increments per day): ``` -git tag 2026020501 -git push origin 2026020501 +sh build.cmd # or build.cmd on windows +git tag $(./build/unsegareborn-* -vn | awk '{print $2}') +git push origin --tags ``` ci builds both platforms and creates a github release with binaries diff --git a/build.cmd b/build.cmd index 15dfa66..d7af4e0 100644 --- a/build.cmd +++ b/build.cmd @@ -5,6 +5,17 @@ goto :WINDOWS #!/bin/sh set -e mkdir -p build +DATESTR=$(date +%Y%m%d) +REV=-1 +TAG_REV=$(git tag -l "${DATESTR}*" 2>/dev/null | sed "s/^${DATESTR}//" | sort -rn | head -1) +[ -n "$TAG_REV" ] && REV=$((10#$TAG_REV)) +if [ -f build/.buildrev ]; then + read LAST_DATE LAST_REV < build/.buildrev + [ "$LAST_DATE" = "$DATESTR" ] && [ "$LAST_REV" -gt "$REV" ] && REV=$LAST_REV +fi +REV=$((REV + 1)) +DATECODE=$(printf "%s%02d" "$DATESTR" "$REV") +echo "$DATESTR $REV" > build/.buildrev SRC="src/lib.c src/main.c src/crypto.c src/keys.c src/exfat.c src/ntfs.c src/stream.c src/aes.c" @@ -28,7 +39,7 @@ case "$(uname -s)" in esac echo "building..." -clang $CFLAGS $LDFLAGS -o $OUT $SRC $LIBS +clang $CFLAGS $LDFLAGS -DVERSION="\"$DATECODE\"" -o $OUT $SRC $LIBS echo "done: $OUT ($(stat -c%s "$OUT" 2>/dev/null || stat -f%z "$OUT") bytes)" exit 0 @@ -64,8 +75,30 @@ for %%L in ( set CFLAGS=-target x86_64-pc-windows-gnu -Oz -maes -msse4.1 -I include -fno-asynchronous-unwind-tables -fno-ident -ffunction-sections -fdata-sections -flto -ffreestanding -fno-builtin -fno-stack-protector -nostdlib -fno-unwind-tables -fno-exceptions -fmerge-all-constants -fno-addrsig set LDFLAGS=-fuse-ld=lld -Wl,--gc-sections -Wl,--icf=all -Wl,-e,_start -Wl,--subsystem,console -Wl,-s -Wl,--lto-Oz %LIBPATH% +for /f %%a in ('wmic os get localdatetime /value ^| find "="') do for /f "tokens=2 delims==" %%b in ("%%a") do set "DT=%%b" +set "DATESTR=%DT:~0,8%" +set REV=-1 +set "LASTTAG=" +for /f %%r in ('git tag -l "%DATESTR%*" 2^>nul ^| sort') do set "LASTTAG=%%r" +if defined LASTTAG call :parsetag +if exist build\.buildrev ( + for /f "tokens=1,2" %%x in (build\.buildrev) do ( + if "%%x"=="%DATESTR%" if %%y GTR %REV% set REV=%%y + ) +) +set /a REV+=1 +set "REVSTR=0%REV%" +set "DATECODE=%DATESTR%%REVSTR:~-2%" +(echo %DATESTR% %REV%)>build\.buildrev +goto :buildstart + +:parsetag +set /a "REV=1%LASTTAG:~8% - 100" +goto :eof + +:buildstart echo building... -clang %CFLAGS% %LDFLAGS% -o build\unsegareborn-win-x64.exe %SRC% -lntdll +clang %CFLAGS% %LDFLAGS% -DVERSION="\"%DATECODE%\"" -o build\unsegareborn-win-x64.exe %SRC% -lntdll if exist build\unsegareborn-win-x64.exe ( for %%F in (build\unsegareborn-win-x64.exe) do echo done: %%~nxF ^(%%~zF bytes^) diff --git a/include/common.h b/include/common.h index 138598f..bf65af5 100644 --- a/include/common.h +++ b/include/common.h @@ -9,25 +9,7 @@ #ifdef PLATFORM_WINDOWS static inline int utf8_to_wide(const char* utf8, WCHAR* wide, int wide_len) { - if (!utf8 || !wide || wide_len <= 0) return 0; - int out = 0; - const unsigned char* s = (const unsigned char*)utf8; - while (*s && out < wide_len - 1) { - uint32_t cp; - if (s[0] < 0x80) { cp = s[0]; s += 1; } - else if ((s[0] & 0xE0) == 0xC0 && (s[1] & 0xC0) == 0x80) { cp = ((s[0] & 0x1F) << 6) | (s[1] & 0x3F); s += 2; } - else if ((s[0] & 0xF0) == 0xE0 && (s[1] & 0xC0) == 0x80 && (s[2] & 0xC0) == 0x80) { cp = ((s[0] & 0x0F) << 12) | ((s[1] & 0x3F) << 6) | (s[2] & 0x3F); s += 3; } - else if ((s[0] & 0xF8) == 0xF0 && (s[1] & 0xC0) == 0x80 && (s[2] & 0xC0) == 0x80 && (s[3] & 0xC0) == 0x80) { cp = ((s[0] & 0x07) << 18) | ((s[1] & 0x3F) << 12) | ((s[2] & 0x3F) << 6) | (s[3] & 0x3F); s += 4; } - else { cp = '?'; s += 1; } - if (cp <= 0xFFFF) wide[out++] = (WCHAR)cp; - else if (cp <= 0x10FFFF && out < wide_len - 2) { cp -= 0x10000; wide[out++] = (WCHAR)(0xD800 | (cp >> 10)); wide[out++] = (WCHAR)(0xDC00 | (cp & 0x3FF)); } - } - wide[out] = 0; - return out; -} - -static inline FILE* fopen_utf8(const char* path, const char* mode) { - return fopen(path, mode); + return (int)utf8_to_utf16(utf8, wide, (size_t)wide_len); } static inline FILE* fopen_prealloc_utf8(const char* path, uint64_t size) { @@ -48,7 +30,26 @@ static inline int remove_utf8(const char* path) { return _wremove(wpath); } -#define FOPEN fopen_utf8 +typedef HANDLE DirHandle; +#define INVALID_DIR_HANDLE INVALID_HANDLE_VALUE + +static inline DirHandle open_output_dir(const char* path) { + WCHAR wpath[1024]; + if (!utf8_to_wide(path, wpath, 1024)) return INVALID_DIR_HANDLE; + return lib_open_dir_handle(wpath); +} + +static inline FILE* fopen_in_dir(DirHandle dir, const char* filename, uint64_t prealloc_size) { + WCHAR wname[256]; + if (!utf8_to_wide(filename, wname, 256)) return NULL; + return lib_fopen_relative(dir, wname, prealloc_size); +} + +static inline void close_output_dir(DirHandle dir) { + if (dir != INVALID_DIR_HANDLE) NtClose(dir); +} + +#define FOPEN fopen #define FOPEN_PREALLOC fopen_prealloc_utf8 #define FWRITE_DIRECT lib_fwrite_direct #define MKDIR(path) mkdir_utf8(path) @@ -58,21 +59,13 @@ static inline int remove_utf8(const char* path) { static inline bool set_file_times(const char* path, uint64_t modified_time, uint64_t access_time) { WCHAR wpath[1024]; if (!utf8_to_wide(path, wpath, 1024)) return false; - - struct { int64_t actime; int64_t modtime; } times; - times.modtime = (int64_t)((modified_time / 10000000ULL) - 11644473600ULL); - times.actime = (int64_t)((access_time / 10000000ULL) - 11644473600ULL); - return _wutime(wpath, ×) == 0; + return lib_wutime(wpath, (int64_t)modified_time, (int64_t)access_time) == 0; } static inline bool set_dir_times(const char* path, uint64_t modified_time, uint64_t access_time) { WCHAR wpath[1024]; if (!utf8_to_wide(path, wpath, 1024)) return false; - - struct { int64_t actime; int64_t modtime; } times; - times.modtime = (int64_t)((modified_time / 10000000ULL) - 11644473600ULL); - times.actime = (int64_t)((access_time / 10000000ULL) - 11644473600ULL); - return lib_wutime_dir(wpath, ×) == 0; + return lib_wutime_dir(wpath, (int64_t)modified_time, (int64_t)access_time) == 0; } static inline bool set_file_times_handle(FILE* f, uint64_t modified_time, uint64_t access_time) { @@ -81,6 +74,14 @@ static inline bool set_file_times_handle(FILE* f, uint64_t modified_time, uint64 #else +typedef int DirHandle; +#define INVALID_DIR_HANDLE (-1) +static inline DirHandle open_output_dir(const char* path) { (void)path; return -1; } +static inline FILE* fopen_in_dir(DirHandle dir, const char* filename, uint64_t prealloc_size) { + (void)dir; (void)filename; (void)prealloc_size; return NULL; +} +static inline void close_output_dir(DirHandle dir) { (void)dir; } + #define FOPEN fopen #define FOPEN_PREALLOC(path, size) fopen(path, "wb") #define FWRITE_DIRECT(f, buf, size) fwrite(buf, 1, size, f) @@ -118,9 +119,8 @@ static inline uint64_t exfat_timestamp_to_ntfs(uint32_t exfat_ts, uint8_t centis uint32_t month = (exfat_ts >> 21) & 0x0F; uint32_t year = ((exfat_ts >> 25) & 0x7F) + 1980; - uint64_t days = 0; - for (uint32_t y = 1601; y < year; y++) - days += (y % 4 == 0 && (y % 100 != 0 || y % 400 == 0)) ? 366 : 365; + uint32_t py = year - 1; + uint64_t days = (uint64_t)py * 365 + py / 4 - py / 100 + py / 400 - 584388ULL; static const uint16_t month_days[] = {0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334}; if (month >= 1 && month <= 12) { days += month_days[month - 1]; @@ -148,23 +148,31 @@ static inline uint64_t exfat_timestamp_to_ntfs(uint32_t exfat_ts, uint8_t centis #endif static inline size_t utf16_to_utf8_common(const uint16_t* utf16, int utf16_len, char* utf8, size_t utf8_size) { - size_t out_pos = 0; - for (int i = 0; i < utf16_len && out_pos < utf8_size - 1; i++) { - uint16_t c = utf16[i]; - if (c < 0x80) utf8[out_pos++] = (char)c; - else if (c < 0x800) { - if (out_pos + 2 > utf8_size - 1) break; - utf8[out_pos++] = (char)(0xC0 | (c >> 6)); - utf8[out_pos++] = (char)(0x80 | (c & 0x3F)); + size_t out = 0; + for (int i = 0; i < utf16_len && out < utf8_size - 1; i++) { + uint32_t cp = utf16[i]; + if (cp >= 0xD800 && cp <= 0xDBFF && i + 1 < utf16_len && utf16[i+1] >= 0xDC00 && utf16[i+1] <= 0xDFFF) + cp = 0x10000 + ((cp - 0xD800) << 10) + (utf16[++i] - 0xDC00); + if (cp < 0x80) utf8[out++] = (char)cp; + else if (cp < 0x800) { + if (out + 2 > utf8_size - 1) break; + utf8[out++] = (char)(0xC0 | (cp >> 6)); + utf8[out++] = (char)(0x80 | (cp & 0x3F)); + } else if (cp < 0x10000) { + if (out + 3 > utf8_size - 1) break; + utf8[out++] = (char)(0xE0 | (cp >> 12)); + utf8[out++] = (char)(0x80 | ((cp >> 6) & 0x3F)); + utf8[out++] = (char)(0x80 | (cp & 0x3F)); } else { - if (out_pos + 3 > utf8_size - 1) break; - utf8[out_pos++] = (char)(0xE0 | (c >> 12)); - utf8[out_pos++] = (char)(0x80 | ((c >> 6) & 0x3F)); - utf8[out_pos++] = (char)(0x80 | (c & 0x3F)); + if (out + 4 > utf8_size - 1) break; + utf8[out++] = (char)(0xF0 | (cp >> 18)); + utf8[out++] = (char)(0x80 | ((cp >> 12) & 0x3F)); + utf8[out++] = (char)(0x80 | ((cp >> 6) & 0x3F)); + utf8[out++] = (char)(0x80 | (cp & 0x3F)); } } - utf8[out_pos] = '\0'; - return out_pos; + utf8[out] = '\0'; + return out; } #define utf16_to_utf8 utf16_to_utf8_common @@ -251,4 +259,11 @@ static inline bool create_directories(const char* path) { return success; } +static inline const char* get_basename(const char* path) { + const char* b = strrchr(path, '/'); + const char* c = strrchr(path, '\\'); + if (c && (!b || c > b)) b = c; + return b ? b + 1 : path; +} + #endif diff --git a/include/crypto.h b/include/crypto.h index 05dbe7a..96d11d3 100644 --- a/include/crypto.h +++ b/include/crypto.h @@ -6,29 +6,24 @@ extern const uint8_t NTFS_HEADER[16]; extern const uint8_t EXFAT_HEADER[16]; extern const uint8_t OPTION_KEY[16]; -extern const uint8_t OPTION_IV[16]; extern const uint8_t APM3_SEED[96]; extern const uint8_t APM3_KEY[16]; extern const uint8_t APM3_IV[16]; typedef struct { uint8_t key[16]; - uint8_t iv[16]; - bool has_iv; bool external; } GameKeys; typedef struct { const char* game_id; uint8_t key[16]; - uint8_t iv[16]; - bool has_iv; } GameKeyEntry; -bool key_lookup(const char* id, uint8_t key[16], uint8_t iv[16], bool* from_external); +bool key_lookup(const char* id, uint8_t key[16], bool* from_external); bool key_any(void); void iv_page(uint64_t off, const uint8_t* base, uint8_t* out); -bool iv_file(const uint8_t key[16], const uint8_t* hdr, const uint8_t* page, uint8_t out[16]); +void iv_file(const uint8_t key[16], const uint8_t* hdr, const uint8_t* page, uint8_t out[16]); bool key_game(const char* id, GameKeys* out); bool key_derive(const char* id, uint8_t key[16], uint8_t iv[16]); diff --git a/include/error.h b/include/error.h index 3c6ccc6..381a523 100644 --- a/include/error.h +++ b/include/error.h @@ -13,22 +13,14 @@ typedef enum { ERR_INVALID_BOOTID, ERR_UNKNOWN_CONTAINER, ERR_KEY_NOT_FOUND, - ERR_INVALID_KEY_FILE, ERR_AES_ALIGNMENT, - ERR_IV_CALCULATION, - ERR_INVALID_NTFS, - ERR_INVALID_EXFAT, - ERR_INVALID_VHD, - ERR_MFT_CORRUPT, - ERR_PATH_UNSAFE, - ERR_DIR_CREATE, ERR_EXTRACTION_FAILED } ErrorCode; static inline const char* error_string(ErrorCode code) { static const char* const errs[] = { "ok", "mem", "open", "read", "write", "seek", "bootid", "container", - "key", "keyfile", "align", "iv", "ntfs", "exfat", "vhd", "mft", "path", "mkdir", "extract" + "key", "align", "extract" }; return (code < sizeof(errs)/sizeof(errs[0])) ? errs[code] : "?"; } diff --git a/include/exfat.h b/include/exfat.h index 65b9ff2..b75ac98 100644 --- a/include/exfat.h +++ b/include/exfat.h @@ -8,10 +8,8 @@ #define EXFAT_ENTRY_SIZE 32 #define EXFAT_ENTRY_EOD 0x00 -#define EXFAT_ENTRY_BITMAP 0x81 #define EXFAT_ENTRY_FILE 0x85 #define EXFAT_ENTRY_STREAM 0xC0 -#define EXFAT_ENTRY_FILENAME 0xC1 #pragma pack(push, 1) @@ -96,6 +94,15 @@ typedef struct { uint64_t atime; } DeferredDirTime; +static inline bool grow_deferred_dirs(DeferredDirTime** dirs, uint32_t* capacity) { + uint32_t new_cap = *capacity ? *capacity * 2 : 256; + DeferredDirTime* new_buf = realloc(*dirs, new_cap * sizeof(DeferredDirTime)); + if (!new_buf) return false; + *dirs = new_buf; + *capacity = new_cap; + return true; +} + typedef struct { FILE* fp; ExfatBootSector boot_sector; @@ -107,14 +114,14 @@ typedef struct { uint32_t* fat; uint8_t* cluster_buf; uint8_t* io_buf; - uint64_t total_bytes; uint64_t extracted_bytes; uint64_t files_extracted; - void* progress; + uint64_t raw_file_pos; bool silent; bool verbose; DecryptStream* stream; char last_dir[MAX_PATH_LENGTH]; + DirHandle cached_dir; DeferredDirTime* deferred_dirs; uint32_t deferred_count; uint32_t deferred_capacity; diff --git a/include/keys.h b/include/keys.h index 5acfb8b..2cece83 100644 --- a/include/keys.h +++ b/include/keys.h @@ -2,5 +2,5 @@ static const GameKeyEntry embedded_keys[] = { #if __has_include("keys.inc") #include "keys.inc" #endif -{NULL, {0}, {0}, false} +{NULL, {0}} }; diff --git a/include/keys.inc.example b/include/keys.inc.example index 7b3a49c..837ff5e 100644 --- a/include/keys.inc.example +++ b/include/keys.inc.example @@ -1,3 +1,3 @@ // copy to keys.inc and add your keys -// format: {"SXXX", {key[16]}, {iv[16]}, has_iv}, -// {"XXXX", {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}, {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}, true}, +// format: {"SXXX", {key[16]}}, +// {"SXXX", {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}}, diff --git a/include/lib.h b/include/lib.h index ec68f35..954ce3e 100644 --- a/include/lib.h +++ b/include/lib.h @@ -32,8 +32,6 @@ typedef long intptr_t; typedef int64_t off_t; typedef int64_t time_t; -typedef int64_t __time64_t; - #ifndef __bool_true_false_are_defined #if __STDC_VERSION__ < 202311L typedef _Bool bool; @@ -54,24 +52,10 @@ typedef __builtin_va_list va_list; extern "C" { #endif -#define EPERM 1 #define ENOENT 2 -#define ESRCH 3 -#define EINTR 4 -#define EIO 5 -#define ENXIO 6 -#define EBADF 9 -#define ENOMEM 12 #define EACCES 13 -#define EFAULT 14 #define EEXIST 17 -#define ENOTDIR 20 -#define EISDIR 21 #define EINVAL 22 -#define EMFILE 24 -#define ENOSPC 28 -#define EROFS 30 -#define ERANGE 34 #ifdef PLATFORM_WINDOWS @@ -93,9 +77,11 @@ typedef int64_t LONGLONG; #define FILE_READ_DATA 0x0001 #define FILE_WRITE_DATA 0x0002 +#define FILE_ADD_FILE 0x0002 #define FILE_APPEND_DATA 0x0004 #define FILE_READ_ATTRIBUTES 0x0080 #define FILE_WRITE_ATTRIBUTES 0x0100 +#define FILE_TRAVERSE 0x0020 #define FILE_LIST_DIRECTORY 0x0001 #define DELETE 0x00010000L #define SYNCHRONIZE 0x00100000L @@ -182,10 +168,11 @@ typedef struct _FILE_BOTH_DIR_INFORMATION { } FILE_BOTH_DIR_INFORMATION; typedef enum _FILE_INFORMATION_CLASS { + FileBothDirectoryInformation = 3, FileBasicInformation = 4, FileStandardInformation = 5, + FileRenameInformation = 10, FilePositionInformation = 14, - FileBothDirectoryInformation = 3, } FILE_INFORMATION_CLASS; typedef struct _RTL_USER_PROCESS_PARAMETERS { @@ -238,7 +225,6 @@ __declspec(dllimport) NTSTATUS __stdcall NtQueryDirectoryFile(HANDLE, HANDLE, vo __declspec(dllimport) NTSTATUS __stdcall NtQuerySystemTime(LARGE_INTEGER*); __declspec(dllimport) NTSTATUS __stdcall NtQueryInformationProcess(HANDLE, ULONG, void*, ULONG, ULONG*); __declspec(dllimport) NTSTATUS __stdcall NtQuerySystemInformation(ULONG, void*, ULONG, ULONG*); -__declspec(dllimport) NTSTATUS __stdcall NtDelayExecution(uint8_t, LARGE_INTEGER*); __declspec(dllimport) void* __stdcall RtlAllocateHeap(HANDLE, ULONG, size_t); __declspec(dllimport) void* __stdcall RtlReAllocateHeap(HANDLE, ULONG, void*, size_t); __declspec(dllimport) uint8_t __stdcall RtlFreeHeap(HANDLE, ULONG, void*); @@ -258,62 +244,37 @@ static inline PEB* lib_get_peb(void) { #define SYS_write 1 #define SYS_open 2 #define SYS_close 3 -#define SYS_fstat 5 #define SYS_lseek 8 #define SYS_mmap 9 -#define SYS_mprotect 10 #define SYS_munmap 11 -#define SYS_brk 12 -#define SYS_ioctl 16 -#define SYS_access 21 -#define SYS_dup2 33 -#define SYS_getpid 39 -#define SYS_exit 60 -#define SYS_uname 63 -#define SYS_fcntl 72 -#define SYS_fsync 74 -#define SYS_ftruncate 77 #define SYS_getdents64 217 -#define SYS_fadvise64 221 #define SYS_exit_group 231 -#define SYS_openat 257 #define SYS_mkdirat 258 #define SYS_newfstatat 262 #define SYS_unlinkat 263 +#define SYS_renameat 264 #define SYS_utimensat 280 #define O_RDONLY 0x0000 #define O_WRONLY 0x0001 #define O_RDWR 0x0002 #define O_CREAT 0x0040 -#define O_EXCL 0x0080 #define O_TRUNC 0x0200 #define O_APPEND 0x0400 #define O_DIRECTORY 0x10000 -#define O_CLOEXEC 0x80000 #define S_IRWXU 0700 #define S_IRUSR 0400 #define S_IWUSR 0200 -#define S_IXUSR 0100 -#define S_IRWXG 0070 #define S_IRGRP 0040 #define S_IXGRP 0010 -#define S_IRWXO 0007 #define S_IROTH 0004 #define S_IXOTH 0001 -#define S_IFMT 0170000 -#define S_IFDIR 0040000 -#define S_IFREG 0100000 -#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) -#define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) - #define PROT_READ 0x1 #define PROT_WRITE 0x2 #define MAP_PRIVATE 0x02 #define MAP_ANONYMOUS 0x20 -#define MAP_FAILED ((void*)-1) #define SEEK_SET 0 #define SEEK_CUR 1 @@ -322,7 +283,6 @@ static inline PEB* lib_get_peb(void) { #define AT_FDCWD (-100) #define DT_DIR 4 -#define DT_REG 8 struct linux_dirent64 { uint64_t d_ino; @@ -358,12 +318,6 @@ struct linux_timespec { int64_t tv_nsec; }; -static inline long syscall0(long n) { - long ret; - __asm__ volatile ("syscall" : "=a"(ret) : "a"(n) : "rcx", "r11", "memory"); - return ret; -} - static inline long syscall1(long n, long a1) { long ret; __asm__ volatile ("syscall" : "=a"(ret) : "a"(n), "D"(a1) : "rcx", "r11", "memory"); @@ -389,14 +343,6 @@ static inline long syscall4(long n, long a1, long a2, long a3, long a4) { return ret; } -static inline long syscall5(long n, long a1, long a2, long a3, long a4, long a5) { - long ret; - register long r10 __asm__("r10") = a4; - register long r8 __asm__("r8") = a5; - __asm__ volatile ("syscall" : "=a"(ret) : "a"(n), "D"(a1), "S"(a2), "d"(a3), "r"(r10), "r"(r8) : "rcx", "r11", "memory"); - return ret; -} - static inline long syscall6(long n, long a1, long a2, long a3, long a4, long a5, long a6) { long ret; register long r10 __asm__("r10") = a4; @@ -408,10 +354,6 @@ static inline long syscall6(long n, long a1, long a2, long a3, long a4, long a5, #endif -#define _IOFBF 0 -#define _IOLBF 1 -#define _IONBF 2 - #define _A_SUBDIR 0x10 #ifndef SEEK_SET @@ -501,24 +443,19 @@ extern FILE* lib_stdin_file; FILE* lib_fopen(const char* path, const char* mode); size_t lib_fread(void* buf, size_t size, size_t count, FILE* f); size_t lib_fwrite(const void* buf, size_t size, size_t count, FILE* f); -int lib_fseek(FILE* f, long offset, int whence); int lib_fseeki64(FILE* f, int64_t offset, int whence); -long lib_ftell(FILE* f); int64_t lib_ftelli64(FILE* f); int lib_fclose(FILE* f); int lib_fflush(FILE* f); int lib_feof(FILE* f); char* lib_fgets(char* buf, int n, FILE* f); void lib_rewind(FILE* f); -void lib_setvbuf(FILE* f, char* buf, int mode, size_t size); #define fopen lib_fopen #define fread lib_fread #define fwrite lib_fwrite -#define fseek lib_fseek #define _fseeki64 lib_fseeki64 #define fseeko lib_fseeki64 -#define ftell lib_ftell #define _ftelli64 lib_ftelli64 #define ftello lib_ftelli64 #define fclose lib_fclose @@ -526,19 +463,19 @@ void lib_setvbuf(FILE* f, char* buf, int mode, size_t size); #define feof lib_feof #define fgets lib_fgets #define rewind lib_rewind -#define setvbuf lib_setvbuf #ifdef PLATFORM_WINDOWS +size_t utf8_to_utf16(const char* utf8, WCHAR* utf16, size_t utf16_max); FILE* lib_wfopen(const WCHAR* path, const WCHAR* mode); FILE* lib_wfopen_prealloc(const WCHAR* path, uint64_t size); +HANDLE lib_open_dir_handle(const WCHAR* path); +FILE* lib_fopen_relative(HANDLE dir_handle, const WCHAR* filename, uint64_t prealloc_size); bool lib_set_file_times_ntfs(FILE* f, int64_t modified_time, int64_t access_time); -int lib_wutime(const WCHAR* path, const void* times); -int lib_wutime_dir(const WCHAR* path, const void* times); +int lib_wutime(const WCHAR* path, int64_t modified_time, int64_t access_time); +int lib_wutime_dir(const WCHAR* path, int64_t modified_time, int64_t access_time); #define _wfopen lib_wfopen -#define _wutime lib_wutime -#endif - size_t lib_fwrite_direct(FILE* f, const void* buf, size_t size); +#endif int lib_printf(const char* fmt, ...); int lib_fprintf(FILE* f, const char* fmt, ...); @@ -559,12 +496,13 @@ int lib_puts(const char* s); int lib_mkdir(const char* path); int lib_rmdir(const char* path); int lib_remove(const char* path); +int lib_rename(const char* old_path, const char* new_path); #define mkdir lib_mkdir -#define _mkdir lib_mkdir #define rmdir lib_rmdir #define _rmdir lib_rmdir #define remove lib_remove +#define rename lib_rename #ifdef PLATFORM_WINDOWS int lib_wmkdir(const WCHAR* path); @@ -587,22 +525,14 @@ int lib_findnext(intptr_t handle, lib_finddata_t* data); int lib_findclose(intptr_t handle); #define _finddata_t lib_finddata_t -#define _finddata64_t lib_finddata_t #define _findfirst lib_findfirst -#define _findfirst64 lib_findfirst #define _findnext lib_findnext -#define _findnext64 lib_findnext #define _findclose lib_findclose -int lib_isatty(int fd); -#define isatty lib_isatty -#define _isatty lib_isatty - time_t lib_time(time_t* t); double lib_difftime(time_t t1, time_t t0); #define time lib_time -#define _time64 lib_time #define difftime lib_difftime int lib_atoi(const char* s); @@ -618,7 +548,6 @@ int* lib_errno_func(void); void lib_exit(int status); #define exit lib_exit -#define _exit lib_exit #ifdef PLATFORM_WINDOWS #define PATH_SEPARATOR "\\" diff --git a/include/ntfs.h b/include/ntfs.h index 73fd22e..7a75097 100644 --- a/include/ntfs.h +++ b/include/ntfs.h @@ -8,8 +8,6 @@ #define VHD_FOOTER_SIZE 512 #define VHD_SECTOR_SIZE 512 #define VHD_BAT_ENTRY_RESERVED 0xFFFFFFFF -#define NTFS_RECORD_SIZE 1024 -#define MFT_RECORD_MAGIC "FILE" #define VHD_COOKIE "conectix" #define VHD_DYNAMIC_COOKIE "cxsparse" #define VHD_TYPE_FIXED 2 @@ -18,8 +16,6 @@ #define VHD_MAX_CHAIN_DEPTH 8 #define FILE_NAME_ATTR 0x30 #define DATA_ATTR 0x80 -#define INDEX_ROOT_ATTR 0x90 -#define INDEX_ALLOCATION_ATTR 0xA0 #define NTFS_SIGNATURE "NTFS " #define NTFS_PARTITION_TYPE 0x07 #define MFT_RECORD_IN_USE 0x0001 @@ -197,7 +193,6 @@ typedef struct VHDContext { struct VHDContext* parent; uint32_t depth; char base_dir[MAX_PATH_LENGTH]; - uint64_t file_pos; } VHDContext; typedef struct { @@ -225,7 +220,6 @@ typedef struct { DataRun runs[MAX_DATA_RUNS]; int run_count; uint64_t file_size; - uint64_t data_offset; char filename[MAX_FILENAME_LENGTH]; } PendingOpt; @@ -247,10 +241,9 @@ typedef struct { uint64_t data_start_offset; bool silent; bool verbose; - uint64_t total_bytes; uint64_t extracted_bytes; uint64_t files_extracted; - void* progress; + int highest_extracted_vhd; uint64_t raw_file_pos; DecryptStream* stream; PendingVHD pending_vhds[MAX_PENDING_VHDS]; @@ -258,23 +251,32 @@ typedef struct { PendingOpt pending_opts[MAX_PENDING_OPTS]; int pending_opt_count; uint8_t* lookup_buffer; - bool apm3_decrypt; - uint8_t apm3_key[16]; - uint8_t apm3_iv[16]; uint8_t* file_buffer; char last_dir[MAX_PATH_LENGTH]; + DirHandle cached_dir; void* deferred_dirs; uint32_t deferred_count; uint32_t deferred_capacity; + uint64_t skip_refs[8]; + int skip_ref_count; } NTFSContext; bool ntfs_init(NTFSContext* ctx, const char* vhd_path, const char* extract_path); bool ntfs_init_stream(NTFSContext* ctx, DecryptStream* stream, const char* extract_path); -int ntfs_detect_vhd_type(NTFSContext* ctx); bool ntfs_extract_all(NTFSContext* ctx); -bool ntfs_extract_pending_vhds(NTFSContext* ctx, bool silent, bool verbose, bool* is_orphan); -int ntfs_get_pending_opt_count(NTFSContext* ctx); +bool ntfs_extract_pending_vhds(NTFSContext* ctx, bool silent, bool verbose, + const char* parent_file, VHDContext* parent_vhd, + bool* is_orphan, VHDContext** out_base_vhd); +bool vhd_extract_ntfs(VHDContext* vhd, const char* base_path, + bool silent, bool verbose, + uint64_t* out_files, uint64_t* out_bytes); +bool vhd_init_internal(VHDContext* ctx, const char* filename, uint32_t depth); +bool ntfs_init_vhd(NTFSContext* ctx, const char* vhd_path, const char* extract_path, + VHDContext* external_parent); const PendingOpt* ntfs_get_pending_opt(NTFSContext* ctx, int index); +bool ntfs_read_from_runs(NTFSContext* ctx, const DataRun* runs, int run_count, + uint64_t file_size, uint64_t read_offset, void* buffer, size_t read_size); void ntfs_close(NTFSContext* ctx); +void vhd_close(VHDContext* ctx); #endif \ No newline at end of file diff --git a/include/progress.h b/include/progress.h deleted file mode 100644 index ee750f2..0000000 --- a/include/progress.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef PROGRESS_H -#define PROGRESS_H - -#include "lib.h" - -typedef struct { uint64_t total; } Progress; - -static inline void progress_init(Progress* p, uint64_t total) { p->total = total; } -static inline void progress_update(Progress* p, uint64_t current) { (void)p; (void)current; } -static inline void progress_finish(Progress* p) { (void)p; } - -#endif diff --git a/include/stream.h b/include/stream.h index c3305a4..dcd21e1 100644 --- a/include/stream.h +++ b/include/stream.h @@ -5,6 +5,7 @@ #include "aes.h" #define DECRYPT_PAGE_SIZE 4096 +#define STREAM_CACHE_PAGES 16 #define MAX_DATA_RUNS 256 typedef struct { @@ -23,14 +24,16 @@ typedef struct DecryptStream DecryptStream; struct DecryptStream { FILE* fp; - DecryptStream* parent_stream; RunSource* run_source; uint64_t data_offset; uint64_t data_size; uint8_t key[16]; uint8_t file_iv[16]; - uint8_t page_buffer[DECRYPT_PAGE_SIZE]; - uint64_t cached_page_offset; + uint8_t page_buffers[STREAM_CACHE_PAGES][DECRYPT_PAGE_SIZE] __attribute__((aligned(16))); + uint64_t cached_page_offsets[STREAM_CACHE_PAGES]; + uint32_t page_lru[STREAM_CACHE_PAGES]; + uint64_t lru_counter; + int last_slot; uint64_t file_pos; AES_ctx aes_ctx; }; @@ -40,7 +43,5 @@ bool stream_init(DecryptStream* ds, FILE* fp, uint64_t data_offset, bool stream_init_from_runs(DecryptStream* ds, RunSource* source, const uint8_t key[16], const uint8_t iv[16]); bool stream_read(DecryptStream* ds, void* buffer, uint64_t offset, size_t size); -bool stream_read_raw(void* ntfs_ctx, const DataRun* runs, int run_count, - uint64_t file_size, uint64_t offset, void* buffer, size_t size); #endif diff --git a/src/aes.c b/src/aes.c index cfeed44..fd3168d 100644 --- a/src/aes.c +++ b/src/aes.c @@ -1,38 +1,5 @@ #include "aes.h" -const uint8_t aes_sbox[256] = { - 0x63,0x7c,0x77,0x7b,0xf2,0x6b,0x6f,0xc5,0x30,0x01,0x67,0x2b,0xfe,0xd7,0xab,0x76, - 0xca,0x82,0xc9,0x7d,0xfa,0x59,0x47,0xf0,0xad,0xd4,0xa2,0xaf,0x9c,0xa4,0x72,0xc0, - 0xb7,0xfd,0x93,0x26,0x36,0x3f,0xf7,0xcc,0x34,0xa5,0xe5,0xf1,0x71,0xd8,0x31,0x15, - 0x04,0xc7,0x23,0xc3,0x18,0x96,0x05,0x9a,0x07,0x12,0x80,0xe2,0xeb,0x27,0xb2,0x75, - 0x09,0x83,0x2c,0x1a,0x1b,0x6e,0x5a,0xa0,0x52,0x3b,0xd6,0xb3,0x29,0xe3,0x2f,0x84, - 0x53,0xd1,0x00,0xed,0x20,0xfc,0xb1,0x5b,0x6a,0xcb,0xbe,0x39,0x4a,0x4c,0x58,0xcf, - 0xd0,0xef,0xaa,0xfb,0x43,0x4d,0x33,0x85,0x45,0xf9,0x02,0x7f,0x50,0x3c,0x9f,0xa8, - 0x51,0xa3,0x40,0x8f,0x92,0x9d,0x38,0xf5,0xbc,0xb6,0xda,0x21,0x10,0xff,0xf3,0xd2, - 0xcd,0x0c,0x13,0xec,0x5f,0x97,0x44,0x17,0xc4,0xa7,0x7e,0x3d,0x64,0x5d,0x19,0x73, - 0x60,0x81,0x4f,0xdc,0x22,0x2a,0x90,0x88,0x46,0xee,0xb8,0x14,0xde,0x5e,0x0b,0xdb, - 0xe0,0x32,0x3a,0x0a,0x49,0x06,0x24,0x5c,0xc2,0xd3,0xac,0x62,0x91,0x95,0xe4,0x79, - 0xe7,0xc8,0x37,0x6d,0x8d,0xd5,0x4e,0xa9,0x6c,0x56,0xf4,0xea,0x65,0x7a,0xae,0x08, - 0xba,0x78,0x25,0x2e,0x1c,0xa6,0xb4,0xc6,0xe8,0xdd,0x74,0x1f,0x4b,0xbd,0x8b,0x8a, - 0x70,0x3e,0xb5,0x66,0x48,0x03,0xf6,0x0e,0x61,0x35,0x57,0xb9,0x86,0xc1,0x1d,0x9e, - 0xe1,0xf8,0x98,0x11,0x69,0xd9,0x8e,0x94,0x9b,0x1e,0x87,0xe9,0xce,0x55,0x28,0xdf, - 0x8c,0xa1,0x89,0x0d,0xbf,0xe6,0x42,0x68,0x41,0x99,0x2d,0x0f,0xb0,0x54,0xbb,0x16 -}; - -const uint8_t aes_rcon[11] = {0x8d,0x01,0x02,0x04,0x08,0x10,0x20,0x40,0x80,0x1b,0x36}; - -static uint8_t aes_rsbox[256]; -static int aes_rsbox_init; - -static void aes_init_rsbox(void) { - if (aes_rsbox_init) return; - for (int i = 0; i < 256; i++) aes_rsbox[aes_sbox[i]] = (uint8_t)i; - aes_rsbox_init = 1; -} - -#define xtime(x) ((((x) << 1) ^ ((((x) >> 7) & 1) * 0x1b)) & 0xff) -#define mul(a,b) ((((b)&1)*a) ^ (((b>>1)&1)*xtime(a)) ^ (((b>>2)&1)*xtime(xtime(a))) ^ (((b>>3)&1)*xtime(xtime(xtime(a))))) - #if defined(__x86_64__) || defined(_M_X64) #define AES_HW_AVAILABLE 1 #else @@ -207,99 +174,45 @@ static void aes_hw_cbc_decrypt(AES_ctx* ctx, uint8_t* buf, size_t len) { "xmm10","xmm11","xmm12","xmm13","xmm14","xmm15","memory","cc" ); } + +__attribute__((noinline)) +static void aes_hw_cbc_encrypt(AES_ctx* ctx, uint8_t* buf, size_t len) { + size_t blocks = len >> 4; + if (!blocks) return; + + __asm__ volatile ( + "movdqa (%[rk]), %%xmm2\n\t" + "movdqu (%[iv]), %%xmm0\n\t" + ".p2align 4\n" + "1:\n\t" + "movdqu (%[buf]), %%xmm1\n\t" + "pxor %%xmm0, %%xmm1\n\t" + "pxor %%xmm2, %%xmm1\n\t" + "aesenc 16(%[rk]), %%xmm1\n\t" + "aesenc 32(%[rk]), %%xmm1\n\t" + "aesenc 48(%[rk]), %%xmm1\n\t" + "aesenc 64(%[rk]), %%xmm1\n\t" + "aesenc 80(%[rk]), %%xmm1\n\t" + "aesenc 96(%[rk]), %%xmm1\n\t" + "aesenc 112(%[rk]), %%xmm1\n\t" + "aesenc 128(%[rk]), %%xmm1\n\t" + "aesenc 144(%[rk]), %%xmm1\n\t" + "aesenclast 160(%[rk]), %%xmm1\n\t" + "movdqu %%xmm1, (%[buf])\n\t" + "movdqa %%xmm1, %%xmm0\n\t" + "addq $16, %[buf]\n\t" + "decq %[n]\n\t" + "jnz 1b\n\t" + "movdqu %%xmm0, (%[iv])\n\t" + : [buf] "+r" (buf), [n] "+r" (blocks) + : [rk] "r" (ctx->round_keys), [iv] "r" (ctx->iv) + : "xmm0", "xmm1", "xmm2", "memory", "cc" + ); +} #endif -static void aes_sw_key_expand(uint8_t* rk, const uint8_t* key) { - uint8_t t[4]; - for (int i = 0; i < 16; ++i) rk[i] = key[i]; - for (int i = 4; i < 44; ++i) { - int k = (i - 1) << 2; - t[0] = rk[k]; t[1] = rk[k+1]; t[2] = rk[k+2]; t[3] = rk[k+3]; - if ((i & 3) == 0) { - uint8_t tmp = t[0]; - t[0] = aes_sbox[t[1]] ^ aes_rcon[i >> 2]; - t[1] = aes_sbox[t[2]]; - t[2] = aes_sbox[t[3]]; - t[3] = aes_sbox[tmp]; - } - int j = i << 2; k = (i - 4) << 2; - rk[j] = rk[k] ^ t[0]; rk[j+1] = rk[k+1] ^ t[1]; - rk[j+2] = rk[k+2] ^ t[2]; rk[j+3] = rk[k+3] ^ t[3]; - } -} - -static void aes_sw_decrypt_block(uint8_t* s, const uint8_t* rk) { - uint8_t t; - for (int i = 0; i < 16; ++i) s[i] ^= rk[160+i]; - for (int r = 9; r >= 0; --r) { - t = s[13]; s[13] = s[9]; s[9] = s[5]; s[5] = s[1]; s[1] = t; - t = s[2]; s[2] = s[10]; s[10] = t; - t = s[6]; s[6] = s[14]; s[14] = t; - t = s[3]; s[3] = s[7]; s[7] = s[11]; s[11] = s[15]; s[15] = t; - for (int i = 0; i < 16; ++i) s[i] = aes_rsbox[s[i]]; - for (int i = 0; i < 16; ++i) s[i] ^= rk[(r<<4)+i]; - if (r == 0) break; - for (int i = 0; i < 4; ++i) { - int j = i << 2; - uint8_t a = s[j], b = s[j+1], c = s[j+2], d = s[j+3]; - s[j] = mul(a,0x0e) ^ mul(b,0x0b) ^ mul(c,0x0d) ^ mul(d,0x09); - s[j+1] = mul(a,0x09) ^ mul(b,0x0e) ^ mul(c,0x0b) ^ mul(d,0x0d); - s[j+2] = mul(a,0x0d) ^ mul(b,0x09) ^ mul(c,0x0e) ^ mul(d,0x0b); - s[j+3] = mul(a,0x0b) ^ mul(b,0x0d) ^ mul(c,0x09) ^ mul(d,0x0e); - } - } -} - -static void aes_sw_cbc_decrypt(AES_ctx* ctx, uint8_t* buf, size_t len) { - uint8_t tmp[16], niv[16]; - for (size_t i = 0; i < len; i += 16) { - for (int j = 0; j < 16; ++j) { niv[j] = buf[i+j]; tmp[j] = buf[i+j]; } - aes_sw_decrypt_block(tmp, ctx->round_keys); - for (int j = 0; j < 16; ++j) buf[i+j] = tmp[j] ^ ctx->iv[j]; - for (int j = 0; j < 16; ++j) ctx->iv[j] = niv[j]; - } -} - -static void aes_sw_encrypt_block(uint8_t* s, const uint8_t* rk) { - uint8_t t; - for (int i = 0; i < 16; ++i) s[i] ^= rk[i]; - for (int r = 1; ; ++r) { - for (int i = 0; i < 16; ++i) s[i] = aes_sbox[s[i]]; - t = s[1]; s[1] = s[5]; s[5] = s[9]; s[9] = s[13]; s[13] = t; - t = s[2]; s[2] = s[10]; s[10] = t; - t = s[6]; s[6] = s[14]; s[14] = t; - t = s[3]; s[3] = s[15]; s[15] = s[11]; s[11] = s[7]; s[7] = t; - if (r == 10) { for (int i = 0; i < 16; ++i) s[i] ^= rk[(r<<4)+i]; break; } - for (int i = 0; i < 4; ++i) { - int j = i << 2; - uint8_t a = s[j], b = s[j+1], c = s[j+2], d = s[j+3]; - s[j] = xtime(a) ^ xtime(b) ^ b ^ c ^ d; - s[j+1] = a ^ xtime(b) ^ xtime(c) ^ c ^ d; - s[j+2] = a ^ b ^ xtime(c) ^ xtime(d) ^ d; - s[j+3] = xtime(a) ^ a ^ b ^ c ^ xtime(d); - } - for (int i = 0; i < 16; ++i) s[i] ^= rk[(r<<4)+i]; - } -} - -static void aes_sw_cbc_encrypt(AES_ctx* ctx, uint8_t* buf, size_t len) { - for (size_t i = 0; i < len; i += 16) { - for (int j = 0; j < 16; ++j) buf[i+j] ^= ctx->iv[j]; - aes_sw_encrypt_block(buf + i, ctx->round_keys); - for (int j = 0; j < 16; ++j) ctx->iv[j] = buf[i+j]; - } -} - void AES_init_ctx_iv(AES_ctx* ctx, const uint8_t* key, const uint8_t* iv) { - aes_init_rsbox(); -#if AES_HW_AVAILABLE - if (aes_hw_supported()) { - aes_hw_key_expand(ctx, key); - } else -#endif - { - aes_sw_key_expand(ctx->round_keys, key); - } + aes_hw_key_expand(ctx, key); for (int i = 0; i < 16; ++i) ctx->iv[i] = iv[i]; } @@ -308,12 +221,9 @@ void AES_ctx_set_iv(AES_ctx* ctx, const uint8_t* iv) { } void AES_CBC_decrypt_buffer(AES_ctx* ctx, uint8_t* buf, size_t len) { -#if AES_HW_AVAILABLE - if (aes_hw_supported()) { aes_hw_cbc_decrypt(ctx, buf, len); return; } -#endif - aes_sw_cbc_decrypt(ctx, buf, len); + aes_hw_cbc_decrypt(ctx, buf, len); } void AES_CBC_encrypt_buffer(AES_ctx* ctx, uint8_t* buf, size_t len) { - aes_sw_cbc_encrypt(ctx, buf, len); + aes_hw_cbc_encrypt(ctx, buf, len); } diff --git a/src/crypto.c b/src/crypto.c index 8cf84d3..9437f48 100644 --- a/src/crypto.c +++ b/src/crypto.c @@ -14,10 +14,6 @@ const uint8_t OPTION_KEY[16] = { 0x5c, 0x84, 0xa9, 0xe7, 0x26, 0xea, 0xa5, 0xdd, 0x35, 0x1f, 0x2b, 0x07, 0x50, 0xc2, 0x36, 0x97 }; -const uint8_t OPTION_IV[16] = { - 0xc0, 0x63, 0xbf, 0x6f, 0x56, 0x2d, 0x08, 0x4d, - 0x79, 0x63, 0xc9, 0x87, 0xf5, 0x28, 0x17, 0x61 -}; const uint8_t APM3_SEED[96] = { 0xC7, 0x3C, 0xDD, 0xBF, 0x7A, 0xFB, 0x0E, 0xBC, 0xE6, 0xDE, 0xD4, 0xD9, 0xB3, 0xDF, 0x3B, 0x03, 0x3F, 0xE1, 0x40, 0xE4, 0xF4, 0xFF, 0x96, 0xC5, 0x79, 0x90, 0x8B, 0x5B, 0x69, 0x6A, 0xBE, 0xEE, @@ -89,7 +85,7 @@ void iv_page(uint64_t off, const uint8_t* base, uint8_t* out) { ); } -bool iv_file(const uint8_t key[16], const uint8_t* hdr, const uint8_t* page, uint8_t out[16]) { +void iv_file(const uint8_t key[16], const uint8_t* hdr, const uint8_t* page, uint8_t out[16]) { uint8_t iv[16]; uint8_t header[16]; memcpy(header, page, 16); @@ -101,12 +97,10 @@ bool iv_file(const uint8_t key[16], const uint8_t* hdr, const uint8_t* page, uin AES_CBC_decrypt_buffer(&ctx, header, 16); memcpy(out, header, 16); - return true; } bool key_game(const char* id, GameKeys* out) { - if (key_lookup(id, out->key, out->iv, &out->external)) { - out->has_iv = true; + if (key_lookup(id, out->key, &out->external)) { return true; } out->external = false; diff --git a/src/exfat.c b/src/exfat.c index d6d7503..1bffeec 100644 --- a/src/exfat.c +++ b/src/exfat.c @@ -1,9 +1,9 @@ #include "exfat.h" -#include "progress.h" -static void count_directory_size(ExfatContext* ctx, uint32_t start_cluster); +#define EXFAT_IO_SIZE (1024 * 1024) static uint64_t get_cluster_offset(ExfatContext* ctx, uint32_t cluster) { + if (cluster < 2) return 0; return ctx->cluster_heap_offset_bytes + ((uint64_t)(cluster - 2) * ctx->bytes_per_cluster); } @@ -11,14 +11,20 @@ static bool exfat_read(ExfatContext* ctx, void* buffer, uint64_t offset, size_t if (ctx->stream) { return stream_read(ctx->stream, buffer, offset, size); } - if (FSEEKO(ctx->fp, offset, SEEK_SET) != 0) { + if (ctx->raw_file_pos != offset) { + if (FSEEKO(ctx->fp, offset, SEEK_SET) != 0) { + return false; + } + } + if (fread(buffer, 1, size, ctx->fp) != size) { return false; } - return fread(buffer, 1, size, ctx->fp) == size; + ctx->raw_file_pos = offset + size; + return true; } static bool read_cluster(ExfatContext* ctx, uint32_t cluster, void* buffer) { - uint32_t offset = get_cluster_offset(ctx, cluster); + uint64_t offset = get_cluster_offset(ctx, cluster); return exfat_read(ctx, buffer, offset, ctx->bytes_per_cluster); } @@ -31,10 +37,7 @@ static uint32_t get_next_cluster(ExfatContext* ctx, uint32_t cluster) { if (next >= 0xFFFFFFF8) { return 0; } - if (next == 0) - { - return cluster + 1; - } + if (next == 0) return 0; if (next >= max_cluster) { return 0; } @@ -66,52 +69,98 @@ static bool combine_path(char* dest, size_t dest_size, const char* dir, const ch return true; } +static void update_cached_dir(ExfatContext* ctx, const char* output_path) { + char parent[MAX_PATH_LENGTH]; + strncpy(parent, output_path, sizeof(parent) - 1); + parent[sizeof(parent) - 1] = '\0'; + char* sep = strrchr(parent, PATH_SEP_CHAR); + if (!sep) return; + *sep = '\0'; + if (strcmp(parent, ctx->last_dir) == 0) return; + close_output_dir(ctx->cached_dir); + ctx->cached_dir = open_output_dir(parent); + strncpy(ctx->last_dir, parent, MAX_PATH_LENGTH - 1); + ctx->last_dir[MAX_PATH_LENGTH - 1] = '\0'; +} + static bool extract_file(ExfatContext* ctx, ExfatFileInfo* file, const char* output_path) { - FILE* out = FOPEN(output_path, "wb"); + update_cached_dir(ctx, output_path); + + FILE* out = NULL; + if (ctx->cached_dir != INVALID_DIR_HANDLE) { + out = fopen_in_dir(ctx->cached_dir, file->name, file->data_length >= 65536 ? file->data_length : 0); + } + if (!out) { + if (file->data_length >= 65536) { + out = FOPEN_PREALLOC(output_path, file->data_length); + } else { + out = FOPEN(output_path, "wb"); + } + } if (!out) { return false; } - setvbuf(out, NULL, _IOFBF, ctx->bytes_per_cluster); - uint32_t current_cluster = file->first_cluster; uint64_t remaining = file->data_length; - bool success = true; - while (remaining > 0 && current_cluster != 0 && success) { - if (!read_cluster(ctx, current_cluster, ctx->io_buf)) { - success = false; - break; + + if (file->no_fat_chain) { + uint64_t offset = get_cluster_offset(ctx, file->first_cluster); + while (remaining > 0 && success) { + size_t chunk = (remaining > EXFAT_IO_SIZE) ? EXFAT_IO_SIZE : (size_t)remaining; + if (!exfat_read(ctx, ctx->io_buf, offset, chunk)) { + success = false; + break; + } + if (FWRITE_DIRECT(out, ctx->io_buf, chunk) != chunk) { + success = false; + break; + } + offset += chunk; + remaining -= chunk; + ctx->extracted_bytes += chunk; } - - size_t write_size = (remaining > ctx->bytes_per_cluster) ? ctx->bytes_per_cluster : (size_t)remaining; - if (fwrite(ctx->io_buf, 1, write_size, out) != write_size) { - success = false; - break; - } - - remaining -= write_size; - - if (file->no_fat_chain) { - current_cluster++; - } else { + } else { + uint32_t max_clusters = (uint32_t)((file->data_length + ctx->bytes_per_cluster - 1) / ctx->bytes_per_cluster); + uint32_t cluster_count = 0; + uint32_t current_cluster = file->first_cluster; + while (remaining > 0 && current_cluster != 0 && success) { + if (++cluster_count > max_clusters) break; + size_t chunk = (remaining > ctx->bytes_per_cluster) ? ctx->bytes_per_cluster : (size_t)remaining; + uint64_t offset = get_cluster_offset(ctx, current_cluster); + if (!exfat_read(ctx, ctx->io_buf, offset, chunk)) { + success = false; + break; + } + if (FWRITE_DIRECT(out, ctx->io_buf, chunk) != chunk) { + success = false; + break; + } + remaining -= chunk; current_cluster = get_next_cluster(ctx, current_cluster); + ctx->extracted_bytes += chunk; } + } - ctx->extracted_bytes += write_size; - if (ctx->progress) { - progress_update((Progress*)ctx->progress, ctx->extracted_bytes); - } + if (success && file->modify_timestamp != 0) { + uint64_t mtime = exfat_timestamp_to_ntfs(file->modify_timestamp, file->modify_10ms, file->modify_utc_offset); + uint64_t atime = exfat_timestamp_to_ntfs(file->access_timestamp, 0, file->access_utc_offset); +#ifdef _WIN32 + set_file_times_handle(out, mtime, atime); +#endif } fclose(out); if (success) { ctx->files_extracted++; +#ifndef _WIN32 if (file->modify_timestamp != 0) { uint64_t mtime = exfat_timestamp_to_ntfs(file->modify_timestamp, file->modify_10ms, file->modify_utc_offset); uint64_t atime = exfat_timestamp_to_ntfs(file->access_timestamp, 0, file->access_utc_offset); set_file_times(output_path, mtime, atime); } +#endif } return success; @@ -126,9 +175,11 @@ static bool process_directory_recursive(ExfatContext* ctx, uint32_t start_cluste } uint32_t current_cluster = start_cluster; + uint32_t cluster_count = 0; bool finished = false; while (!finished && current_cluster != 0) { + if (++cluster_count > ctx->boot_sector.cluster_count) break; if (!read_cluster(ctx, current_cluster, ctx->cluster_buf)) { return false; } @@ -144,6 +195,9 @@ static bool process_directory_recursive(ExfatContext* ctx, uint32_t start_cluste } if (entry_type == EXFAT_ENTRY_FILE) { + if (entry_offset + EXFAT_ENTRY_SIZE * 2 > ctx->bytes_per_cluster) { + i++; entry_offset += EXFAT_ENTRY_SIZE; continue; + } ExfatFileEntry* file_entry = (ExfatFileEntry*)entry_ptr; ExfatStreamEntry* stream_entry = (ExfatStreamEntry*)(entry_ptr + EXFAT_ENTRY_SIZE); if (stream_entry->entry_type != EXFAT_ENTRY_STREAM) { @@ -154,6 +208,10 @@ static bool process_directory_recursive(ExfatContext* ctx, uint32_t start_cluste int total_name_chars = stream_entry->name_length; int num_name_entries = (total_name_chars + 14) / 15; + if (entry_offset + EXFAT_ENTRY_SIZE * (2 + num_name_entries) > ctx->bytes_per_cluster) { + i++; entry_offset += EXFAT_ENTRY_SIZE; continue; + } + char full_name[MAX_FILENAME_LENGTH]; uint16_t full_name_unicode[MAX_FILENAME_LENGTH]; int pos = 0; @@ -175,6 +233,13 @@ static bool process_directory_recursive(ExfatContext* ctx, uint32_t start_cluste memset(&file_info, 0, sizeof(file_info)); strncpy(file_info.name, full_name, MAX_PATH_LENGTH - 1); file_info.name[MAX_PATH_LENGTH - 1] = '\0'; + if (file_entry->file_attributes & 0x04) { + int total_entries = 2 + num_name_entries; + i += total_entries; + entry_offset += EXFAT_ENTRY_SIZE * total_entries; + continue; + } + file_info.first_cluster = stream_entry->first_cluster; file_info.data_length = stream_entry->data_length; file_info.is_directory = ((file_entry->file_attributes & 0x10) != 0); @@ -199,14 +264,8 @@ static bool process_directory_recursive(ExfatContext* ctx, uint32_t start_cluste if (create_directories(full_path)) { process_directory_recursive(ctx, file_info.first_cluster, full_path, depth + 1); if (file_info.modify_timestamp != 0) { - if (ctx->deferred_count >= ctx->deferred_capacity) { - uint32_t new_cap = ctx->deferred_capacity ? ctx->deferred_capacity * 2 : 256; - DeferredDirTime* new_buf = realloc(ctx->deferred_dirs, new_cap * sizeof(DeferredDirTime)); - if (new_buf) { - ctx->deferred_dirs = new_buf; - ctx->deferred_capacity = new_cap; - } - } + if (ctx->deferred_count >= ctx->deferred_capacity) + grow_deferred_dirs(&ctx->deferred_dirs, &ctx->deferred_capacity); if (ctx->deferred_count < ctx->deferred_capacity) { DeferredDirTime* d = &ctx->deferred_dirs[ctx->deferred_count++]; STRCPY_S(d->path, sizeof(d->path), full_path); @@ -238,20 +297,29 @@ static bool process_directory_recursive(ExfatContext* ctx, uint32_t start_cluste return true; } -static bool process_directory(ExfatContext* ctx, uint32_t start_cluster, const char* output_dir) { - return process_directory_recursive(ctx, start_cluster, output_dir, 0); -} - static bool exfat_setup_fields(ExfatContext* ctx) { + ctx->cached_dir = INVALID_DIR_HANDLE; + + if (ctx->boot_sector.bytes_per_sector_shift < 9 || ctx->boot_sector.bytes_per_sector_shift > 12) return false; + if (ctx->boot_sector.sectors_per_cluster_shift > 25 - ctx->boot_sector.bytes_per_sector_shift) return false; + ctx->bytes_per_sector = (1 << ctx->boot_sector.bytes_per_sector_shift); ctx->bytes_per_cluster = ctx->bytes_per_sector * (1 << ctx->boot_sector.sectors_per_cluster_shift); + if (ctx->bytes_per_cluster == 0) return false; + + if (ctx->boot_sector.fat_offset > (0xFFFFFFFFU / ctx->bytes_per_sector)) return false; + if (ctx->boot_sector.cluster_heap_offset > (0xFFFFFFFFU / ctx->bytes_per_sector)) return false; + if (ctx->boot_sector.fat_length > (0xFFFFFFFFU / ctx->bytes_per_sector)) return false; + ctx->cluster_heap_offset_bytes = ctx->boot_sector.cluster_heap_offset * ctx->bytes_per_sector; ctx->fat_offset_bytes = ctx->boot_sector.fat_offset * ctx->bytes_per_sector; ctx->fat_length_bytes = ctx->boot_sector.fat_length * ctx->bytes_per_sector; + if (ctx->fat_length_bytes == 0 || ctx->fat_length_bytes > (1U << 30)) return false; + ctx->fat = malloc(ctx->fat_length_bytes); ctx->cluster_buf = malloc(ctx->bytes_per_cluster); - ctx->io_buf = malloc(ctx->bytes_per_cluster); + ctx->io_buf = malloc(EXFAT_IO_SIZE); if (!ctx->fat || !ctx->cluster_buf || !ctx->io_buf) { free(ctx->fat); free(ctx->cluster_buf); @@ -277,7 +345,7 @@ bool exfat_init(ExfatContext* ctx, const char* filename) { return false; } - if (fseek(ctx->fp, (long)ctx->fat_offset_bytes, SEEK_SET) != 0 || + if (FSEEKO(ctx->fp, ctx->fat_offset_bytes, SEEK_SET) != 0 || fread(ctx->fat, 1, ctx->fat_length_bytes, ctx->fp) != ctx->fat_length_bytes) { exfat_close(ctx); return false; @@ -304,93 +372,14 @@ bool exfat_init_stream(ExfatContext* ctx, DecryptStream* stream) { return true; } -static void count_directory_size_recursive(ExfatContext* ctx, uint32_t start_cluster, int depth) { - if (depth > EXFAT_MAX_RECURSION_DEPTH) return; - - uint32_t current_cluster = start_cluster; - bool finished = false; - - while (!finished && current_cluster != 0) { - if (!read_cluster(ctx, current_cluster, ctx->cluster_buf)) { - break; - } - - uint32_t entries_per_cluster = ctx->bytes_per_cluster / EXFAT_ENTRY_SIZE; - uint32_t entry_offset = 0; - for (uint32_t i = 0; i < entries_per_cluster; ) { - uint8_t* entry_ptr = ctx->cluster_buf + entry_offset; - uint8_t entry_type = *entry_ptr; - if (entry_type == EXFAT_ENTRY_EOD) { - finished = true; - break; - } - - if (entry_type == EXFAT_ENTRY_FILE) { - ExfatFileEntry* file_entry = (ExfatFileEntry*)entry_ptr; - ExfatStreamEntry* stream_entry = (ExfatStreamEntry*)(entry_ptr + EXFAT_ENTRY_SIZE); - if (stream_entry->entry_type != EXFAT_ENTRY_STREAM) { - i++; - entry_offset += EXFAT_ENTRY_SIZE; - continue; - } - - bool is_directory = ((file_entry->file_attributes & 0x10) != 0); - uint32_t first_cluster = stream_entry->first_cluster; - uint64_t data_length = stream_entry->data_length; - int num_name_entries = (stream_entry->name_length + 14) / 15; - int total_entries = 2 + num_name_entries; - i += total_entries; - entry_offset += EXFAT_ENTRY_SIZE * total_entries; - - if (is_directory) { - count_directory_size_recursive(ctx, first_cluster, depth + 1); - if (!read_cluster(ctx, current_cluster, ctx->cluster_buf)) { - return; - } - } else { - ctx->total_bytes += data_length; - } - continue; - } - i++; - entry_offset += EXFAT_ENTRY_SIZE; - } - - if (!finished) { - current_cluster = get_next_cluster(ctx, current_cluster); - } - } -} - -static void count_directory_size(ExfatContext* ctx, uint32_t start_cluster) { - count_directory_size_recursive(ctx, start_cluster, 0); -} - bool exfat_extract_all(ExfatContext* ctx, const char* output_dir) { if (!create_directories(output_dir)) { return false; } - ctx->total_bytes = 0; ctx->extracted_bytes = 0; - count_directory_size(ctx, ctx->boot_sector.first_cluster_of_root_dir); - if (ctx->verbose && !ctx->silent) { - printf("%llu B\n", (unsigned long long)ctx->total_bytes); - } - - Progress progress; - if (!ctx->silent) { - progress_init(&progress, ctx->total_bytes > 0 ? ctx->total_bytes : 1); - ctx->progress = &progress; - } - - bool result = process_directory(ctx, ctx->boot_sector.first_cluster_of_root_dir, output_dir); - - if (!ctx->silent && ctx->progress) { - progress_finish(&progress); - } - ctx->progress = NULL; + bool result = process_directory_recursive(ctx, ctx->boot_sector.first_cluster_of_root_dir, output_dir, 0); for (uint32_t i = ctx->deferred_count; i > 0; i--) { DeferredDirTime* d = &ctx->deferred_dirs[i - 1]; @@ -405,6 +394,8 @@ bool exfat_extract_all(ExfatContext* ctx, const char* output_dir) { } void exfat_close(ExfatContext* ctx) { + close_output_dir(ctx->cached_dir); + ctx->cached_dir = INVALID_DIR_HANDLE; if (!ctx->stream && ctx->fp) { fclose(ctx->fp); ctx->fp = NULL; diff --git a/src/keys.c b/src/keys.c index 99ed3fb..df39d5c 100644 --- a/src/keys.c +++ b/src/keys.c @@ -4,7 +4,6 @@ typedef struct { char game_id[8]; uint8_t key[16]; - uint8_t iv[16]; } ExternalKey; static ExternalKey* external_keys; @@ -59,11 +58,6 @@ static void load_keys_from_file(const char* path) { char* brace1_end = strchr(brace1, '}'); if (!brace1_end) continue; - char* brace2 = strchr(brace1_end, '{'); - if (!brace2) continue; - char* brace2_end = strchr(brace2, '}'); - if (!brace2_end) continue; - if (external_keys_count >= external_keys_cap) { size_t new_cap = external_keys_cap ? external_keys_cap * 2 : 8; ExternalKey* new_keys = realloc(external_keys, new_cap * sizeof(ExternalKey)); @@ -77,7 +71,6 @@ static void load_keys_from_file(const char* path) { memcpy(k->game_id, quote1 + 1, id_len); if (!parse_hex_bytes(brace1, brace1_end, k->key, 16)) continue; - if (!parse_hex_bytes(brace2, brace2_end, k->iv, 16)) continue; external_keys_count++; } @@ -102,14 +95,13 @@ bool key_any(void) { return count_embedded_keys() > 0 || external_keys_count > 0; } -bool key_lookup(const char* id, uint8_t out_key[16], uint8_t out_iv[16], bool* from_external) { +bool key_lookup(const char* id, uint8_t out_key[16], bool* from_external) { try_load_external_keys(); if (from_external) *from_external = false; for (size_t i = 0; i < external_keys_count; i++) { if (strcmp(external_keys[i].game_id, id) == 0) { memcpy(out_key, external_keys[i].key, 16); - memcpy(out_iv, external_keys[i].iv, 16); if (from_external) *from_external = true; return true; } @@ -119,7 +111,6 @@ bool key_lookup(const char* id, uint8_t out_key[16], uint8_t out_iv[16], bool* f while (entry->game_id != NULL) { if (strcmp(entry->game_id, id) == 0) { memcpy(out_key, entry->key, 16); - memcpy(out_iv, entry->iv, 16); return true; } entry++; diff --git a/src/lib.c b/src/lib.c index f7c9652..a2d1658 100644 --- a/src/lib.c +++ b/src/lib.c @@ -102,9 +102,11 @@ void* lib_malloc(size_t size) { } void* lib_calloc(size_t count, size_t size) { + if (size && count > (size_t)-1 / size) return NULL; size_t total = count * size; if (!total) return NULL; - return RtlAllocateHeap(lib_heap, 0x08, total); +#define HEAP_ZERO_MEMORY 0x08 + return RtlAllocateHeap(lib_heap, HEAP_ZERO_MEMORY, total); } void* lib_realloc(void* ptr, size_t size) { @@ -172,6 +174,7 @@ void* lib_malloc(size_t size) { } void* lib_calloc(size_t count, size_t size) { + if (size && count > (size_t)-1 / size) return NULL; size_t total = count * size; void* ptr = lib_malloc(total); if (ptr) lib_memset(ptr, 0, total); @@ -312,23 +315,31 @@ void* memset(void* dst, int c, size_t n) __attribute__((alias("lib_memset"))); #define memset lib_memset int lib_memcmp(const void* s1, const void* s2, size_t n) { - const uint8_t* p1 = s1; const uint8_t* p2 = s2; + const uint8_t *p1 = s1, *p2 = s2; + while (n >= 8) { + uint64_t a, b; + __builtin_memcpy(&a, p1, 8); + __builtin_memcpy(&b, p2, 8); + if (a != b) goto byte_cmp; + p1 += 8; p2 += 8; n -= 8; + } +byte_cmp: while (n--) { if (*p1 != *p2) return *p1 - *p2; p1++; p2++; } return 0; } #ifdef PLATFORM_WINDOWS -static size_t utf8_to_utf16(const char* utf8, WCHAR* utf16, size_t utf16_max) { +size_t utf8_to_utf16(const char* utf8, WCHAR* utf16, size_t utf16_max) { size_t out = 0; const uint8_t* s = (const uint8_t*)utf8; while (*s && out < utf16_max - 1) { uint32_t cp; if (s[0] < 0x80) { cp = s[0]; s += 1; } - else if ((s[0] & 0xE0) == 0xC0) { cp = ((s[0] & 0x1F) << 6) | (s[1] & 0x3F); s += 2; } - else if ((s[0] & 0xF0) == 0xE0) { cp = ((s[0] & 0x0F) << 12) | ((s[1] & 0x3F) << 6) | (s[2] & 0x3F); s += 3; } - else if ((s[0] & 0xF8) == 0xF0) { cp = ((s[0] & 0x07) << 18) | ((s[1] & 0x3F) << 12) | ((s[2] & 0x3F) << 6) | (s[3] & 0x3F); s += 4; } - else { cp = '?'; s += 1; } + else if ((s[0] & 0xE0) == 0xC0 && s[1]) { cp = ((s[0] & 0x1F) << 6) | (s[1] & 0x3F); s += 2; } + else if ((s[0] & 0xF0) == 0xE0 && s[1] && s[2]) { cp = ((s[0] & 0x0F) << 12) | ((s[1] & 0x3F) << 6) | (s[2] & 0x3F); s += 3; } + else if ((s[0] & 0xF8) == 0xF0 && s[1] && s[2] && s[3]) { cp = ((s[0] & 0x07) << 18) | ((s[1] & 0x3F) << 12) | ((s[2] & 0x3F) << 6) | (s[3] & 0x3F); s += 4; } + else { s += 1; continue; } if (cp <= 0xFFFF) utf16[out++] = (WCHAR)cp; else if (cp <= 0x10FFFF && out < utf16_max - 2) { cp -= 0x10000; @@ -458,6 +469,44 @@ FILE* lib_wfopen_prealloc(const WCHAR* path, uint64_t size) { return f; } +HANDLE lib_open_dir_handle(const WCHAR* path) { + UNICODE_STRING nt_path = {0}; + if (!path_to_nt(path, &nt_path)) return INVALID_HANDLE_VALUE; + OBJECT_ATTRIBUTES oa = { sizeof(OBJECT_ATTRIBUTES), NULL, &nt_path, OBJ_CASE_INSENSITIVE, NULL, NULL }; + HANDLE h; IO_STATUS_BLOCK iosb; + NTSTATUS status = NtCreateFile(&h, FILE_ADD_FILE | FILE_TRAVERSE | SYNCHRONIZE, &oa, &iosb, NULL, + FILE_ATTRIBUTE_NORMAL, FILE_SHARE_READ | FILE_SHARE_WRITE, FILE_OPEN, + FILE_DIRECTORY_FILE | FILE_SYNCHRONOUS_IO_NONALERT, NULL, 0); + RtlFreeUnicodeString(&nt_path); + if (!NT_SUCCESS(status)) return INVALID_HANDLE_VALUE; + return h; +} + +FILE* lib_fopen_relative(HANDLE dir_handle, const WCHAR* filename, uint64_t prealloc_size) { + UNICODE_STRING name_str; + name_str.Buffer = (WCHAR*)filename; + name_str.Length = 0; + while (filename[name_str.Length / 2]) name_str.Length += 2; + name_str.MaximumLength = name_str.Length + 2; + + OBJECT_ATTRIBUTES oa = { sizeof(OBJECT_ATTRIBUTES), dir_handle, &name_str, OBJ_CASE_INSENSITIVE, NULL, NULL }; + HANDLE h; IO_STATUS_BLOCK iosb; + LARGE_INTEGER alloc = {0}; + if (prealloc_size) alloc.QuadPart = (LONGLONG)prealloc_size; + + NTSTATUS status = NtCreateFile(&h, FILE_WRITE_DATA | FILE_WRITE_ATTRIBUTES | SYNCHRONIZE, &oa, &iosb, + prealloc_size ? &alloc : NULL, FILE_ATTRIBUTE_NORMAL, FILE_SHARE_READ, FILE_OVERWRITE_IF, + FILE_NON_DIRECTORY_FILE | FILE_SYNCHRONOUS_IO_NONALERT | FILE_SEQUENTIAL_ONLY, NULL, 0); + if (!NT_SUCCESS(status)) return NULL; + + FILE* f = pool_alloc_file(); + if (!f) { NtClose(h); return NULL; } + + f->handle = h; f->buf_pos = 0; f->buf_fill = 0; f->file_pos = 0; f->flags = LIB_FILE_WRITE; + if (fd_next < MAX_FD_TABLE) fd_table[fd_next++] = h; + return f; +} + size_t lib_fwrite_direct(FILE* f, const void* buf, size_t size) { if (!f || !buf || !size || !(f->flags & LIB_FILE_WRITE)) return 0; IO_STATUS_BLOCK iosb = {0}; @@ -493,6 +542,7 @@ static bool refill_read_buffer(FILE* f) { size_t lib_fread(void* buf, size_t size, size_t count, FILE* f) { if (!f || !buf || !size || !count || !(f->flags & LIB_FILE_READ)) return 0; + if (size > 1 && count > (size_t)-1 / size) return 0; size_t total = size * count, read_total = 0; uint8_t* dst = buf; while (total > 0) { @@ -504,7 +554,7 @@ size_t lib_fread(void* buf, size_t size, size_t count, FILE* f) { } else { if (total >= f->buf_cap) { IO_STATUS_BLOCK iosb = {0}; LARGE_INTEGER offset; offset.QuadPart = f->file_pos; - size_t chunk = (total / f->buf_cap) * f->buf_cap; + size_t chunk = total & ~((size_t)f->buf_cap - 1); NTSTATUS status = NtReadFile(f->handle, NULL, NULL, NULL, &iosb, dst, (ULONG)chunk, &offset, NULL); if (status == STATUS_END_OF_FILE || iosb.Information == 0) { f->flags |= LIB_FILE_EOF; break; } if (!NT_SUCCESS(status)) { f->flags |= LIB_FILE_ERROR; break; } @@ -518,12 +568,13 @@ size_t lib_fread(void* buf, size_t size, size_t count, FILE* f) { size_t lib_fwrite(const void* buf, size_t size, size_t count, FILE* f) { if (!f || !buf || !size || !count || !(f->flags & LIB_FILE_WRITE)) return 0; + if (size > 1 && count > (size_t)-1 / size) return 0; size_t total = size * count, written = 0; const uint8_t* src = buf; while (total >= f->buf_cap) { if (f->buf_fill > 0 && !flush_write_buffer(f)) return written / size; IO_STATUS_BLOCK iosb = {0}; LARGE_INTEGER offset; offset.QuadPart = f->file_pos; - size_t chunk = (total / f->buf_cap) * f->buf_cap; + size_t chunk = total & ~((size_t)f->buf_cap - 1); NTSTATUS status = NtWriteFile(f->handle, NULL, NULL, NULL, &iosb, (void*)src, (ULONG)chunk, &offset, NULL); if (!NT_SUCCESS(status)) { f->flags |= LIB_FILE_ERROR; return written / size; } f->file_pos += iosb.Information; src += iosb.Information; total -= iosb.Information; written += iosb.Information; @@ -578,13 +629,6 @@ int lib_fclose(FILE* f) { return ret; } -int lib_isatty(int fd) { - HANDLE h = (fd == 0) ? lib_stdin_handle : (fd == 1) ? lib_stdout_handle : (fd == 2) ? lib_stderr_handle : NULL; - if (!h || h == INVALID_HANDLE_VALUE) return 0; - FILE_STANDARD_INFORMATION info; IO_STATUS_BLOCK iosb; - return !NT_SUCCESS(NtQueryInformationFile(h, &iosb, &info, sizeof(info), FileStandardInformation)); -} - #define EPOCH_DIFF 116444736000000000ULL time_t lib_time(time_t* t) { @@ -649,6 +693,7 @@ static bool refill_read_buffer(FILE* f) { size_t lib_fread(void* buf, size_t size, size_t count, FILE* f) { if (!f || !buf || !size || !count || !(f->flags & LIB_FILE_READ)) return 0; + if (size > 1 && count > (size_t)-1 / size) return 0; size_t total = size * count, read_total = 0; uint8_t* dst = buf; while (total > 0) { @@ -670,8 +715,17 @@ size_t lib_fread(void* buf, size_t size, size_t count, FILE* f) { size_t lib_fwrite(const void* buf, size_t size, size_t count, FILE* f) { if (!f || !buf || !size || !count || !(f->flags & LIB_FILE_WRITE)) return 0; + if (size > 1 && count > (size_t)-1 / size) return 0; size_t total = size * count, written = 0; const uint8_t* src = buf; + while (total >= f->buf_cap) { + if (f->buf_fill > 0 && !flush_write_buffer(f)) return written / size; + size_t chunk = total & ~((size_t)f->buf_cap - 1); + long ret = syscall3(SYS_write, f->fd, (long)src, chunk); + if (ret < 0) { f->flags |= LIB_FILE_ERROR; return written / size; } + f->file_pos += ret; src += ret; total -= ret; written += ret; + if ((size_t)ret < chunk) return written / size; + } while (total > 0) { size_t space = f->buf_cap - f->buf_fill; if (space > 0) { @@ -709,11 +763,6 @@ int lib_fclose(FILE* f) { return ret; } -int lib_isatty(int fd) { - char buf[64]; - return syscall3(SYS_ioctl, fd, 0x5413, (long)buf) >= 0; -} - time_t lib_time(time_t* t) { struct linux_timespec ts; syscall2(228, 0, (long)&ts); @@ -723,8 +772,6 @@ time_t lib_time(time_t* t) { #endif -int lib_fseek(FILE* f, long offset, int whence) { return lib_fseeki64(f, (int64_t)offset, whence); } -long lib_ftell(FILE* f) { return (long)lib_ftelli64(f); } int lib_feof(FILE* f) { return f ? (f->flags & LIB_FILE_EOF) != 0 : 0; } static int lib_fgetc(FILE* f) { unsigned char c; return (lib_fread(&c, 1, 1, f) == 1) ? c : EOF; } static int lib_fputc(int c, FILE* f) { unsigned char ch = (unsigned char)c; return (lib_fwrite(&ch, 1, 1, f) == 1) ? ch : EOF; } @@ -744,7 +791,6 @@ char* lib_fgets(char* buf, int n, FILE* f) { } void lib_rewind(FILE* f) { if (f) { lib_fseeki64(f, 0, SEEK_SET); } } -void lib_setvbuf(FILE* f, char* buf, int mode, size_t size) { (void)f; (void)buf; (void)mode; (void)size; } double lib_difftime(time_t t1, time_t t0) { return (double)(t1 - t0); } static char* fmt_uint64(char* buf, uint64_t val, int base, int width, char pad, bool upper) { @@ -876,6 +922,41 @@ int lib_rmdir(const char* path) { return -1; } +int lib_rename(const char* old_path, const char* new_path) { + WCHAR wold[1024]; utf8_to_utf16(old_path, wold, 1024); + WCHAR wnew[1024]; utf8_to_utf16(new_path, wnew, 1024); + + HANDLE h = nt_open_file(wold, DELETE | SYNCHRONIZE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + FILE_OPEN, FILE_DIRECTORY_FILE); + if (h == INVALID_HANDLE_VALUE) return -1; + + UNICODE_STRING nt_new = {0}; + if (!path_to_nt(wnew, &nt_new)) { NtClose(h); return -1; } + + struct { + uint8_t ReplaceIfExists; + HANDLE RootDirectory; + ULONG FileNameLength; + WCHAR FileName[1024]; + } rename_info; + rename_info.ReplaceIfExists = 0; + rename_info.RootDirectory = NULL; + rename_info.FileNameLength = nt_new.Length; + memcpy(rename_info.FileName, nt_new.Buffer, nt_new.Length); + + IO_STATUS_BLOCK iosb; + NTSTATUS status = NtSetInformationFile(h, &iosb, &rename_info, + (ULONG)(sizeof(rename_info) - sizeof(rename_info.FileName) + nt_new.Length), + FileRenameInformation); + + RtlFreeUnicodeString(&nt_new); + NtClose(h); + if (NT_SUCCESS(status)) return 0; + lib_errno_val = (int)status; + return -1; +} + typedef struct { HANDLE dir_handle; WCHAR pattern[260]; uint8_t buffer[4096]; uint32_t buf_pos; uint32_t buf_len; bool first_call; } FindState; static HANDLE open_directory_for_enum(const char* pattern, WCHAR* out_pattern) { @@ -948,37 +1029,21 @@ bool lib_set_file_times_ntfs(FILE* f, int64_t modified_time, int64_t access_time return NT_SUCCESS(NtSetInformationFile(f->handle, &iosb, &info, sizeof(info), FileBasicInformation)); } -int lib_wutime(const WCHAR* path, const void* times_ptr) { - HANDLE h = nt_open_file(path, FILE_WRITE_ATTRIBUTES, FILE_SHARE_READ | FILE_SHARE_WRITE, FILE_OPEN, FILE_NON_DIRECTORY_FILE); +static int lib_wutime_impl(const WCHAR* path, int64_t modified_time, int64_t access_time, bool is_dir) { + HANDLE h = nt_open_file(path, FILE_WRITE_ATTRIBUTES, FILE_SHARE_READ | FILE_SHARE_WRITE, FILE_OPEN, + is_dir ? FILE_DIRECTORY_FILE : FILE_NON_DIRECTORY_FILE); if (h == INVALID_HANDLE_VALUE) return -1; - typedef struct { int64_t actime; int64_t modtime; } utimbuf64; - const utimbuf64* times = times_ptr; FILE_BASIC_INFORMATION info = {0}; IO_STATUS_BLOCK iosb; NtQueryInformationFile(h, &iosb, &info, sizeof(info), FileBasicInformation); - if (times) { - info.LastAccessTime.QuadPart = (times->actime + 11644473600LL) * 10000000LL; - info.LastWriteTime.QuadPart = (times->modtime + 11644473600LL) * 10000000LL; - } + info.LastAccessTime.QuadPart = access_time; + info.LastWriteTime.QuadPart = modified_time; NTSTATUS status = NtSetInformationFile(h, &iosb, &info, sizeof(info), FileBasicInformation); NtClose(h); return NT_SUCCESS(status) ? 0 : -1; } -int lib_wutime_dir(const WCHAR* path, const void* times_ptr) { - HANDLE h = nt_open_file(path, FILE_WRITE_ATTRIBUTES, FILE_SHARE_READ | FILE_SHARE_WRITE, FILE_OPEN, FILE_DIRECTORY_FILE); - if (h == INVALID_HANDLE_VALUE) return -1; - typedef struct { int64_t actime; int64_t modtime; } utimbuf64; - const utimbuf64* times = times_ptr; - FILE_BASIC_INFORMATION info = {0}; IO_STATUS_BLOCK iosb; - NtQueryInformationFile(h, &iosb, &info, sizeof(info), FileBasicInformation); - if (times) { - info.LastAccessTime.QuadPart = (times->actime + 11644473600LL) * 10000000LL; - info.LastWriteTime.QuadPart = (times->modtime + 11644473600LL) * 10000000LL; - } - NTSTATUS status = NtSetInformationFile(h, &iosb, &info, sizeof(info), FileBasicInformation); - NtClose(h); - return NT_SUCCESS(status) ? 0 : -1; -} +int lib_wutime(const WCHAR* path, int64_t modified_time, int64_t access_time) { return lib_wutime_impl(path, modified_time, access_time, false); } +int lib_wutime_dir(const WCHAR* path, int64_t modified_time, int64_t access_time) { return lib_wutime_impl(path, modified_time, access_time, true); } #else @@ -1000,6 +1065,12 @@ int lib_rmdir(const char* path) { return 0; } +int lib_rename(const char* old_path, const char* new_path) { + long ret = syscall4(SYS_renameat, AT_FDCWD, (long)old_path, AT_FDCWD, (long)new_path); + if (ret < 0) { lib_errno_val = (int)(-ret); return -1; } + return 0; +} + typedef struct { int fd; char dir_path[1024]; diff --git a/src/main.c b/src/main.c index d2baa3e..341df58 100644 --- a/src/main.c +++ b/src/main.c @@ -2,7 +2,6 @@ #include "bootid.h" #include "crypto.h" #include "aes.h" -#include "progress.h" #include "exfat.h" #include "ntfs.h" #include "error.h" @@ -12,16 +11,38 @@ #define PAGE_SIZE 4096 #define BUFFER_SIZE (PAGE_SIZE * 256) #define MAX_PATH_LENGTH 256 -#define VERSION "2026020601" +#ifndef VERSION +#define VERSION "0000000000" +#endif + +typedef struct { + VHDContext* vhd; + NTFSContext* inner_ntfs; + DecryptStream* inner_stream; + RunSource* inner_run_src; + NTFSContext* outer_ntfs; + DecryptStream* outer_stream; + FILE* file; + char opt_path[MAX_PATH_LENGTH]; +} CachedParentVHD; typedef struct { bool silent; bool verbose; bool extract_fs; bool write_intermediate; + bool keep_versions; char* output_filename; const char* output_dir; const char* parent_file; + CachedParentVHD cached_parent; + char cached_parent_dir[MAX_PATH_LENGTH]; + bool cached_parent_consumed; + bool caching_parent; + bool stacked_to_parent; + char cached_parent_output_dir[MAX_PATH_LENGTH]; + char stacked_final_name[MAX_PATH_LENGTH]; + char stacked_final_inner[MAX_PATH_LENGTH]; uint64_t total_files_extracted; uint64_t total_bytes_extracted; time_t start_time; @@ -43,6 +64,27 @@ typedef struct { #define PRINT(ctx, ...) do { if (!(ctx)->silent) printf(__VA_ARGS__); } while(0) #define VERBOSE(ctx, ...) do { if ((ctx)->verbose && !(ctx)->silent) printf(__VA_ARGS__); } while(0) +static inline bool validate_bootid_offsets(const BootId* b, uint64_t* out_offset, uint64_t* out_size) { + if (b->header_block_count > b->block_count) return false; + if (b->block_size > 0 && b->block_count > ((uint64_t)-1) / b->block_size) return false; + *out_offset = b->header_block_count * b->block_size; + *out_size = (b->block_count - b->header_block_count) * b->block_size; + return true; +} + +static void free_cached_parent(CachedParentVHD* cp) { + if (!cp->vhd) return; + vhd_close(cp->vhd); free(cp->vhd); + if (cp->inner_ntfs) { ntfs_close(cp->inner_ntfs); free(cp->inner_ntfs); } + free(cp->inner_stream); + free(cp->inner_run_src); + if (cp->outer_ntfs) { ntfs_close(cp->outer_ntfs); free(cp->outer_ntfs); } + free(cp->outer_stream); + if (cp->file) fclose(cp->file); + if (cp->opt_path[0]) REMOVE(cp->opt_path); + memset(cp, 0, sizeof(*cp)); +} + static inline void path_join(char* dest, size_t size, const char* dir, const char* name) { if (dir) { snprintf(dest, size, "%s%s%s", dir, PATH_SEPARATOR, name); @@ -51,6 +93,32 @@ static inline void path_join(char* dest, size_t size, const char* dir, const cha } } +static void remove_dir_tree(const char* dir_path) { + char search[MAX_PATH_LENGTH]; + snprintf(search, sizeof(search), "%s%s*", dir_path, PATH_SEPARATOR); + + lib_finddata_t fd; + intptr_t h = _findfirst(search, &fd); + if (h == -1) { RMDIR(dir_path); return; } + + do { + if (fd.name[0] == '.' && (fd.name[1] == '\0' || + (fd.name[1] == '.' && fd.name[2] == '\0'))) continue; + + char full[MAX_PATH_LENGTH]; + path_join(full, sizeof(full), dir_path, fd.name); + + if (fd.attrib & _A_SUBDIR) { + remove_dir_tree(full); + } else { + REMOVE(full); + } + } while (_findnext(h, &fd) == 0); + + _findclose(h); + RMDIR(dir_path); +} + static ErrorCode do_file(AppContext* app_ctx, const char* path); static bool test_keys(FILE* file, uint64_t data_offset, const uint8_t* test_key, @@ -59,17 +127,17 @@ static bool test_keys(FILE* file, uint64_t data_offset, const uint8_t* test_key, uint8_t page_iv[16]; uint8_t decrypted[16]; - long saved_pos = ftell(file); - if (fseek(file, (long)data_offset, SEEK_SET) != 0) { - fseek(file, saved_pos, SEEK_SET); + int64_t saved_pos = FTELLO(file); + if (FSEEKO(file, data_offset, SEEK_SET) != 0) { + FSEEKO(file, saved_pos, SEEK_SET); return false; } if (fread(buffer, 1, 16, file) != 16) { - fseek(file, saved_pos, SEEK_SET); + FSEEKO(file, saved_pos, SEEK_SET); return false; } - fseek(file, saved_pos, SEEK_SET); + FSEEKO(file, saved_pos, SEEK_SET); iv_page(0, test_iv, page_iv); @@ -112,8 +180,7 @@ static ErrorCode parse_bootid(AppContext* app_ctx, FILE* file, DecryptInfo* info info->is_apm3 = (info->bootid.container_type == CONTAINER_TYPE_OPTION) && IS_APM3_OPTION(info->bootid.game_id); info->is_inner_apm3 = false; - info->data_offset = info->bootid.header_block_count * info->bootid.block_size; - info->data_size = (info->bootid.block_count - info->bootid.header_block_count) * info->bootid.block_size; + if (!validate_bootid_offsets(&info->bootid, &info->data_offset, &info->data_size)) return ERR_INVALID_BOOTID; const char* id = (info->bootid.container_type == CONTAINER_TYPE_OS) ? info->os_id : info->game_id; @@ -132,8 +199,6 @@ static ErrorCode parse_bootid(AppContext* app_ctx, FILE* file, DecryptInfo* info if (got_keys) key_source = keys.external ? "ext" : "int"; } else if (info->is_apm3) { memcpy(keys.key, OPTION_KEY, 16); - memcpy(keys.iv, OPTION_IV, 16); - keys.has_iv = true; got_keys = true; key_source = "apm3"; } else { @@ -141,14 +206,11 @@ static ErrorCode parse_bootid(AppContext* app_ctx, FILE* file, DecryptInfo* info if (key_derive(info->game_id, derived_key, derived_iv) && test_keys(file, info->data_offset, derived_key, derived_iv, NTFS_HEADER)) { memcpy(keys.key, derived_key, 16); - memcpy(keys.iv, derived_iv, 16); - keys.has_iv = true; got_keys = true; info->is_inner_apm3 = true; key_source = "derived"; } else { memcpy(keys.key, OPTION_KEY, 16); - keys.has_iv = false; got_keys = true; key_source = "optkey"; } @@ -162,26 +224,16 @@ static ErrorCode parse_bootid(AppContext* app_ctx, FILE* file, DecryptInfo* info VERBOSE(app_ctx, " key:%s\n", key_source); memcpy(info->key, keys.key, 16); - bool has_iv = !info->bootid.use_custom_iv && keys.has_iv; - if (has_iv) { - memcpy(info->iv, keys.iv, 16); - VERBOSE(app_ctx, " iv:key\n"); - } else { - if (fseek(file, (long)info->data_offset, SEEK_SET) != 0) { - FAIL(ERR_FILE_SEEK); - return ERR_FILE_SEEK; - } - if (fread(read_buffer, 1, PAGE_SIZE, file) != PAGE_SIZE) { - FAIL(ERR_FILE_READ); - return ERR_FILE_READ; - } - const uint8_t* header = (info->bootid.container_type == CONTAINER_TYPE_OPTION && !info->is_apm3 && !info->is_inner_apm3) ? EXFAT_HEADER : NTFS_HEADER; - if (!iv_file(info->key, header, read_buffer, info->iv)) { - FAIL(ERR_IV_CALCULATION); - return ERR_IV_CALCULATION; - } - VERBOSE(app_ctx, " iv:calc\n"); + if (FSEEKO(file, info->data_offset, SEEK_SET) != 0) { + FAIL(ERR_FILE_SEEK); + return ERR_FILE_SEEK; } + if (fread(read_buffer, 1, 16, file) != 16) { + FAIL(ERR_FILE_READ); + return ERR_FILE_READ; + } + const uint8_t* header = (info->bootid.container_type == CONTAINER_TYPE_OPTION && !info->is_apm3 && !info->is_inner_apm3) ? EXFAT_HEADER : NTFS_HEADER; + iv_file(info->key, header, read_buffer, info->iv); return ERR_OK; } @@ -229,27 +281,17 @@ static void copy_runs_from_opt(RunSource* dst, const PendingOpt* opt, void* ntfs dst->ntfs_ctx = ntfs_ctx; dst->run_count = (opt->run_count < MAX_DATA_RUNS) ? opt->run_count : MAX_DATA_RUNS; dst->file_size = opt->file_size; - for (int r = 0; r < dst->run_count; r++) { - dst->runs[r].offset = opt->runs[r].offset; - dst->runs[r].length = opt->runs[r].length; - } + memcpy(dst->runs, opt->runs, dst->run_count * sizeof(DataRun)); } #define EXTRACT_F_ALLOW_VHD 0x01 #define EXTRACT_F_RMDIR_ON_ORPHAN 0x02 -#define EXTRACT_F_CHECK_DIFF_PARENT 0x04 static void strip_extension(char* path) { char* ext = strrchr(path, '.'); if (ext) *ext = '\0'; } -static const char* get_basename(const char* path) { - const char* b = strrchr(path, '/'); - if (!b) b = strrchr(path, '\\'); - return b ? b + 1 : path; -} - static void do_inner_opts(AppContext* app_ctx, const char* dir_path); static bool derive_inner_keys(const char* game_id, const uint8_t* first_page, uint8_t* out_key, uint8_t* out_iv, FsType* out_fs); @@ -269,7 +311,7 @@ static void process_pending_opts(AppContext* app_ctx, NTFSContext* ctx, const ch copy_runs_from_opt(&run_src, opt, ctx); uint8_t bootid_enc[96]; - if (!stream_read_raw(ctx, run_src.runs, run_src.run_count, + if (!ntfs_read_from_runs(ctx, run_src.runs, run_src.run_count, run_src.file_size, 0, bootid_enc, 96)) { fprintf(stderr, "optrd\n"); continue; @@ -283,11 +325,11 @@ static void process_pending_opts(AppContext* app_ctx, NTFSContext* ctx, const ch memcpy(inner_game_id, inner_bootid.game_id, 4); inner_game_id[4] = '\0'; - uint64_t inner_data_offset = inner_bootid.header_block_count * inner_bootid.block_size; - uint64_t inner_data_size = (inner_bootid.block_count - inner_bootid.header_block_count) * inner_bootid.block_size; + uint64_t inner_data_offset, inner_data_size; + if (!validate_bootid_offsets(&inner_bootid, &inner_data_offset, &inner_data_size)) continue; uint8_t first_page[PAGE_SIZE]; - if (!stream_read_raw(ctx, run_src.runs, run_src.run_count, + if (!ntfs_read_from_runs(ctx, run_src.runs, run_src.run_count, run_src.file_size, inner_data_offset, first_page, PAGE_SIZE)) { fprintf(stderr, "pgrd\n"); continue; @@ -328,51 +370,38 @@ static void process_pending_opts(AppContext* app_ctx, NTFSContext* ctx, const ch extract_stream_fs(app_ctx, inner_stream, inner_output_dir, inner_fs, EXTRACT_F_ALLOW_VHD | EXTRACT_F_RMDIR_ON_ORPHAN); - free(inner_run_src); - free(inner_stream); + if (app_ctx->cached_parent.inner_ntfs && + app_ctx->cached_parent.inner_ntfs->stream == inner_stream) { + app_ctx->cached_parent.inner_stream = inner_stream; + app_ctx->cached_parent.inner_run_src = inner_run_src; + } else { + free(inner_run_src); + free(inner_stream); + } } } -static int find_highest_vhd(const char* dir) { - char vhd_path[MAX_PATH_LENGTH]; - int highest = -1; - for (int n = 0; n < 10; n++) { - snprintf(vhd_path, sizeof(vhd_path), "%s%sinternal_%d.vhd", dir, PATH_SEPARATOR, n); - FILE* f = FOPEN(vhd_path, "rb"); - if (f) { fclose(f); highest = n; } - } - return highest; -} - static bool extract_file_fs(AppContext* app_ctx, const char* filepath, const char* out_dir, FsType fs_type, uint32_t flags) { bool ok = false; if (fs_type == FS_NTFS) { - NTFSContext ctx = {0}; - ctx.silent = app_ctx->silent; - ctx.verbose = app_ctx->verbose; - - if (ntfs_init(&ctx, filepath, out_dir)) { - if (flags & EXTRACT_F_CHECK_DIFF_PARENT) { - int vhd_type = ntfs_detect_vhd_type(&ctx); - if (vhd_type == VHD_TYPE_DIFFERENCING && !app_ctx->parent_file) { - PRINT(app_ctx, "\ndiff VHD, use -p\n"); - ntfs_close(&ctx); - return true; - } - } + NTFSContext* ctx = calloc(1, sizeof(NTFSContext)); + if (!ctx) return false; + ctx->silent = app_ctx->silent; + ctx->verbose = app_ctx->verbose; + if (ntfs_init(ctx, filepath, out_dir)) { VERBOSE(app_ctx, "ntfs MFT=%llu c=%u\n", - (unsigned long long)ctx.total_mft_records, ctx.bytes_per_cluster); + (unsigned long long)ctx->total_mft_records, ctx->bytes_per_cluster); - if (ntfs_extract_all(&ctx)) { + if (ntfs_extract_all(ctx)) { ok = true; - app_ctx->total_files_extracted += ctx.files_extracted; - app_ctx->total_bytes_extracted += ctx.extracted_bytes; + app_ctx->total_files_extracted += ctx->files_extracted; + app_ctx->total_bytes_extracted += ctx->extracted_bytes; if (flags & EXTRACT_F_ALLOW_VHD) { - int highest = find_highest_vhd(out_dir); + int highest = ctx->highest_extracted_vhd; if (highest >= 0) { char vhd_path[MAX_PATH_LENGTH]; snprintf(vhd_path, sizeof(vhd_path), "%s%sinternal_%d.vhd", @@ -381,26 +410,67 @@ static bool extract_file_fs(AppContext* app_ctx, const char* filepath, char vhd_out[MAX_PATH_LENGTH]; snprintf(vhd_out, sizeof(vhd_out), "%s%scontents", out_dir, PATH_SEPARATOR); - NTFSContext vhd_ctx = {0}; - vhd_ctx.silent = app_ctx->silent; - vhd_ctx.verbose = app_ctx->verbose; + bool cached = false; + if (app_ctx->caching_parent) { + VHDContext* base = malloc(sizeof(VHDContext)); + if (base) { + if (vhd_init_internal(base, vhd_path, 0) && + base->footer.disk_type == VHD_TYPE_DYNAMIC) { + free_cached_parent(&app_ctx->cached_parent); + app_ctx->cached_parent.vhd = base; + strncpy(app_ctx->cached_parent_dir, vhd_out, MAX_PATH_LENGTH - 1); + app_ctx->cached_parent_dir[MAX_PATH_LENGTH - 1] = '\0'; + app_ctx->cached_parent_consumed = false; + cached = true; + } else { + vhd_close(base); free(base); + } + } + } - if (ntfs_init(&vhd_ctx, vhd_path, vhd_out)) { - VERBOSE(app_ctx, "vhd MFT=%llu c=%u\n", - (unsigned long long)vhd_ctx.total_mft_records, vhd_ctx.bytes_per_cluster); - if (ntfs_extract_all(&vhd_ctx)) { - app_ctx->total_files_extracted += vhd_ctx.files_extracted; - app_ctx->total_bytes_extracted += vhd_ctx.extracted_bytes; + if (!cached) { + NTFSContext* vhd_ctx = calloc(1, sizeof(NTFSContext)); + if (vhd_ctx) { + vhd_ctx->silent = app_ctx->silent; + vhd_ctx->verbose = app_ctx->verbose; + + VHDContext* pv = app_ctx->cached_parent.vhd; + const char* vhd_target = (pv && !app_ctx->keep_versions) ? app_ctx->cached_parent_dir : vhd_out; + bool init_ok = pv ? + ntfs_init_vhd(vhd_ctx, vhd_path, vhd_target, pv) : + ntfs_init(vhd_ctx, vhd_path, vhd_target); + bool used_cached = init_ok && pv && vhd_ctx->vhd.parent == pv; + + if (init_ok) { + VERBOSE(app_ctx, "vhd MFT=%llu c=%u\n", + (unsigned long long)vhd_ctx->total_mft_records, vhd_ctx->bytes_per_cluster); + if (ntfs_extract_all(vhd_ctx)) { + app_ctx->total_files_extracted += vhd_ctx->files_extracted; + app_ctx->total_bytes_extracted += vhd_ctx->extracted_bytes; + } + if (used_cached) { + vhd_ctx->vhd.parent = NULL; + app_ctx->cached_parent_consumed = true; + if (!app_ctx->keep_versions) { + app_ctx->stacked_to_parent = true; + const char* ibn = get_basename(out_dir); + strncpy(app_ctx->stacked_final_inner, ibn, MAX_PATH_LENGTH - 1); + app_ctx->stacked_final_inner[MAX_PATH_LENGTH - 1] = '\0'; + } + } + ntfs_close(vhd_ctx); + } + free(vhd_ctx); } - ntfs_close(&vhd_ctx); } } } do_inner_opts(app_ctx, out_dir); } - ntfs_close(&ctx); + ntfs_close(ctx); } + free(ctx); } else { ExfatContext ctx; if (exfat_init(&ctx, filepath)) { @@ -440,9 +510,7 @@ static bool derive_inner_keys(const char* game_id, const uint8_t* first_page, } memcpy(out_key, OPTION_KEY, 16); - if (!iv_file(out_key, EXFAT_HEADER, first_page, out_iv)) { - return false; - } + iv_file(out_key, EXFAT_HEADER, first_page, out_iv); *out_fs = FS_EXFAT; return true; } @@ -452,43 +520,75 @@ static bool extract_stream_fs(AppContext* app_ctx, DecryptStream* stream, bool ok = false; if (fs_type == FS_NTFS) { - NTFSContext ctx = {0}; - ctx.silent = app_ctx->silent; - ctx.verbose = app_ctx->verbose; - - if (ntfs_init_stream(&ctx, stream, out_dir)) { - if (flags & EXTRACT_F_CHECK_DIFF_PARENT) { - int vhd_type = ntfs_detect_vhd_type(&ctx); - if (vhd_type == VHD_TYPE_DIFFERENCING && !app_ctx->parent_file) { - PRINT(app_ctx, "diff VHD, use -p\n"); - ntfs_close(&ctx); - return true; - } - } + NTFSContext* ctx = calloc(1, sizeof(NTFSContext)); + if (!ctx) return false; + ctx->silent = app_ctx->silent; + ctx->verbose = app_ctx->verbose; + if (ntfs_init_stream(ctx, stream, out_dir)) { VERBOSE(app_ctx, "MFT=%llu c=%u\n", - (unsigned long long)ctx.total_mft_records, ctx.bytes_per_cluster); + (unsigned long long)ctx->total_mft_records, ctx->bytes_per_cluster); - ctx.silent = true; - if (ntfs_extract_all(&ctx)) { - ctx.silent = app_ctx->silent; + ctx->silent = true; + if (ntfs_extract_all(ctx)) { + ctx->silent = app_ctx->silent; + + uint64_t saved_files = ctx->files_extracted; + uint64_t saved_bytes = ctx->extracted_bytes; bool is_orphan = false; - if ((flags & EXTRACT_F_ALLOW_VHD) && ctx.pending_vhd_count > 0) { - ntfs_extract_pending_vhds(&ctx, app_ctx->silent, app_ctx->verbose, &is_orphan); + if ((flags & EXTRACT_F_ALLOW_VHD) && ctx->pending_vhd_count > 0) { + VHDContext* base_vhd = NULL; + VHDContext* pv = app_ctx->cached_parent.vhd; + bool want_cache = app_ctx->caching_parent && !pv; + + if (pv && !app_ctx->keep_versions) { + strncpy(ctx->base_path, app_ctx->cached_parent_dir, + sizeof(ctx->base_path) - 1); + ctx->base_path[sizeof(ctx->base_path) - 1] = '\0'; + } + + ntfs_extract_pending_vhds(ctx, app_ctx->silent, app_ctx->verbose, + app_ctx->parent_file, pv, + &is_orphan, want_cache ? &base_vhd : NULL); + + if (base_vhd) { + free_cached_parent(&app_ctx->cached_parent); + base_vhd->run_source->ntfs_ctx = ctx; + app_ctx->cached_parent.vhd = base_vhd; + app_ctx->cached_parent.inner_ntfs = ctx; + strncpy(app_ctx->cached_parent_dir, out_dir, MAX_PATH_LENGTH - 1); + app_ctx->cached_parent_dir[MAX_PATH_LENGTH - 1] = '\0'; + app_ctx->cached_parent_consumed = false; + ctx = NULL; + } + + if (pv && !is_orphan) { + app_ctx->cached_parent_consumed = true; + if (!app_ctx->keep_versions) { + app_ctx->stacked_to_parent = true; + const char* ibn = get_basename(out_dir); + strncpy(app_ctx->stacked_final_inner, ibn, MAX_PATH_LENGTH - 1); + app_ctx->stacked_final_inner[MAX_PATH_LENGTH - 1] = '\0'; + } + } } if ((flags & EXTRACT_F_RMDIR_ON_ORPHAN) && is_orphan) { - VERBOSE(app_ctx, " orphan\n"); + PRINT(app_ctx, " orphan, use -p\n"); RMDIR(out_dir); } else { ok = true; - app_ctx->total_files_extracted += ctx.files_extracted; - app_ctx->total_bytes_extracted += ctx.extracted_bytes; + app_ctx->total_files_extracted += saved_files; + app_ctx->total_bytes_extracted += saved_bytes; } + + if (ctx) ntfs_close(ctx); + } else { + ntfs_close(ctx); } - ntfs_close(&ctx); } + free(ctx); } else { ExfatContext ctx; if (exfat_init_stream(&ctx, stream)) { @@ -550,11 +650,14 @@ static bool do_inner_opt(AppContext* app_ctx, const char* opt_path, const char* memcpy(inner_game_id, bootid.game_id, 4); inner_game_id[4] = '\0'; - uint64_t data_offset = bootid.header_block_count * bootid.block_size; - uint64_t data_size = (bootid.block_count - bootid.header_block_count) * bootid.block_size; + uint64_t data_offset, data_size; + if (!validate_bootid_offsets(&bootid, &data_offset, &data_size)) { + fclose(file); + return false; + } uint8_t first_page[PAGE_SIZE]; - if (fseek(file, (long)data_offset, SEEK_SET) != 0 || + if (FSEEKO(file, data_offset, SEEK_SET) != 0 || fread(first_page, 1, PAGE_SIZE, file) != PAGE_SIZE) { fclose(file); return false; @@ -582,8 +685,16 @@ static bool do_inner_opt(AppContext* app_ctx, const char* opt_path, const char* bool success = extract_stream_fs(app_ctx, stream, output_dir, fs_type, EXTRACT_F_ALLOW_VHD | EXTRACT_F_RMDIR_ON_ORPHAN); - free(stream); - fclose(file); + if (app_ctx->cached_parent.inner_ntfs && + app_ctx->cached_parent.inner_ntfs->stream == stream) { + app_ctx->cached_parent.inner_stream = stream; + app_ctx->cached_parent.file = file; + strncpy(app_ctx->cached_parent.opt_path, opt_path, MAX_PATH_LENGTH - 1); + app_ctx->cached_parent.opt_path[MAX_PATH_LENGTH - 1] = '\0'; + } else { + free(stream); + fclose(file); + } return success; } @@ -682,7 +793,7 @@ ErrorCode do_file(AppContext* app_ctx, const char* path) { goto cleanup; } - if (fseek(file, (long)info.data_offset, SEEK_SET) != 0) { + if (FSEEKO(file, info.data_offset, SEEK_SET) != 0) { FAIL(ERR_FILE_SEEK); result = ERR_FILE_SEEK; goto cleanup; @@ -691,11 +802,6 @@ ErrorCode do_file(AppContext* app_ctx, const char* path) { AES_ctx page_ctx; AES_init_ctx_iv(&page_ctx, info.key, info.iv); - Progress progress; - if (!app_ctx->silent) { - progress_init(&progress, info.data_size); - } - uint64_t total_bytes_read = 0; uint64_t bytes_remaining = info.data_size; @@ -732,14 +838,8 @@ ErrorCode do_file(AppContext* app_ctx, const char* path) { total_bytes_read += read_size; bytes_remaining -= read_size; - if (!app_ctx->silent) { - progress_update(&progress, total_bytes_read); - } } - if (!app_ctx->silent) { - progress_finish(&progress); - } PRINT(app_ctx, "ok: %s\n", output_filename); if (app_ctx->extract_fs) { @@ -795,6 +895,11 @@ static ErrorCode do_stream(AppContext* app_ctx, const char* path) { format_basename(&info, basename_no_ext, sizeof(basename_no_ext)); path_join(output_dir, sizeof(output_dir), app_ctx->output_dir, basename_no_ext); + if (app_ctx->caching_parent) { + strncpy(app_ctx->cached_parent_output_dir, output_dir, MAX_PATH_LENGTH - 1); + app_ctx->cached_parent_output_dir[MAX_PATH_LENGTH - 1] = '\0'; + } + bool extraction_success = false; if (info.bootid.container_type == CONTAINER_TYPE_OPTION && !info.is_apm3 && !info.is_inner_apm3) { @@ -810,58 +915,92 @@ static ErrorCode do_stream(AppContext* app_ctx, const char* path) { inner_game_id[4] = '\0'; VERBOSE(app_ctx, "apm3:%s\n", inner_game_id); - NTFSContext ctx = { 0 }; - ctx.silent = app_ctx->silent; - ctx.verbose = app_ctx->verbose; - ctx.apm3_decrypt = false; - - if (ntfs_init_stream(&ctx, stream, output_dir)) { - int vhd_type = ntfs_detect_vhd_type(&ctx); - if (vhd_type == VHD_TYPE_DIFFERENCING && !app_ctx->parent_file) { - PRINT(app_ctx, "diff VHD, use -p\n"); - ntfs_close(&ctx); - result = ERR_OK; - goto cleanup; - } + NTFSContext* ctx = calloc(1, sizeof(NTFSContext)); + if (ctx) { + ctx->silent = app_ctx->silent; + ctx->verbose = app_ctx->verbose; + if (ntfs_init_stream(ctx, stream, output_dir)) { VERBOSE(app_ctx, "MFT=%llu c=%u\n", - (unsigned long long)ctx.total_mft_records, ctx.bytes_per_cluster); + (unsigned long long)ctx->total_mft_records, ctx->bytes_per_cluster); - ctx.silent = true; + ctx->silent = true; - if (ntfs_extract_all(&ctx)) { + if (ntfs_extract_all(ctx)) { extraction_success = true; - ctx.silent = app_ctx->silent; + ctx->silent = app_ctx->silent; + + if (ctx->pending_vhd_count > 0) { + if (app_ctx->cached_parent.vhd && !app_ctx->keep_versions) { + strncpy(ctx->base_path, app_ctx->cached_parent_dir, + sizeof(ctx->base_path) - 1); + ctx->base_path[sizeof(ctx->base_path) - 1] = '\0'; + } - if (ctx.pending_vhd_count > 0) { bool is_orphan = false; - ntfs_extract_pending_vhds(&ctx, app_ctx->silent, app_ctx->verbose, &is_orphan); + ntfs_extract_pending_vhds(ctx, app_ctx->silent, app_ctx->verbose, + app_ctx->parent_file, app_ctx->cached_parent.vhd, + &is_orphan, NULL); + if (app_ctx->cached_parent.vhd && !is_orphan) { + app_ctx->cached_parent_consumed = true; + if (!app_ctx->keep_versions) + app_ctx->stacked_to_parent = true; + } + if (is_orphan) { + PRINT(app_ctx, " orphan, use -p\n"); + } } - app_ctx->total_files_extracted += ctx.files_extracted; - app_ctx->total_bytes_extracted += ctx.extracted_bytes; + app_ctx->total_files_extracted += ctx->files_extracted; + app_ctx->total_bytes_extracted += ctx->extracted_bytes; - if (info.is_apm3 && ctx.pending_opt_count > 0) { - process_pending_opts(app_ctx, &ctx, output_dir); + if (info.is_apm3 && ctx->pending_opt_count > 0) { + process_pending_opts(app_ctx, ctx, output_dir); } - ntfs_close(&ctx); + if (app_ctx->cached_parent.inner_run_src && + app_ctx->cached_parent.inner_run_src->ntfs_ctx == ctx) { + app_ctx->cached_parent.outer_ntfs = ctx; + app_ctx->cached_parent.outer_stream = stream; + app_ctx->cached_parent.file = file; + stream = NULL; + file = NULL; + ctx = NULL; + } else { + ntfs_close(ctx); + } } else { fprintf(stderr, "ntfs\n"); - ntfs_close(&ctx); + ntfs_close(ctx); } } else { fprintf(stderr, "ntfsi\n"); } + free(ctx); + } } else { extraction_success = extract_stream_fs(app_ctx, stream, output_dir, FS_NTFS, - EXTRACT_F_ALLOW_VHD | EXTRACT_F_CHECK_DIFF_PARENT); + EXTRACT_F_ALLOW_VHD); + + if (app_ctx->cached_parent.inner_ntfs && + app_ctx->cached_parent.inner_ntfs->stream == stream) { + app_ctx->cached_parent.inner_stream = stream; + app_ctx->cached_parent.file = file; + stream = NULL; + file = NULL; + } } if (extraction_success) { + if (app_ctx->stacked_to_parent) { + remove_dir_tree(output_dir); + strncpy(app_ctx->stacked_final_name, output_dir, MAX_PATH_LENGTH - 1); + app_ctx->stacked_final_name[MAX_PATH_LENGTH - 1] = '\0'; + app_ctx->stacked_to_parent = false; + } result = ERR_OK; } @@ -908,7 +1047,6 @@ static bool get_parent_process_name(WCHAR* out_name, size_t max_chars) { ULONG buf_size = 1024 * 1024; uint8_t* buf = NULL; - #define STATUS_INFO_LENGTH_MISMATCH ((NTSTATUS)0xC0000004L) for (int attempt = 0; attempt < 3; attempt++) { buf = lib_malloc(buf_size); if (!buf) return false; @@ -1006,13 +1144,9 @@ static bool from_explorer(void) { #endif static void show_usage(void) { - puts("unsegaREBORN [flags] \n-o dir -n -w -p parent -s -v -vn"); + puts("unsegaREBORN [flags] \n-o dir -n -w -p parent -k -s -v -vn"); } -#ifdef PLATFORM_WINDOWS -__declspec(dllimport) NTSTATUS __stdcall NtDelayExecution(uint8_t Alertable, LARGE_INTEGER* DelayInterval); -#endif - static void wait_for_enter(void) { #ifdef PLATFORM_WINDOWS extern HANDLE lib_stdin_handle; @@ -1072,6 +1206,7 @@ int lib_main(int argc, char** argv) { else if (strcmp(a, "-w") == 0) app_ctx.write_intermediate = true; else if (strcmp(a, "-o") == 0 && i + 1 < argc) app_ctx.output_dir = argv[++i]; else if (strcmp(a, "-p") == 0 && i + 1 < argc) app_ctx.parent_file = argv[++i]; + else if (strcmp(a, "-k") == 0 || strcmp(a, "--keep") == 0) app_ctx.keep_versions = true; continue; } input_files[input_file_count++] = a; @@ -1080,6 +1215,63 @@ int lib_main(int argc, char** argv) { if (input_file_count == 0) { fprintf(stderr, "no files\n"); return 1; } if (!key_any()) fprintf(stderr, "no keys\n"); + for (int i = 1; i < input_file_count; i++) { + for (int j = i; j > 0; j--) { + if (strcmp(get_basename(input_files[j-1]), get_basename(input_files[j])) > 0) { + const char* tmp = input_files[j-1]; + input_files[j-1] = input_files[j]; + input_files[j] = tmp; + } else break; + } + } + + if (!app_ctx.parent_file && input_file_count > 1) { + app_ctx.parent_file = input_files[0]; + for (int i = 0; i < input_file_count - 1; i++) + input_files[i] = input_files[i + 1]; + input_file_count--; + } + + if (app_ctx.parent_file) { + app_ctx.caching_parent = true; + ErrorCode perr = do_file(&app_ctx, app_ctx.parent_file); + app_ctx.caching_parent = false; + app_ctx.parent_file = NULL; + + if (perr == ERR_OK && app_ctx.write_intermediate && + app_ctx.extract_fs && app_ctx.output_filename) { + const char* pbn = get_basename(app_ctx.output_filename); + char pbn_no_ext[MAX_PATH_LENGTH]; + strncpy(pbn_no_ext, pbn, sizeof(pbn_no_ext) - 1); + pbn_no_ext[sizeof(pbn_no_ext) - 1] = '\0'; + strip_extension(pbn_no_ext); + + char parent_out[MAX_PATH_LENGTH]; + if (app_ctx.output_dir) + path_join(parent_out, sizeof(parent_out), app_ctx.output_dir, pbn_no_ext); + else { + strncpy(parent_out, app_ctx.output_filename, sizeof(parent_out) - 1); + parent_out[sizeof(parent_out) - 1] = '\0'; + strip_extension(parent_out); + } + + FsType pfs = strstr(app_ctx.output_filename, ".exfat") ? FS_EXFAT : FS_NTFS; + strncpy(app_ctx.cached_parent_output_dir, parent_out, MAX_PATH_LENGTH - 1); + app_ctx.cached_parent_output_dir[MAX_PATH_LENGTH - 1] = '\0'; + app_ctx.caching_parent = true; + extract_file_fs(&app_ctx, app_ctx.output_filename, parent_out, pfs, + EXTRACT_F_ALLOW_VHD); + app_ctx.caching_parent = false; + + free(app_ctx.output_filename); + app_ctx.output_filename = NULL; + } + + if (perr != ERR_OK || !app_ctx.cached_parent.vhd) { + fprintf(stderr, "parent failed\n"); + } + } + bool any_failed = false; for (int i = 0; i < input_file_count; ++i) { const char* file_path = input_files[i]; @@ -1103,7 +1295,14 @@ int lib_main(int argc, char** argv) { FsType fs_type = strstr(app_ctx.output_filename, ".exfat") ? FS_EXFAT : FS_NTFS; extract_file_fs(&app_ctx, app_ctx.output_filename, output_dir, fs_type, - EXTRACT_F_ALLOW_VHD | EXTRACT_F_CHECK_DIFF_PARENT); + EXTRACT_F_ALLOW_VHD); + + if (app_ctx.stacked_to_parent) { + remove_dir_tree(output_dir); + strncpy(app_ctx.stacked_final_name, output_dir, MAX_PATH_LENGTH - 1); + app_ctx.stacked_final_name[MAX_PATH_LENGTH - 1] = '\0'; + app_ctx.stacked_to_parent = false; + } free(app_ctx.output_filename); app_ctx.output_filename = NULL; @@ -1115,6 +1314,17 @@ int lib_main(int argc, char** argv) { } } + if (app_ctx.cached_parent.vhd) { + if (!app_ctx.cached_parent_consumed || app_ctx.keep_versions) { + uint64_t vhd_files = 0, vhd_bytes = 0; + if (vhd_extract_ntfs(app_ctx.cached_parent.vhd, app_ctx.cached_parent_dir, + app_ctx.silent, app_ctx.verbose, &vhd_files, &vhd_bytes)) { + app_ctx.total_files_extracted += vhd_files; + app_ctx.total_bytes_extracted += vhd_bytes; + } + } + } + if (!app_ctx.silent && app_ctx.total_files_extracted > 0) { char size_buf[32]; fmt_size(app_ctx.total_bytes_extracted, size_buf, sizeof(size_buf)); @@ -1127,5 +1337,27 @@ int lib_main(int argc, char** argv) { free(app_ctx.output_filename); } + free_cached_parent(&app_ctx.cached_parent); + + if (app_ctx.stacked_final_name[0] && app_ctx.cached_parent_output_dir[0]) { + rename(app_ctx.cached_parent_output_dir, app_ctx.stacked_final_name); + + if (app_ctx.stacked_final_inner[0]) { + size_t outer_len = strlen(app_ctx.cached_parent_output_dir); + const char* inner_suffix = app_ctx.cached_parent_dir + outer_len; + + char old_inner[MAX_PATH_LENGTH]; + snprintf(old_inner, sizeof(old_inner), "%s%s", + app_ctx.stacked_final_name, inner_suffix); + + char new_inner[MAX_PATH_LENGTH]; + path_join(new_inner, sizeof(new_inner), + app_ctx.stacked_final_name, app_ctx.stacked_final_inner); + + if (strcmp(old_inner, new_inner) != 0) + rename(old_inner, new_inner); + } + } + return any_failed ? 1 : 0; } diff --git a/src/ntfs.c b/src/ntfs.c index c3a740d..61e3a51 100644 --- a/src/ntfs.c +++ b/src/ntfs.c @@ -1,15 +1,13 @@ #include "ntfs.h" #include "exfat.h" -#include "progress.h" #define BUFFER_SIZE (1024 * 1024) uint32_t g_dir_cache[DIR_CACHE_SIZE]; bool g_dir_cache_init; static bool vhd_read(VHDContext* ctx, void* buffer, uint64_t offset, size_t size); -static bool vhd_init_internal(VHDContext* ctx, const char* filename, uint32_t depth); -static void vhd_close(VHDContext* ctx); -static bool read_bytes_from_runs(NTFSContext* ctx, const DataRun* runs, int run_count, +bool vhd_init_internal(VHDContext* ctx, const char* filename, uint32_t depth); +bool ntfs_read_from_runs(NTFSContext* ctx, const DataRun* runs, int run_count, uint64_t file_size, uint64_t read_offset, void* buffer, size_t read_size); static uint32_t swap32(uint32_t value) { @@ -91,6 +89,8 @@ static bool apply_mft_fixups(const NTFSContext* ctx, uint8_t* record_buffer, siz static bool read_file_info(NTFSContext* ctx, uint64_t ref_number, FileInfo* info) { memset(info, 0, sizeof(FileInfo)); + if (ref_number >= ctx->total_mft_records) return false; + uint64_t mft_offset = ctx->mft_offset + (ref_number * ctx->mft_record_size); if (!ctx->lookup_buffer) { @@ -162,7 +162,7 @@ static inline size_t hash_ref(uint64_t ref, size_t capacity) { ref ^= ref >> 33; ref *= 0xc4ceb9fe1a85ec53ULL; ref ^= ref >> 33; - return ref % capacity; + return ref & (capacity - 1); } static bool init_directory_cache(DirectoryCache* cache) { @@ -171,7 +171,7 @@ static bool init_directory_cache(DirectoryCache* cache) { cache->entries = calloc(cache->capacity, sizeof(DirectoryEntry)); if (!cache->entries) return false; - uint64_t hash = 5 % cache->capacity; + uint64_t hash = 5 & (cache->capacity - 1); cache->entries[hash].ref_number = 5; cache->entries[hash].path[0] = '\0'; cache->entries[hash].occupied = true; @@ -195,7 +195,7 @@ static bool resize_directory_cache(DirectoryCache* cache) { if (cache->entries[i].occupied) { size_t idx = hash_ref(cache->entries[i].ref_number, new_capacity); while (new_entries[idx].occupied) { - idx = (idx + 1) % new_capacity; + idx = (idx + 1) & (new_capacity - 1); } new_entries[idx] = cache->entries[i]; } @@ -219,7 +219,7 @@ static bool add_directory_to_cache(DirectoryCache* cache, uint64_t ref_number, c cache->entries[idx].path[MAX_PATH_LENGTH - 1] = '\0'; return true; } - idx = (idx + 1) % cache->capacity; + idx = (idx + 1) & (cache->capacity - 1); } cache->entries[idx].ref_number = ref_number; @@ -236,7 +236,7 @@ static const char* get_cached_path(DirectoryCache* cache, uint64_t ref_number) { if (cache->entries[idx].ref_number == ref_number) { return cache->entries[idx].path; } - idx = (idx + 1) % cache->capacity; + idx = (idx + 1) & (cache->capacity - 1); } return NULL; } @@ -251,6 +251,10 @@ static bool build_path_impl(NTFSContext* ctx, uint64_t ref_number, char* buffer, return true; } + for (int i = 0; i < ctx->skip_ref_count; i++) { + if (ctx->skip_refs[i] == ref_number) return false; + } + const char* cached_path = get_cached_path(&ctx->dir_cache, ref_number); if (cached_path) { strncpy(buffer, cached_path, buffer_size - 1); @@ -354,16 +358,13 @@ static bool extract_data_from_runs(NTFSContext* ctx, const DataRun* runs, int ru total_written += to_read; ctx->extracted_bytes += to_read; - if (ctx->progress) { - progress_update((Progress*)ctx->progress, ctx->extracted_bytes); - } } } return success; } -static bool read_bytes_from_runs(NTFSContext* ctx, const DataRun* runs, int run_count, +bool ntfs_read_from_runs(NTFSContext* ctx, const DataRun* runs, int run_count, uint64_t file_size, uint64_t read_offset, void* buffer, size_t read_size) { if (read_offset + read_size > file_size) { return false; @@ -439,6 +440,20 @@ static int parse_data_runs(const uint8_t* run_list, size_t run_list_len, DataRun return count; } +static int64_t find_ntfs_partition_offset(const uint8_t* mbr) { + if (mbr[510] != 0x55 || mbr[511] != 0xAA) return -1; + for (int i = 0; i < 4; i++) { + const uint8_t* part = mbr + 0x1BE + (i * 16); + if (part[4] == NTFS_PARTITION_TYPE) { + uint32_t lba_start; + memcpy(&lba_start, part + 8, sizeof(uint32_t)); + if (lba_start == 0) continue; + return (int64_t)lba_start * VHD_SECTOR_SIZE; + } + } + return -1; +} + static int parse_internal_vhd_number(const char* filename) { if (strncmp(filename, "internal_", 9) != 0) return -1; @@ -456,54 +471,54 @@ static int parse_internal_vhd_number(const char* filename) { return atoi(num_str); } -static bool store_pending_vhd(NTFSContext* ctx, const MFTRecordHeader* record, int vhd_number) { - if (ctx->pending_vhd_count >= MAX_PENDING_VHDS) return false; - +static bool extract_nonresident_data_runs(const MFTRecordHeader* record, uint32_t mft_record_size, + DataRun* out_runs, int* out_run_count, uint64_t* out_file_size) { + uint32_t safe_used = record->bytes_used; + if (safe_used > mft_record_size) safe_used = mft_record_size; + if (record->attrs_offset >= mft_record_size) return false; const uint8_t* attr = (const uint8_t*)record + record->attrs_offset; - while (attr < (const uint8_t*)record + record->bytes_used) { + while (attr < (const uint8_t*)record + safe_used) { const AttributeHeader* header = (const AttributeHeader*)attr; - if (header->type == 0xFFFFFFFF || header->length == 0) break; - if (header->type == DATA_ATTR && header->name_length == 0 && header->non_resident) { - PendingVHD* pending = &ctx->pending_vhds[ctx->pending_vhd_count]; - const uint8_t* run_list = attr + header->data.non_resident.mapping_pairs_offset; - size_t run_list_len = header->length - header->data.non_resident.mapping_pairs_offset; - pending->run_count = parse_data_runs(run_list, run_list_len, pending->runs, MAX_DATA_RUNS); - pending->file_size = header->data.non_resident.data_size; - pending->vhd_number = vhd_number; - ctx->pending_vhd_count++; - return true; + if (header->data.non_resident.mapping_pairs_offset >= header->length) break; + *out_file_size = header->data.non_resident.data_size; + *out_run_count = parse_data_runs( + attr + header->data.non_resident.mapping_pairs_offset, + header->length - header->data.non_resident.mapping_pairs_offset, + out_runs, MAX_DATA_RUNS); + return *out_run_count > 0; } attr += header->length; } return false; } +static bool store_pending_vhd(NTFSContext* ctx, const MFTRecordHeader* record, int vhd_number) { + if (ctx->pending_vhd_count >= MAX_PENDING_VHDS) return false; + + PendingVHD* pending = &ctx->pending_vhds[ctx->pending_vhd_count]; + if (!extract_nonresident_data_runs(record, ctx->mft_record_size, + pending->runs, &pending->run_count, &pending->file_size)) { + return false; + } + pending->vhd_number = vhd_number; + ctx->pending_vhd_count++; + return true; +} + static bool store_pending_opt(NTFSContext* ctx, const char* filename, const MFTRecordHeader* record) { if (ctx->pending_opt_count >= MAX_PENDING_OPTS) return false; - const uint8_t* attr = (const uint8_t*)record + record->attrs_offset; - while (attr < (const uint8_t*)record + record->bytes_used) { - const AttributeHeader* header = (const AttributeHeader*)attr; - - if (header->type == 0xFFFFFFFF || header->length == 0) break; - - if (header->type == DATA_ATTR && header->name_length == 0 && header->non_resident) { - PendingOpt* pending = &ctx->pending_opts[ctx->pending_opt_count]; - const uint8_t* run_list = attr + header->data.non_resident.mapping_pairs_offset; - size_t run_list_len = header->length - header->data.non_resident.mapping_pairs_offset; - pending->run_count = parse_data_runs(run_list, run_list_len, pending->runs, MAX_DATA_RUNS); - pending->file_size = header->data.non_resident.data_size; - pending->data_offset = ctx->data_start_offset; - strncpy(pending->filename, filename, MAX_FILENAME_LENGTH - 1); - pending->filename[MAX_FILENAME_LENGTH - 1] = '\0'; - ctx->pending_opt_count++; - return true; - } - attr += header->length; + PendingOpt* pending = &ctx->pending_opts[ctx->pending_opt_count]; + if (!extract_nonresident_data_runs(record, ctx->mft_record_size, + pending->runs, &pending->run_count, &pending->file_size)) { + return false; } - return false; + strncpy(pending->filename, filename, MAX_FILENAME_LENGTH - 1); + pending->filename[MAX_FILENAME_LENGTH - 1] = '\0'; + ctx->pending_opt_count++; + return true; } static bool extract_file(NTFSContext* ctx, const MFTRecordHeader* record, @@ -511,9 +526,12 @@ static bool extract_file(NTFSContext* ctx, const MFTRecordHeader* record, const char* ext = strrchr(filename, '.'); bool is_opt_file = ext && strcmp(ext, ".opt") == 0; + int vhd_num = parse_internal_vhd_number(filename); + + if (vhd_num >= 0 && vhd_num > ctx->highest_extracted_vhd) + ctx->highest_extracted_vhd = vhd_num; if (ctx->stream) { - int vhd_num = parse_internal_vhd_number(filename); if (vhd_num >= 0) { return store_pending_vhd(ctx, record, vhd_num); } @@ -524,8 +542,9 @@ static bool extract_file(NTFSContext* ctx, const MFTRecordHeader* record, uint64_t file_size = 0; const uint8_t* data_attr = NULL; + uint32_t safe_bytes = min(record->bytes_used, ctx->mft_record_size); const uint8_t* attr = (const uint8_t*)record + record->attrs_offset; - while (attr < (const uint8_t*)record + record->bytes_used) { + while (attr < (const uint8_t*)record + safe_bytes) { const AttributeHeader* header = (const AttributeHeader*)attr; if (header->type == 0xFFFFFFFF || header->length == 0) break; if (header->type == DATA_ATTR && header->name_length == 0) { @@ -545,21 +564,28 @@ static bool extract_file(NTFSContext* ctx, const MFTRecordHeader* record, strncpy(parent_path, full_path, sizeof(parent_path) - 1); parent_path[sizeof(parent_path) - 1] = '\0'; - char* last_separator = strrchr(parent_path, PATH_SEPARATOR[0]); + char* last_separator = strrchr(parent_path, PATH_SEP_CHAR); if (last_separator) { *last_separator = '\0'; if (strcmp(parent_path, ctx->last_dir) != 0) { create_directories(parent_path); + close_output_dir(ctx->cached_dir); + ctx->cached_dir = open_output_dir(parent_path); strncpy(ctx->last_dir, parent_path, MAX_PATH_LENGTH - 1); ctx->last_dir[MAX_PATH_LENGTH - 1] = '\0'; } } - FILE* out_file; - if (file_size >= 65536) { - out_file = FOPEN_PREALLOC(full_path, file_size); - } else { - out_file = FOPEN(full_path, "wb"); + FILE* out_file = NULL; + if (ctx->cached_dir != INVALID_DIR_HANDLE) { + out_file = fopen_in_dir(ctx->cached_dir, filename, file_size >= 65536 ? file_size : 0); + } + if (!out_file) { + if (file_size >= 65536) { + out_file = FOPEN_PREALLOC(full_path, file_size); + } else { + out_file = FOPEN(full_path, "wb"); + } } if (!out_file) { return false; @@ -569,6 +595,11 @@ static bool extract_file(NTFSContext* ctx, const MFTRecordHeader* record, const AttributeHeader* header = (const AttributeHeader*)data_attr; if (header->non_resident) { + if (header->data.non_resident.mapping_pairs_offset >= header->length) { + fclose(out_file); + REMOVE(full_path); + return false; + } DataRun runs[256]; const uint8_t* run_list = data_attr + header->data.non_resident.mapping_pairs_offset; size_t run_list_len = header->length - header->data.non_resident.mapping_pairs_offset; @@ -577,14 +608,16 @@ static bool extract_file(NTFSContext* ctx, const MFTRecordHeader* record, success = extract_data_from_runs(ctx, runs, run_count, file_size, out_file); } else { + if (header->data.resident.value_offset + header->data.resident.value_length > header->length) { + fclose(out_file); + REMOVE(full_path); + return false; + } const uint8_t* data = data_attr + header->data.resident.value_offset; uint32_t len = header->data.resident.value_length; success = (FWRITE_DIRECT(out_file, data, len) == len); if (success) { ctx->extracted_bytes += len; - if (ctx->progress) { - progress_update((Progress*)ctx->progress, ctx->extracted_bytes); - } } } @@ -627,9 +660,10 @@ static bool process_mft_record(NTFSContext* ctx, const uint8_t* record_data) { uint64_t parent_ref = 0; uint64_t modification_time = 0; uint64_t access_time = 0; + uint32_t file_flags = 0; bool got_filename = false; bool is_directory = (record->flags & MFT_RECORD_IS_DIRECTORY) != 0; - uint64_t record_num = record->record_number & 0xFFFFFFFFFFFF; + uint64_t record_num = *(const uint32_t*)(record_data + 0x2C); const uint8_t* record_end = record_data + bytes_used; const uint8_t* attr = record_data + attrs_offset; @@ -644,6 +678,15 @@ static bool process_mft_record(NTFSContext* ctx, const uint8_t* record_data) { break; } + if (header->type == 0x10 && !header->non_resident) { + uint16_t vo = header->data.resident.value_offset; + uint32_t vl = header->data.resident.value_length; + if (vo + vl <= header->length && vl >= 36) { + const uint8_t* si = attr + vo; + file_flags = *(const uint32_t*)(si + 32); + } + } + if (header->type == FILE_NAME_ATTR && !header->non_resident) { uint16_t value_offset = header->data.resident.value_offset; uint32_t value_length = header->data.resident.value_length; @@ -673,14 +716,28 @@ static bool process_mft_record(NTFSContext* ctx, const uint8_t* record_data) { return true; } + if (is_directory && parent_ref == 5 && (file_flags & 0x04)) { + if (ctx->skip_ref_count < 8) + ctx->skip_refs[ctx->skip_ref_count++] = record_num; + return true; + } + if (!is_safe_path(filename)) { return true; } - + char parent_path[MAX_PATH_LENGTH]; + bool parent_ok = build_path_recursively(ctx, parent_ref, parent_path, sizeof(parent_path)); + if (!parent_ok && parent_ref != 5) { + return true; + } char full_path[MAX_PATH_LENGTH]; - get_full_path(ctx, parent_ref, filename, full_path, sizeof(full_path)); + if (!parent_ok || parent_path[0] == '\0') { + snprintf(full_path, sizeof(full_path), "%s%s%s", ctx->base_path, PATH_SEPARATOR, filename); + } else { + snprintf(full_path, sizeof(full_path), "%s%s%s%s%s", ctx->base_path, PATH_SEPARATOR, parent_path, PATH_SEPARATOR, filename); + } if (is_directory) { if (!create_directories(full_path)) { @@ -690,13 +747,8 @@ static bool process_mft_record(NTFSContext* ctx, const uint8_t* record_data) { if (modification_time != 0) { DeferredDirTime* dirs = (DeferredDirTime*)ctx->deferred_dirs; if (ctx->deferred_count >= ctx->deferred_capacity) { - uint32_t new_cap = ctx->deferred_capacity ? ctx->deferred_capacity * 2 : 256; - DeferredDirTime* new_buf = realloc(dirs, new_cap * sizeof(DeferredDirTime)); - if (new_buf) { - ctx->deferred_dirs = new_buf; - ctx->deferred_capacity = new_cap; - dirs = new_buf; - } + if (grow_deferred_dirs(&dirs, &ctx->deferred_capacity)) + ctx->deferred_dirs = dirs; } if (ctx->deferred_count < ctx->deferred_capacity) { DeferredDirTime* d = &dirs[ctx->deferred_count++]; @@ -721,7 +773,7 @@ static bool process_mft_record(NTFSContext* ctx, const uint8_t* record_data) { static bool vhd_raw_read(VHDContext* ctx, void* buffer, uint64_t offset, size_t size) { if (ctx->run_source) { NTFSContext* ntfs = (NTFSContext*)ctx->run_source->ntfs_ctx; - return read_bytes_from_runs(ntfs, ctx->run_source->runs, ctx->run_source->run_count, + return ntfs_read_from_runs(ntfs, ctx->run_source->runs, ctx->run_source->run_count, ctx->run_source->file_size, offset, buffer, size); } if (FSEEKO(ctx->fp, offset, SEEK_SET) != 0) { @@ -788,7 +840,7 @@ static bool vhd_read_dynamic_block(VHDContext* ctx, uint8_t* buf, uint64_t offse for (uint32_t s = start_sector; s < end_sector; s++) { uint32_t byte_idx = s / 8; uint32_t bit_idx = 7 - (s % 8); - bool sector_present = (ctx->sector_bitmap[byte_idx] >> bit_idx) & 1; + bool sector_present = (byte_idx < ctx->sector_bitmap_size) && ((ctx->sector_bitmap[byte_idx] >> bit_idx) & 1); uint32_t sector_start_in_block = s * VHD_SECTOR_SIZE; uint32_t sector_end_in_block = sector_start_in_block + VHD_SECTOR_SIZE; @@ -853,12 +905,10 @@ static void extract_base_dir(const char* filepath, char* base_dir, size_t base_d } static const char* extract_filename(const char* path) { - const char* last_slash = strrchr(path, '/'); - const char* last_backslash = strrchr(path, '\\'); - const char* filename = path; - if (last_slash && last_slash > filename) filename = last_slash + 1; - if (last_backslash && last_backslash > filename - 1) filename = last_backslash + 1; - return filename; + const char* b = strrchr(path, '/'); + const char* c = strrchr(path, '\\'); + if (c && (!b || c > b)) b = c; + return b ? b + 1 : path; } static bool try_parent_path(const char* base_dir, const char* filename, char* parent_path, size_t path_size) { @@ -912,7 +962,7 @@ static bool resolve_parent_path(VHDContext* ctx, char* parent_path, size_t path_ return false; } -static void vhd_close(VHDContext* ctx) { +void vhd_close(VHDContext* ctx) { if (!ctx) return; if (ctx->parent) { @@ -951,6 +1001,10 @@ static bool vhd_init_dynamic_header(VHDContext* ctx) { ctx->dyn_header.block_size = swap32(ctx->dyn_header.block_size); ctx->dyn_header.parent_timestamp = swap32(ctx->dyn_header.parent_timestamp); + if (ctx->dyn_header.block_size == 0 || + (ctx->dyn_header.block_size & (ctx->dyn_header.block_size - 1)) != 0 || + ctx->dyn_header.block_size > (256ULL << 20)) return false; + size_t bat_size = (size_t)ctx->dyn_header.max_bat_entries * sizeof(uint32_t); if (bat_size == 0 || bat_size > (1ULL << 30)) { @@ -1010,21 +1064,25 @@ static bool vhd_read_footer(VHDContext* ctx) { return true; } -static bool vhd_init_common(VHDContext* ctx, const char* base_dir, uint32_t depth) { +static bool vhd_init_footer_and_header(VHDContext* ctx) { if (!vhd_read_footer(ctx)) { return false; } - - if (ctx->footer.disk_type == VHD_TYPE_DYNAMIC) { + if (ctx->footer.disk_type == VHD_TYPE_DYNAMIC || + ctx->footer.disk_type == VHD_TYPE_DIFFERENCING) { if (!vhd_init_dynamic_header(ctx)) { return false; } } - else if (ctx->footer.disk_type == VHD_TYPE_DIFFERENCING) { - if (!vhd_init_dynamic_header(ctx)) { - return false; - } + return true; +} +static bool vhd_init_common(VHDContext* ctx, const char* base_dir, uint32_t depth) { + if (!vhd_init_footer_and_header(ctx)) { + return false; + } + + if (ctx->footer.disk_type == VHD_TYPE_DIFFERENCING) { char parent_path[MAX_PATH_LENGTH]; if (resolve_parent_path(ctx, parent_path, sizeof(parent_path))) { ctx->parent = malloc(sizeof(VHDContext)); @@ -1042,7 +1100,7 @@ static bool vhd_init_common(VHDContext* ctx, const char* base_dir, uint32_t dept return true; } -static bool vhd_init_internal(VHDContext* ctx, const char* filename, uint32_t depth) { +bool vhd_init_internal(VHDContext* ctx, const char* filename, uint32_t depth) { memset(ctx, 0, sizeof(VHDContext)); ctx->depth = depth; @@ -1065,11 +1123,53 @@ static bool vhd_init_internal(VHDContext* ctx, const char* filename, uint32_t de return true; } -static bool vhd_init(VHDContext* ctx, const char* filename) { - return vhd_init_internal(ctx, filename, 0); +static bool vhd_init_with_parent(VHDContext* ctx, const char* filename, VHDContext* external_parent) { + memset(ctx, 0, sizeof(VHDContext)); + extract_base_dir(filename, ctx->base_dir, sizeof(ctx->base_dir)); + + ctx->fp = FOPEN(filename, "rb"); + if (!ctx->fp) return false; + + if (!vhd_init_footer_and_header(ctx)) { + vhd_close(ctx); + return false; + } + + if (ctx->footer.disk_type == VHD_TYPE_DIFFERENCING) { + bool got_parent = false; + if (external_parent && + memcmp(ctx->dyn_header.parent_id, external_parent->footer.unique_id, 16) == 0) { + ctx->parent = external_parent; + got_parent = true; + } else if (external_parent) { + fprintf(stderr, "parent mismatch\n"); + } + + if (!got_parent) { + char parent_path[MAX_PATH_LENGTH]; + if (resolve_parent_path(ctx, parent_path, sizeof(parent_path))) { + ctx->parent = malloc(sizeof(VHDContext)); + if (ctx->parent) { + if (!vhd_init_internal(ctx->parent, parent_path, 1)) { + free(ctx->parent); + ctx->parent = NULL; + } else { + got_parent = true; + } + } + } + } + + if (!got_parent) { + vhd_close(ctx); + return false; + } + } + + return true; } -static bool vhd_init_from_runs(VHDContext* ctx, NTFSContext* ntfs, const DataRun* runs, +static bool vhd_setup_run_source(VHDContext* ctx, NTFSContext* ntfs, const DataRun* runs, int run_count, uint64_t file_size, const char* base_dir) { memset(ctx, 0, sizeof(VHDContext)); @@ -1085,6 +1185,14 @@ static bool vhd_init_from_runs(VHDContext* ctx, NTFSContext* ntfs, const DataRun ctx->run_source->bytes_per_cluster = ntfs->bytes_per_cluster; memcpy(ctx->run_source->runs, runs, ctx->run_source->run_count * sizeof(DataRun)); strncpy(ctx->base_dir, base_dir, sizeof(ctx->base_dir) - 1); + return true; +} + +static bool vhd_init_from_runs(VHDContext* ctx, NTFSContext* ntfs, const DataRun* runs, + int run_count, uint64_t file_size, const char* base_dir) { + if (!vhd_setup_run_source(ctx, ntfs, runs, run_count, file_size, base_dir)) { + return false; + } if (!vhd_init_common(ctx, base_dir, 0)) { vhd_close(ctx); @@ -1102,17 +1210,29 @@ static bool ntfs_setup_mft(NTFSContext* ctx, uint64_t ntfs_offset) { ctx->bytes_per_sector = ctx->boot.bytes_per_sector; ctx->bytes_per_cluster = (uint32_t)ctx->boot.bytes_per_sector * ctx->boot.sectors_per_cluster; + + if (ctx->bytes_per_sector == 0 || ctx->bytes_per_sector > 4096 || + (ctx->bytes_per_sector & (ctx->bytes_per_sector - 1)) != 0) return false; + if (ctx->boot.sectors_per_cluster == 0 || + (ctx->boot.sectors_per_cluster & (ctx->boot.sectors_per_cluster - 1)) != 0) return false; + ctx->mft_offset = ntfs_offset + (ctx->boot.mft_cluster_number * ctx->bytes_per_cluster); + if (ctx->boot.clusters_per_mft_record < 0 && ctx->boot.clusters_per_mft_record < -31) return false; + if (ctx->boot.clusters_per_mft_record > 0) { ctx->mft_record_size = ctx->boot.clusters_per_mft_record * ctx->bytes_per_cluster; } else { ctx->mft_record_size = 1U << (-ctx->boot.clusters_per_mft_record); } + if (ctx->mft_record_size == 0 || ctx->mft_record_size > 65536) return false; + ctx->file_buffer = malloc(BUFFER_SIZE); if (!ctx->file_buffer) return false; + ctx->cached_dir = INVALID_DIR_HANDLE; ctx->last_dir[0] = '\0'; + ctx->highest_extracted_vhd = -1; uint8_t* mft_record = malloc(ctx->mft_record_size); if (!mft_record) return false; @@ -1129,10 +1249,11 @@ static bool ntfs_setup_mft(NTFSContext* ctx, uint64_t ntfs_offset) { return false; } + uint32_t safe_used = min(record->bytes_used, ctx->mft_record_size); const uint8_t* attr = mft_record + record->attrs_offset; - while (attr < mft_record + record->bytes_used) { + while (attr + sizeof(AttributeHeader) <= mft_record + safe_used) { const AttributeHeader* header = (const AttributeHeader*)attr; - if (header->type == 0xFFFFFFFF || header->length == 0) break; + if (header->type == 0xFFFFFFFF || header->length < sizeof(AttributeHeader)) break; if (header->type == DATA_ATTR && header->name_length == 0) { if (header->non_resident) { ctx->mft_data_size = header->data.non_resident.data_size; @@ -1172,7 +1293,7 @@ bool ntfs_init(NTFSContext* ctx, const char* path, const char* extract_path) { if (fread(signature, 1, 8, fp) == 8 && memcmp(signature, VHD_COOKIE, 8) == 0) { fclose(fp); ctx->is_vhd = true; - if (!vhd_init(&ctx->vhd, path)) { + if (!vhd_init_internal(&ctx->vhd, path, 0)) { free_directory_cache(&ctx->dir_cache); return false; } @@ -1190,23 +1311,12 @@ bool ntfs_init(NTFSContext* ctx, const char* path, const char* extract_path) { if (ctx->is_vhd) { uint8_t sector[VHD_SECTOR_SIZE]; if (ntfs_read(ctx, sector, 0, VHD_SECTOR_SIZE)) { - if (sector[0x1FE] == 0x55 && sector[0x1FF] == 0xAA) { - for (int i = 0; i < 4; i++) { - const uint8_t* part = sector + 0x1BE + (i * 16); - if (part[4] == NTFS_PARTITION_TYPE) { - uint32_t start_sector = - (uint32_t)part[8] | - ((uint32_t)part[9] << 8) | - ((uint32_t)part[10] << 16) | - ((uint32_t)part[11] << 24); - ntfs_offset = (uint64_t)start_sector * VHD_SECTOR_SIZE; - - if (ntfs_read(ctx, sector, ntfs_offset, VHD_SECTOR_SIZE) && - memcmp(sector + 3, NTFS_SIGNATURE, 8) == 0) { - found_ntfs = true; - break; - } - } + int64_t part_offset = find_ntfs_partition_offset(sector); + if (part_offset >= 0) { + ntfs_offset = (uint64_t)part_offset; + if (ntfs_read(ctx, sector, ntfs_offset, VHD_SECTOR_SIZE) && + memcmp(sector + 3, NTFS_SIGNATURE, 8) == 0) { + found_ntfs = true; } } } @@ -1279,208 +1389,52 @@ bool ntfs_init_stream(NTFSContext* ctx, DecryptStream* stream, const char* extra return true; } -static uint64_t get_file_data_size(NTFSContext* ctx, const uint8_t* record_data) { - const MFTRecordHeader* record = (const MFTRecordHeader*)record_data; - - if (memcmp(record->magic, "FILE", 4) != 0 || !(record->flags & MFT_RECORD_IN_USE)) { - return 0; - } - - if (record->flags & MFT_RECORD_IS_DIRECTORY) { - return 0; - } - - char filename[MAX_FILENAME_LENGTH]; - bool got_filename = false; - - const uint8_t* attr = (const uint8_t*)record + record->attrs_offset; - while (attr < (const uint8_t*)record + record->bytes_used) { - const AttributeHeader* header = (const AttributeHeader*)attr; - if (header->type == 0xFFFFFFFF || header->length == 0) break; - - if (header->type == FILE_NAME_ATTR && !header->non_resident) { - const FileNameAttribute* fname = - (const FileNameAttribute*)(attr + header->data.resident.value_offset); - if (fname->namespace != 2) { - fs_name_to_utf8(fname->name, fname->name_length, filename, sizeof(filename)); - got_filename = true; - break; - } - } - attr += header->length; - } - - if (!got_filename || filename[0] == '$' || !is_safe_path(filename)) { - return 0; - } - - attr = (const uint8_t*)record + record->attrs_offset; - while (attr < (const uint8_t*)record + record->bytes_used) { - const AttributeHeader* header = (const AttributeHeader*)attr; - if (header->type == 0xFFFFFFFF || header->length == 0) break; - - if (header->type == DATA_ATTR && header->name_length == 0) { - if (header->non_resident) { - return header->data.non_resident.data_size; - } else { - return header->data.resident.value_length; - } - } - attr += header->length; - } - - return 0; -} - -static void count_total_bytes(NTFSContext* ctx) { - ctx->total_bytes = 0; - - uint8_t* record_buffer = malloc(ctx->mft_record_size); - if (!record_buffer) return; - - uint64_t current_offset = ctx->mft_offset; - uint64_t total_records = ctx->total_mft_records; - - ctx->raw_file_pos = 0; - - for (uint64_t i = 0; i < total_records; i++) { - if (!ntfs_read(ctx, record_buffer, current_offset, ctx->mft_record_size)) { - break; - } - - if (!apply_mft_fixups(ctx, record_buffer, ctx->mft_record_size)) { - current_offset += ctx->mft_record_size; - continue; - } - - ctx->total_bytes += get_file_data_size(ctx, record_buffer); - current_offset += ctx->mft_record_size; - } - - free(record_buffer); -} - -int ntfs_detect_vhd_type(NTFSContext* ctx) { - uint8_t* record_buffer = malloc(ctx->mft_record_size); - if (!record_buffer) return -1; - - uint64_t current_offset = ctx->mft_offset; - uint64_t total_records = ctx->total_mft_records; - int detected_type = -1; - - ctx->raw_file_pos = 0; - - for (uint64_t i = 0; i < total_records && detected_type == -1; i++) { - if (!ntfs_read(ctx, record_buffer, current_offset, ctx->mft_record_size)) { - current_offset += ctx->mft_record_size; - continue; - } - - if (!apply_mft_fixups(ctx, record_buffer, ctx->mft_record_size)) { - current_offset += ctx->mft_record_size; - continue; - } - - const MFTRecordHeader* record = (const MFTRecordHeader*)record_buffer; - if (memcmp(record->magic, "FILE", 4) != 0 || !(record->flags & MFT_RECORD_IN_USE)) { - current_offset += ctx->mft_record_size; - continue; - } - - if (record->flags & MFT_RECORD_IS_DIRECTORY) { - current_offset += ctx->mft_record_size; - continue; - } - - char filename[MAX_FILENAME_LENGTH] = {0}; - const uint8_t* attr = (const uint8_t*)record + record->attrs_offset; - - while (attr < (const uint8_t*)record + record->bytes_used) { - const AttributeHeader* header = (const AttributeHeader*)attr; - if (header->type == 0xFFFFFFFF || header->length == 0) break; - - if (header->type == FILE_NAME_ATTR && !header->non_resident) { - const FileNameAttribute* fname = - (const FileNameAttribute*)(attr + header->data.resident.value_offset); - if (fname->namespace != 2) { - fs_name_to_utf8(fname->name, fname->name_length, filename, sizeof(filename)); - break; - } - } - attr += header->length; - } - - if (strncmp(filename, "internal_", 9) == 0 && - strstr(filename, ".vhd") != NULL) { - - attr = (const uint8_t*)record + record->attrs_offset; - while (attr < (const uint8_t*)record + record->bytes_used) { - const AttributeHeader* header = (const AttributeHeader*)attr; - if (header->type == 0xFFFFFFFF || header->length == 0) break; - - if (header->type == DATA_ATTR && header->name_length == 0 && header->non_resident) { - uint64_t file_size = header->data.non_resident.data_size; - - if (file_size >= 512) { - DataRun runs[256]; - const uint8_t* run_list = attr + header->data.non_resident.mapping_pairs_offset; - size_t run_list_len = header->length - header->data.non_resident.mapping_pairs_offset; - int run_count = parse_data_runs(run_list, run_list_len, runs, 256); - - uint8_t footer[512]; - if (read_bytes_from_runs(ctx, runs, run_count, file_size, - file_size - 512, footer, 512)) { - if (memcmp(footer, "conectix", 8) == 0) { - detected_type = (footer[60] << 24) | (footer[61] << 16) | - (footer[62] << 8) | footer[63]; - } - } - } - break; - } - attr += header->length; - } - } - - current_offset += ctx->mft_record_size; - } - - free(record_buffer); - return detected_type; -} +#define MFT_BATCH_RECORDS 1024 bool ntfs_extract_all(NTFSContext* ctx) { if (!create_directories(ctx->base_path)) return false; - uint8_t* record_buffer = malloc(ctx->mft_record_size); - if (!record_buffer) return false; - - if (!ctx->is_vhd) { - count_total_bytes(ctx); - } + bool ok = true; ctx->extracted_bytes = 0; - if (ctx->verbose && !ctx->silent && ctx->total_bytes > 0) { - printf("%llu B\n", (unsigned long long)ctx->total_bytes); - } - - Progress progress; - if (!ctx->silent && ctx->total_bytes > 0) { - progress_init(&progress, ctx->total_bytes); - ctx->progress = &progress; - } - - uint64_t current_offset = ctx->mft_offset; uint64_t total_records = ctx->total_mft_records; + uint32_t rec_size = ctx->mft_record_size; + size_t batch_buf_size = MFT_BATCH_RECORDS * rec_size; ctx->raw_file_pos = 0; + uint8_t* batch_buf = malloc(batch_buf_size); + if (!batch_buf) return false; + time_t start_time = time(NULL); time_t last_update = start_time; uint64_t last_bytes = 0; + uint64_t mft_offset = ctx->mft_offset; + uint64_t records_left = total_records; - for (uint64_t i = 0; i < total_records; i++) { - if (!ctx->silent && !ctx->progress && ctx->is_vhd) { + while (records_left > 0) { + uint64_t batch_count = (records_left > MFT_BATCH_RECORDS) ? MFT_BATCH_RECORDS : records_left; + size_t batch_bytes = (size_t)(batch_count * rec_size); + + if (!ntfs_read(ctx, batch_buf, mft_offset, batch_bytes)) { + ok = false; + break; + } + + for (uint64_t j = 0; j < batch_count; j++) { + uint8_t* record_data = batch_buf + j * rec_size; + + if (!apply_mft_fixups(ctx, record_data, rec_size)) continue; + + if (memcmp(record_data, "FILE", 4) == 0) { + process_mft_record(ctx, record_data); + } + } + + mft_offset += batch_bytes; + records_left -= batch_count; + + if (!ctx->silent) { time_t now = time(NULL); if (now != last_update) { int64_t elapsed = (int64_t)difftime(now, last_update); @@ -1495,30 +1449,17 @@ bool ntfs_extract_all(NTFSContext* ctx) { last_bytes = ctx->extracted_bytes; } } - - if (!ntfs_read(ctx, record_buffer, current_offset, ctx->mft_record_size)) break; - - if (!apply_mft_fixups(ctx, record_buffer, ctx->mft_record_size)) break; - - const MFTRecordHeader* record = (const MFTRecordHeader*)record_buffer; - if (memcmp(record->magic, "FILE", 4) == 0) { - process_mft_record(ctx, record_buffer); - } - - current_offset += ctx->mft_record_size; } - if (!ctx->silent && !ctx->progress && ctx->is_vhd) { + free(batch_buf); + + if (!ctx->silent && ctx->extracted_bytes > 0) { uint64_t total_mb_w = ctx->extracted_bytes / (1024 * 1024); uint64_t total_mb_f = (ctx->extracted_bytes % (1024 * 1024)) * 100 / (1024 * 1024); - printf("\r%llu.%02llu MB %llu f\n", + printf("\r%llu.%02llu MB %llu f %ds \n", (unsigned long long)total_mb_w, (unsigned long long)total_mb_f, - (unsigned long long)ctx->files_extracted); - } - - if (ctx->progress) { - progress_finish(&progress); - ctx->progress = NULL; + (unsigned long long)ctx->files_extracted, + (int)difftime(time(NULL), start_time)); } if (ctx->pending_vhd_count == 0 && ctx->deferred_dirs) { @@ -1532,47 +1473,21 @@ bool ntfs_extract_all(NTFSContext* ctx) { ctx->deferred_count = 0; ctx->deferred_capacity = 0; - free(record_buffer); - return true; + return ok; } -bool ntfs_extract_pending_vhds(NTFSContext* ctx, bool silent, bool verbose, bool* is_orphan) { - if (is_orphan) *is_orphan = false; - if (ctx->pending_vhd_count == 0) return true; - - int highest_vhd = -1; - PendingVHD* highest = NULL; - for (int i = 0; i < ctx->pending_vhd_count; i++) { - if (ctx->pending_vhds[i].vhd_number > highest_vhd) { - highest_vhd = ctx->pending_vhds[i].vhd_number; - highest = &ctx->pending_vhds[i]; - } - } - - if (!highest) return true; - - VHDContext vhd_ctx = {0}; - if (!vhd_init_from_runs(&vhd_ctx, ctx, highest->runs, highest->run_count, - highest->file_size, ctx->base_path)) { - if (is_orphan) *is_orphan = true; - return true; - } - - if (vhd_ctx.footer.disk_type == VHD_TYPE_DIFFERENCING) { - if (is_orphan) *is_orphan = true; - vhd_close(&vhd_ctx); - return true; - } - +bool vhd_extract_ntfs(VHDContext* vhd, const char* base_path, + bool silent, bool verbose, + uint64_t* out_files, uint64_t* out_bytes) { NTFSContext inner_ctx = {0}; inner_ctx.silent = silent; inner_ctx.verbose = verbose; inner_ctx.is_vhd = true; - inner_ctx.vhd = vhd_ctx; - strncpy(inner_ctx.base_path, ctx->base_path, sizeof(inner_ctx.base_path) - 1); + inner_ctx.vhd = *vhd; + strncpy(inner_ctx.base_path, base_path, sizeof(inner_ctx.base_path) - 1); if (!init_directory_cache(&inner_ctx.dir_cache)) { - vhd_close(&inner_ctx.vhd); + memset(&inner_ctx.vhd, 0, sizeof(VHDContext)); return false; } @@ -1587,33 +1502,21 @@ bool ntfs_extract_pending_vhds(NTFSContext* ctx, bool silent, bool verbose, bool found_ntfs = true; } - if (!found_ntfs && read_ok && boot[510] == 0x55 && boot[511] == 0xAA) { - for (int i = 0; i < 4; i++) { - uint8_t* entry = boot + 446 + (i * 16); - uint8_t ptype = entry[4]; - uint32_t lba_start = entry[8] | (entry[9] << 8) | (entry[10] << 16) | (entry[11] << 24); - - if (ptype == NTFS_PARTITION_TYPE && lba_start > 0) { - ntfs_offset = (uint64_t)lba_start * 512; - if (vhd_read(&inner_ctx.vhd, boot, ntfs_offset, sizeof(boot)) && - boot[0] == 0xEB && boot[1] == 0x52 && boot[2] == 0x90 && - memcmp(boot + 3, NTFS_SIGNATURE, 8) == 0) { - found_ntfs = true; - break; - } + if (!found_ntfs && read_ok) { + int64_t part_offset = find_ntfs_partition_offset(boot); + if (part_offset >= 0) { + ntfs_offset = (uint64_t)part_offset; + if (vhd_read(&inner_ctx.vhd, boot, ntfs_offset, sizeof(boot)) && + boot[0] == 0xEB && boot[1] == 0x52 && boot[2] == 0x90 && + memcmp(boot + 3, NTFS_SIGNATURE, 8) == 0) { + found_ntfs = true; } } } - if (!found_ntfs) { + if (!found_ntfs || !ntfs_setup_mft(&inner_ctx, ntfs_offset)) { free_directory_cache(&inner_ctx.dir_cache); - vhd_close(&inner_ctx.vhd); - return false; - } - - if (!ntfs_setup_mft(&inner_ctx, ntfs_offset)) { - free_directory_cache(&inner_ctx.dir_cache); - vhd_close(&inner_ctx.vhd); + memset(&inner_ctx.vhd, 0, sizeof(VHDContext)); return false; } @@ -1623,28 +1526,183 @@ bool ntfs_extract_pending_vhds(NTFSContext* ctx, bool silent, bool verbose, bool } bool success = ntfs_extract_all(&inner_ctx); + if (out_files) *out_files = inner_ctx.files_extracted; + if (out_bytes) *out_bytes = inner_ctx.extracted_bytes; - ctx->files_extracted += inner_ctx.files_extracted; - ctx->extracted_bytes += inner_ctx.extracted_bytes; - + memset(&inner_ctx.vhd, 0, sizeof(VHDContext)); ntfs_close(&inner_ctx); + return success; +} - DeferredDirTime* dirs = (DeferredDirTime*)ctx->deferred_dirs; - for (uint32_t i = ctx->deferred_count; i > 0; i--) { - DeferredDirTime* d = &dirs[i - 1]; - set_dir_times(d->path, d->mtime, d->atime); +bool ntfs_init_vhd(NTFSContext* ctx, const char* vhd_path, const char* extract_path, + VHDContext* external_parent) { + bool silent = ctx->silent; + bool verbose = ctx->verbose; + memset(ctx, 0, sizeof(NTFSContext)); + ctx->silent = silent; + ctx->verbose = verbose; + ctx->is_vhd = true; + strncpy(ctx->base_path, extract_path, sizeof(ctx->base_path) - 1); + + if (!init_directory_cache(&ctx->dir_cache)) return false; + + if (!vhd_init_with_parent(&ctx->vhd, vhd_path, external_parent)) { + free_directory_cache(&ctx->dir_cache); + return false; } - free(ctx->deferred_dirs); - ctx->deferred_dirs = NULL; - ctx->deferred_count = 0; - ctx->deferred_capacity = 0; + + uint8_t boot[512]; + uint64_t ntfs_offset = 0; + bool found_ntfs = false; + + if (ntfs_read(ctx, boot, 0, sizeof(boot))) { + if (boot[0] == 0xEB && boot[1] == 0x52 && boot[2] == 0x90 && + memcmp(boot + 3, NTFS_SIGNATURE, 8) == 0) { + found_ntfs = true; + } + + if (!found_ntfs) { + int64_t part_offset = find_ntfs_partition_offset(boot); + if (part_offset >= 0) { + ntfs_offset = (uint64_t)part_offset; + if (ntfs_read(ctx, boot, ntfs_offset, sizeof(boot)) && + memcmp(boot + 3, NTFS_SIGNATURE, 8) == 0) { + found_ntfs = true; + } + } + } + } + + if (!found_ntfs || !ntfs_setup_mft(ctx, ntfs_offset)) { + if (external_parent && ctx->vhd.parent == external_parent) + ctx->vhd.parent = NULL; + ntfs_close(ctx); + return false; + } + + return true; +} + +bool ntfs_extract_pending_vhds(NTFSContext* ctx, bool silent, bool verbose, + const char* parent_file, VHDContext* parent_vhd, + bool* is_orphan, VHDContext** out_base_vhd) { + if (is_orphan) *is_orphan = false; + if (out_base_vhd) *out_base_vhd = NULL; + if (ctx->pending_vhd_count == 0) return true; + + int highest_vhd = -1; + PendingVHD* highest = NULL; + for (int i = 0; i < ctx->pending_vhd_count; i++) { + if (ctx->pending_vhds[i].vhd_number > highest_vhd) { + highest_vhd = ctx->pending_vhds[i].vhd_number; + highest = &ctx->pending_vhds[i]; + } + } + + if (!highest) return true; + + bool used_cached_parent = false; + VHDContext vhd_ctx = {0}; + if (!vhd_init_from_runs(&vhd_ctx, ctx, highest->runs, highest->run_count, + highest->file_size, ctx->base_path)) { + + if (!vhd_setup_run_source(&vhd_ctx, ctx, highest->runs, highest->run_count, + highest->file_size, ctx->base_path)) { + if (is_orphan) *is_orphan = true; + return true; + } + + if (!vhd_read_footer(&vhd_ctx) || !vhd_init_dynamic_header(&vhd_ctx)) { + vhd_close(&vhd_ctx); + if (is_orphan) *is_orphan = true; + return true; + } + + if (vhd_ctx.footer.disk_type != VHD_TYPE_DIFFERENCING) { + vhd_close(&vhd_ctx); + if (is_orphan) *is_orphan = true; + return true; + } + + VHDContext* parent_ctx = NULL; + + if (parent_vhd) { + if (memcmp(vhd_ctx.dyn_header.parent_id, parent_vhd->footer.unique_id, 16) == 0) { + vhd_ctx.parent = parent_vhd; + used_cached_parent = true; + parent_ctx = parent_vhd; + } else { + fprintf(stderr, "parent mismatch\n"); + } + } + + if (!parent_ctx) { + int parent_vhd_num = -1; + PendingVHD* parent_pending = NULL; + for (int i = 0; i < ctx->pending_vhd_count; i++) { + if (&ctx->pending_vhds[i] == highest) continue; + if (ctx->pending_vhds[i].vhd_number > parent_vhd_num) { + parent_vhd_num = ctx->pending_vhds[i].vhd_number; + parent_pending = &ctx->pending_vhds[i]; + } + } + + if (parent_pending) { + parent_ctx = malloc(sizeof(VHDContext)); + if (parent_ctx) { + if (!vhd_init_from_runs(parent_ctx, ctx, parent_pending->runs, + parent_pending->run_count, parent_pending->file_size, ctx->base_path)) { + free(parent_ctx); + parent_ctx = NULL; + } + } + } + } + + if (!parent_ctx && parent_file) { + parent_ctx = malloc(sizeof(VHDContext)); + if (parent_ctx) { + if (!vhd_init_internal(parent_ctx, parent_file, 1)) { + free(parent_ctx); + parent_ctx = NULL; + } + } + } + + if (!parent_ctx) { + vhd_close(&vhd_ctx); + if (is_orphan) *is_orphan = true; + return true; + } + + if (!used_cached_parent) { + vhd_ctx.parent = parent_ctx; + } + } + + if (out_base_vhd && vhd_ctx.footer.disk_type == VHD_TYPE_DYNAMIC) { + *out_base_vhd = malloc(sizeof(VHDContext)); + if (*out_base_vhd) { + memcpy(*out_base_vhd, &vhd_ctx, sizeof(VHDContext)); + return true; + } + } + + uint64_t files = 0, bytes = 0; + bool success = vhd_extract_ntfs(&vhd_ctx, ctx->base_path, silent, verbose, &files, &bytes); + ctx->files_extracted += files; + ctx->extracted_bytes += bytes; + + if (!success && is_orphan) *is_orphan = true; + + if (used_cached_parent) { + vhd_ctx.parent = NULL; + } + vhd_close(&vhd_ctx); return success; } -int ntfs_get_pending_opt_count(NTFSContext* ctx) { - return ctx ? ctx->pending_opt_count : 0; -} const PendingOpt* ntfs_get_pending_opt(NTFSContext* ctx, int index) { if (!ctx || index < 0 || index >= ctx->pending_opt_count) return NULL; @@ -1652,13 +1710,12 @@ const PendingOpt* ntfs_get_pending_opt(NTFSContext* ctx, int index) { } void ntfs_close(NTFSContext* ctx) { - if (ctx->stream) { - } - else if (ctx->is_vhd) { - vhd_close(&ctx->vhd); - } - else { - if (ctx->raw.fp) { + close_output_dir(ctx->cached_dir); + if (!ctx->stream) { + if (ctx->is_vhd) { + vhd_close(&ctx->vhd); + } + else if (ctx->raw.fp) { fclose(ctx->raw.fp); } } diff --git a/src/stream.c b/src/stream.c index 9fd638a..bf04a71 100644 --- a/src/stream.c +++ b/src/stream.c @@ -3,22 +3,22 @@ #include "common.h" #include "ntfs.h" -static bool read_from_runs_internal(void* ntfs_ctx, const DataRun* runs, int run_count, - uint64_t file_size, uint64_t read_offset, void* buffer, size_t read_size); - bool stream_init(DecryptStream* ds, FILE* fp, uint64_t data_offset, uint64_t data_size, const uint8_t key[16], const uint8_t iv[16]) { if (!ds || !fp) return false; memset(ds, 0, sizeof(DecryptStream)); ds->fp = fp; - ds->parent_stream = NULL; ds->run_source = NULL; ds->data_offset = data_offset; ds->data_size = data_size; memcpy(ds->key, key, 16); memcpy(ds->file_iv, iv, 16); - ds->cached_page_offset = (uint64_t)-1; + for (int i = 0; i < STREAM_CACHE_PAGES; i++) { + ds->cached_page_offsets[i] = (uint64_t)-1; + ds->page_lru[i] = 0; + } + ds->lru_counter = 0; ds->file_pos = (uint64_t)-1; AES_init_ctx_iv(&ds->aes_ctx, key, iv); @@ -32,13 +32,16 @@ bool stream_init_from_runs(DecryptStream* ds, RunSource* source, memset(ds, 0, sizeof(DecryptStream)); ds->fp = NULL; - ds->parent_stream = NULL; ds->run_source = source; ds->data_offset = 0; ds->data_size = source->file_size; memcpy(ds->key, key, 16); memcpy(ds->file_iv, iv, 16); - ds->cached_page_offset = (uint64_t)-1; + for (int i = 0; i < STREAM_CACHE_PAGES; i++) { + ds->cached_page_offsets[i] = (uint64_t)-1; + ds->page_lru[i] = 0; + } + ds->lru_counter = 0; ds->file_pos = (uint64_t)-1; AES_init_ctx_iv(&ds->aes_ctx, key, iv); @@ -46,50 +49,6 @@ bool stream_init_from_runs(DecryptStream* ds, RunSource* source, return true; } -static bool read_from_runs_internal(void* ntfs_ctx, const DataRun* runs, int run_count, - uint64_t file_size, uint64_t read_offset, void* buffer, size_t read_size) { - if (read_offset + read_size > file_size) { - return false; - } - - NTFSContext* ctx = (NTFSContext*)ntfs_ctx; - uint64_t current_file_pos = 0; - uint8_t* out = (uint8_t*)buffer; - size_t bytes_remaining = read_size; - - for (int i = 0; i < run_count && bytes_remaining > 0; i++) { - uint64_t run_bytes = runs[i].length * ctx->bytes_per_cluster; - uint64_t run_start = current_file_pos; - uint64_t run_end = run_start + run_bytes; - - if (read_offset < run_end && read_offset + read_size > run_start) { - uint64_t start_in_run = (read_offset > run_start) ? (read_offset - run_start) : 0; - uint64_t end_in_run = ((read_offset + read_size) < run_end) ? - (read_offset + read_size - run_start) : run_bytes; - size_t chunk_size = (size_t)(end_in_run - start_in_run); - - uint64_t disk_offset = ctx->data_start_offset + - (runs[i].offset * ctx->bytes_per_cluster) + start_in_run; - - if (!stream_read(ctx->stream, out, disk_offset, chunk_size)) { - return false; - } - - out += chunk_size; - bytes_remaining -= chunk_size; - } - - current_file_pos = run_end; - } - - return bytes_remaining == 0; -} - -bool stream_read_raw(void* ntfs_ctx, const DataRun* runs, int run_count, - uint64_t file_size, uint64_t offset, void* buffer, size_t size) { - return read_from_runs_internal(ntfs_ctx, runs, run_count, file_size, offset, buffer, size); -} - bool stream_read(DecryptStream* ds, void* buffer, uint64_t offset, size_t size) { if (!ds || !buffer || size == 0) return false; if (offset > ds->data_size) return false; @@ -103,20 +62,33 @@ bool stream_read(DecryptStream* ds, void* buffer, uint64_t offset, size_t size) uint64_t page_offset = (current_offset / DECRYPT_PAGE_SIZE) * DECRYPT_PAGE_SIZE; size_t offset_in_page = (size_t)(current_offset % DECRYPT_PAGE_SIZE); - if (ds->cached_page_offset != page_offset) { + int slot = -1; + if (ds->cached_page_offsets[ds->last_slot] == page_offset) { + slot = ds->last_slot; + } else { + for (int i = 0; i < STREAM_CACHE_PAGES; i++) { + if (ds->cached_page_offsets[i] == page_offset) { + slot = i; + break; + } + } + } + + if (slot < 0) { + slot = 0; + for (int i = 1; i < STREAM_CACHE_PAGES; i++) { + if (ds->page_lru[i] < ds->page_lru[slot]) slot = i; + } + size_t page_bytes_available = (size_t)(ds->data_size - page_offset); size_t read_size = (page_bytes_available > DECRYPT_PAGE_SIZE) ? DECRYPT_PAGE_SIZE : page_bytes_available; - if (ds->parent_stream) { - if (!stream_read(ds->parent_stream, ds->page_buffer, page_offset, read_size)) { - return false; - } - } else if (ds->run_source) { + if (ds->run_source) { uint64_t run_offset = ds->data_offset + page_offset; - if (!read_from_runs_internal(ds->run_source->ntfs_ctx, ds->run_source->runs, + if (!ntfs_read_from_runs((NTFSContext*)ds->run_source->ntfs_ctx, ds->run_source->runs, ds->run_source->run_count, ds->run_source->file_size, - run_offset, ds->page_buffer, read_size)) { + run_offset, ds->page_buffers[slot], read_size)) { return false; } } else { @@ -124,23 +96,26 @@ bool stream_read(DecryptStream* ds, void* buffer, uint64_t offset, size_t size) if (ds->file_pos != file_pos_wanted) { if (FSEEKO(ds->fp, file_pos_wanted, SEEK_SET) != 0) return false; } - if (fread(ds->page_buffer, 1, read_size, ds->fp) != read_size) return false; + if (fread(ds->page_buffers[slot], 1, read_size, ds->fp) != read_size) return false; ds->file_pos = file_pos_wanted + read_size; } if (read_size < DECRYPT_PAGE_SIZE) { - memset(ds->page_buffer + read_size, 0, DECRYPT_PAGE_SIZE - read_size); + memset(ds->page_buffers[slot] + read_size, 0, DECRYPT_PAGE_SIZE - read_size); } uint8_t page_iv[16]; iv_page(page_offset, ds->file_iv, page_iv); AES_ctx_set_iv(&ds->aes_ctx, page_iv); - AES_CBC_decrypt_buffer(&ds->aes_ctx, ds->page_buffer, DECRYPT_PAGE_SIZE); + AES_CBC_decrypt_buffer(&ds->aes_ctx, ds->page_buffers[slot], DECRYPT_PAGE_SIZE); - ds->cached_page_offset = page_offset; + ds->cached_page_offsets[slot] = page_offset; } + ds->page_lru[slot] = ++ds->lru_counter; + ds->last_slot = slot; + size_t bytes_in_page = DECRYPT_PAGE_SIZE - offset_in_page; size_t bytes_remaining = size - bytes_read; size_t copy_size = (bytes_in_page < bytes_remaining) ? bytes_in_page : bytes_remaining; @@ -148,7 +123,7 @@ bool stream_read(DecryptStream* ds, void* buffer, uint64_t offset, size_t size) uint64_t max_valid = ds->data_size - current_offset; if (copy_size > max_valid) copy_size = (size_t)max_valid; - memcpy(out + bytes_read, ds->page_buffer + offset_in_page, copy_size); + memcpy(out + bytes_read, ds->page_buffers[slot] + offset_in_page, copy_size); bytes_read += copy_size; if (copy_size == 0) break;