Merge pull request 'test: add unit + CLI + e2e tests with synthetic fixtures, and a local Docker runner' (#6) from jujuforce/fsdecrypt:test/add-unit-tests-and-ci into trunk

.gitattributes

@@ -0,0 +1 @@
+*.sh text eol=lf

.gitignore

@@ -9,3 +9,15 @@
 flamegraph.svg
 CLAUDE.md
 memory/
+
+# e2e fixtures: ignore anything dropped into fixtures/ (your own real containers
+# stay private) EXCEPT the committed synthetic test fixtures and their README.
+/fixtures/*
+!/fixtures/README.md
+!/fixtures/TEST.bin
+!/fixtures/TEST_1.00.00_20240101120000_0.app
+!/fixtures/TEST_1.01.00_20240102120000_1_1.00.00.app
+!/fixtures/TEST_T001_20240101120000_0.opt
+
+# Fixture generator (kept local, not committed)
+/tests/fixtures_gen/

README.md

@@ -57,6 +57,30 @@ For games not in the built-in key database, place a file named `{GAME_ID}.bin` i
 - **16 bytes** for key only (IV derived automatically)
 - **32 bytes** for key + IV
 
+## Testing
+
+Tests run locally (there is no hosted CI). The unit tests are plain `cargo test`;
+the Linux `x86_64-musl` release target is exercised via Docker when you need it.
+
+```bash
+cargo test            # unit tests (IV derivation, timestamp decoding, key lookup)
+scripts/test.sh       # native tests + smoke test, then the same in a musl container
+scripts/test.sh --no-docker   # skip the Docker/Linux step
+```
+
+`scripts/test.sh` uses the [`clux/muslrust`](https://hub.docker.com/r/clux/muslrust)
+image to build and test the static Linux binary — the same way release artifacts
+are produced — so both targets can be validated from any host.
+
+### End-to-end fixtures
+
+The `tests/e2e.rs` test decrypts real containers placed in a `fixtures/`
+directory at the repo root and checks that each extracts successfully. That
+folder is git-ignored — the proprietary sample containers are never committed —
+and the test is a no-op when it is empty, so just drop a few `.app`/`.opt` files
+in `fixtures/` and run `cargo test` (or `scripts/test.sh`) to exercise the full
+decrypt-and-extract path.
+
 ## License
 
 [BSD Zero Clause License](LICENSE) (0BSD)

fixtures/README.md

@@ -0,0 +1,34 @@
+# Test fixtures
+
+These are **synthetic** SEGA fscrypt containers used by `tests/e2e.rs`. They
+contain only dummy files (a few text/XML/stub binaries), so they are safe to
+commit — there is no proprietary game content inside.
+
+| File | Type | What it exercises |
+|------|------|-------------------|
+| `TEST_T001_20240101120000_0.opt` | OPTION | exFAT decrypt + extract (built-in OPTION key) |
+| `TEST_1.00.00_20240101120000_0.app` | APP (base) | outer NTFS → `internal_0.vhd` (fixed VHD) → inner NTFS extract |
+| `TEST_1.01.00_20240102120000_1_1.00.00.app` | APP (delta) | differencing `internal_1.vhd`, auto-merged against the base by VHD GUID |
+| `TEST.bin` | key | external key (16-byte key + 16-byte IV) for game id `TEST` |
+
+The delta is a real differencing VHD linked to the base; decrypting it (with the
+base alongside) merges the two and yields the base files plus the delta's added
+`data/patch_notes.txt` and a modified `readme.txt`.
+
+The `.app` fixtures use the synthetic game id `TEST`, which is **not** in the
+built-in key table — so they are decrypted via the external-key-file fallback
+using the committed `TEST.bin` (this also exercises that fallback path).
+`tests/e2e.rs` copies `TEST.bin` next to the containers and runs from there so
+the binary finds it. The OPTION fixture uses the built-in OPTION key. Everything
+here — keys and filesystem data alike — is synthetic.
+
+You can also drop your own **real** `.app`/`.opt` files in this folder to test
+against them; everything here except these committed fixtures is git-ignored.
+
+## Regenerating
+
+These are generated by a local, uncommitted script set (NTFS/exFAT image
+creation in Docker + a small SEGA fscrypt packer adapted from
+[`beerpsi/x`](https://gitea.tendokyu.moe/beerpsi/x)). The byte output is not
+reproducible (NTFS/exFAT embed creation timestamps), but regenerated fixtures are
+functionally equivalent.

scripts/test.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+#
+# Local test runner (no hosted CI). Runs the unit tests and a smoke test
+# natively, then — when Docker is available — repeats them for the Linux
+# x86_64-musl release target inside a container, so both targets can be
+# validated from any host without a CI runner.
+#
+# Usage:
+#   scripts/test.sh            # native tests + Docker linux/musl (if Docker present)
+#   scripts/test.sh --no-docker  # native only
+#
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+
+use_docker=1
+[ "${1:-}" = "--no-docker" ] && use_docker=0
+
+# --- helpers ---------------------------------------------------------------
+
+smoke() {
+    # $1 = path to a fsdecrypt binary
+    local bin="$1"
+    "$bin" --version
+    local junk
+    junk="$(mktemp --suffix=.app 2>/dev/null || echo "${TMPDIR:-/tmp}/fsd_junk.app")"
+    head -c 4096 /dev/urandom > "$junk"
+    # Garbage input must fail gracefully (non-zero exit), never panic.
+    if "$bin" "$junk"; then
+        echo "ERROR: expected a non-zero exit on garbage input" >&2
+        rm -f "$junk"; return 1
+    fi
+    rm -f "$junk"
+    echo "smoke test passed"
+}
+
+# --- native ----------------------------------------------------------------
+
+echo "==> cargo test (native)"
+cargo test --locked
+
+echo "==> cargo build --release (native)"
+cargo build --release --locked
+
+native_bin="target/release/fsdecrypt"
+[ -f "$native_bin.exe" ] && native_bin="$native_bin.exe"
+echo "==> smoke test (native)"
+smoke "$native_bin"
+
+# --- Linux x86_64-musl via Docker ------------------------------------------
+
+if [ "$use_docker" = "1" ] && command -v docker >/dev/null 2>&1; then
+    echo "==> Docker: cargo test + release build for x86_64-unknown-linux-musl"
+    # Use a docker-friendly host path (Git Bash on Windows needs the C:/... form).
+    host_path="$PWD"
+    case "$(uname -s)" in
+        MINGW*|MSYS*|CYGWIN*) host_path="$(pwd -W)" ;;
+    esac
+    MSYS_NO_PATHCONV=1 docker run --rm \
+        -v "$host_path:/volume" -w /volume \
+        -e CARGO_TARGET_DIR=/volume/target-musl \
+        clux/muslrust:stable \
+        bash -c '
+            set -e
+            cargo test --locked
+            cargo build --release --locked
+        '
+    echo "==> smoke test (linux/musl)"
+    MSYS_NO_PATHCONV=1 docker run --rm \
+        -v "$host_path:/volume" -w /volume \
+        clux/muslrust:stable \
+        ./target-musl/x86_64-unknown-linux-musl/release/fsdecrypt --version
+    echo "linux/musl checks passed"
+elif [ "$use_docker" = "1" ]; then
+    echo "==> Docker not found — skipping Linux/musl checks (run with --no-docker to silence)"
+fi
+
+echo "All checks passed."

src/crypto.rs

@@ -515,3 +515,76 @@ pub fn get_game_keys(game_id: &str) -> Option<GameKeys> {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use aes::cipher::{block_padding::NoPadding, BlockEncryptMut, KeyIvInit};
+
+    type Aes128CbcEnc = cbc::Encryptor<aes::Aes128Enc>;
+
+    #[test]
+    fn page_iv_offset_zero_equals_file_iv() {
+        // offset 0 XORs every byte with 0, so the page IV is the file IV.
+        let file_iv = [0xAB; 16];
+        let mut page_iv = [0u8; 16];
+        calculate_page_iv(0, &file_iv, &mut page_iv);
+        assert_eq!(page_iv, file_iv);
+    }
+
+    #[test]
+    fn page_iv_wraps_every_8_bytes() {
+        // Zero file IV isolates the offset contribution: the little-endian bytes
+        // of the offset fill 0..8 and repeat in 8..16 because of the `i % 8` wrap.
+        let mut page_iv = [0u8; 16];
+        calculate_page_iv(0x0102_0304_0506_0708, &[0u8; 16], &mut page_iv);
+        assert_eq!(page_iv, [8, 7, 6, 5, 4, 3, 2, 1, 8, 7, 6, 5, 4, 3, 2, 1]);
+    }
+
+    #[test]
+    fn page_iv_xors_offset_with_file_iv() {
+        let mut page_iv = [0u8; 16];
+        calculate_page_iv(0x0000_0000_0000_00FF, &[0xFF; 16], &mut page_iv);
+        // byte 0 (and its wrap at 8): 0xFF ^ 0xFF = 0; the rest: 0xFF ^ 0 = 0xFF.
+        let mut expected = [0xFFu8; 16];
+        expected[0] = 0x00;
+        expected[8] = 0x00;
+        assert_eq!(page_iv, expected);
+    }
+
+    #[test]
+    fn file_iv_round_trips_against_known_header() {
+        // Mirror the real format: the first page block decrypts under
+        // (key, file_iv) to the FS header. So building the ciphertext as
+        // CBC_Enc(key, file_iv, header) must let calculate_file_iv recover file_iv.
+        let key = [0x11u8; 16];
+        let file_iv = [0x22u8; 16];
+        let mut block = NTFS_HEADER;
+        Aes128CbcEnc::new_from_slices(&key, &file_iv)
+            .unwrap()
+            .encrypt_padded_mut::<NoPadding>(&mut block, 16)
+            .unwrap();
+
+        let recovered = calculate_file_iv(key, NTFS_HEADER, &block).unwrap();
+        assert_eq!(recovered, file_iv);
+    }
+
+    #[test]
+    fn game_keys_known_lookup() {
+        let keys = get_game_keys("SBZS").expect("SBZS should be in the key table");
+        assert_eq!(keys.key, hex!("2ecbcff65ce0abecc10547f8ac8351d8"));
+    }
+
+    #[test]
+    fn game_keys_unknown_returns_none() {
+        // No "ZZZZ.bin" exists in the crate root, so the fallback yields None.
+        assert!(get_game_keys("ZZZZ").is_none());
+    }
+
+    #[test]
+    fn fs_header_magics_start_with_jump_opcode() {
+        // Both NTFS and exFAT boot sectors begin with the 0xEB short-jump byte.
+        assert_eq!(NTFS_HEADER[0], 0xEB);
+        assert_eq!(EXFAT_HEADER[0], 0xEB);
+    }
+}

src/main.rs

@@ -25,22 +25,28 @@ mod crypto;
 mod stream;
 mod vhd;
 
+/// Decode an exFAT `UtcOffset` byte into seconds east of UTC.
+///
+/// The byte packs an `OffsetValid` flag (bit 7) with a 7-bit two's-complement
+/// `OffsetFromUtc` in 15-minute units. When `OffsetValid` is 0 the timestamp has
+/// no timezone info and the offset bits must be ignored (treated as UTC).
+fn exfat_utc_offset_seconds(raw: u8) -> i32 {
+    if raw & 0x80 == 0 {
+        0
+    } else {
+        // Sign-extend the 7-bit value, then convert quarter-hours to seconds.
+        let offset_quarters = (((raw & 0x7F) << 1) as i8) >> 1;
+        offset_quarters as i32 * 15 * 60
+    }
+}
+
 fn exfat_timestamp_to_system_time(
     timestamp: &exfat_fs::timestamp::Timestamp,
 ) -> Result<SystemTime> {
     let exfat_date = timestamp.date();
     let exfat_time = timestamp.time();
 
-    // The exFAT UtcOffset byte packs an OffsetValid flag (bit 7) with a 7-bit
-    // two's-complement OffsetFromUtc in 15-minute units. When OffsetValid is 0
-    // the timestamp has no timezone info and the offset bits must be ignored.
-    let raw = timestamp.utc_offset() as u8;
-    let offset_seconds = if raw & 0x80 == 0 {
-        0
-    } else {
-        let offset_quarters = (((raw & 0x7F) << 1) as i8) >> 1;
-        offset_quarters as i32 * 15 * 60
-    };
+    let offset_seconds = exfat_utc_offset_seconds(timestamp.utc_offset() as u8);
     let fixed_offset = FixedOffset::east_opt(offset_seconds).unwrap_or_else(|| FixedOffset::east_opt(0).unwrap());
 
     let chrono_date_time = match fixed_offset.with_ymd_and_hms(
@@ -149,14 +155,23 @@ fn extract_exfat_elements(
     Ok(())
 }
 
-fn ntfs_time_to_system_time(ntfs_time: NtfsTime) -> SystemTime {
+/// Number of 100ns intervals between the Windows epoch (1601-01-01) and the
+/// Unix epoch (1970-01-01).
+const NT_INTERVALS_TO_UNIX_EPOCH: u64 = 116_444_736_000_000_000;
+
+/// Convert a raw NTFS timestamp (100ns intervals since 1601-01-01) to a
+/// `SystemTime`.
+fn nt_timestamp_to_system_time(intervals_since_windows_epoch: u64) -> SystemTime {
     // An NTFS "interval" is 100 nanoseconds.
-    // The Windows epoch is 1601-01-01, while the Unix epoch is 1970-01-01.
-    let intervals_since_windows_epoch = ntfs_time.nt_timestamp();
-    let intervals_since_unix_epoch = intervals_since_windows_epoch - 116_444_736_000_000_000;
+    let intervals_since_unix_epoch =
+        intervals_since_windows_epoch - NT_INTERVALS_TO_UNIX_EPOCH;
     let nanos_since_unix_epoch = intervals_since_unix_epoch * 100;
 
-    return SystemTime::UNIX_EPOCH + Duration::from_nanos(nanos_since_unix_epoch);
+    SystemTime::UNIX_EPOCH + Duration::from_nanos(nanos_since_unix_epoch)
+}
+
+fn ntfs_time_to_system_time(ntfs_time: NtfsTime) -> SystemTime {
+    nt_timestamp_to_system_time(ntfs_time.nt_timestamp())
 }
 
 fn extract_internal_vhd(image_path: &Path, sequence_number: u8) -> Result<PathBuf> {
@@ -514,3 +529,51 @@ fn process_chain(base: &ExtractedVhd, deltas: &[&ExtractedVhd]) -> Result<()> {
 
     Ok(())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::{exfat_utc_offset_seconds, nt_timestamp_to_system_time, NT_INTERVALS_TO_UNIX_EPOCH};
+    use std::time::{Duration, SystemTime};
+
+    #[test]
+    fn exfat_offset_invalid_flag_is_utc() {
+        // OffsetValid (bit 7) clear -> the offset bits are ignored (treat as UTC).
+        assert_eq!(exfat_utc_offset_seconds(0x00), 0);
+        assert_eq!(exfat_utc_offset_seconds(0x7F), 0);
+    }
+
+    #[test]
+    fn exfat_offset_zero_when_valid() {
+        assert_eq!(exfat_utc_offset_seconds(0x80), 0);
+    }
+
+    #[test]
+    fn exfat_offset_positive_japan() {
+        // UTC+9 = +36 quarter-hours = 0x80 | 0x24 = 0xA4.
+        assert_eq!(exfat_utc_offset_seconds(0xA4), 9 * 3600);
+    }
+
+    #[test]
+    fn exfat_offset_negative() {
+        // UTC-8 = -32 quarter-hours; -32 in 7-bit two's complement is 0x60,
+        // plus the valid flag -> 0xE0.
+        assert_eq!(exfat_utc_offset_seconds(0xE0), -8 * 3600);
+    }
+
+    #[test]
+    fn ntfs_epoch_maps_to_unix_epoch() {
+        assert_eq!(
+            nt_timestamp_to_system_time(NT_INTERVALS_TO_UNIX_EPOCH),
+            SystemTime::UNIX_EPOCH
+        );
+    }
+
+    #[test]
+    fn ntfs_one_second_after_epoch() {
+        // 1 second == 10_000_000 intervals of 100ns.
+        assert_eq!(
+            nt_timestamp_to_system_time(NT_INTERVALS_TO_UNIX_EPOCH + 10_000_000),
+            SystemTime::UNIX_EPOCH + Duration::from_secs(1)
+        );
+    }
+}

tests/cli.rs

@@ -0,0 +1,24 @@
+//! End-to-end CLI tests that run the actual built binary.
+
+use std::process::Command;
+
+/// `--help` must exit successfully and print the usage/options. Run with
+/// `cargo test -- --nocapture` to see the help text in the test output.
+#[test]
+fn help_shows_usage_and_options() {
+    let output = Command::new(env!("CARGO_BIN_EXE_fsdecrypt"))
+        .arg("--help")
+        .output()
+        .expect("failed to run fsdecrypt --help");
+
+    let stdout = String::from_utf8_lossy(&output.stdout);
+    println!("\n--- fsdecrypt --help ---\n{stdout}");
+
+    assert!(output.status.success(), "--help should exit 0");
+    assert!(stdout.contains("Usage"), "help should show a usage line");
+    assert!(stdout.contains("--no-extract"), "help should list options");
+    assert!(
+        stdout.contains("decryptor for some SEGA containers"),
+        "help should show the about text"
+    );
+}

tests/e2e.rs

@@ -0,0 +1,96 @@
+//! End-to-end decryption test driven by the fixtures in `fixtures/`.
+//!
+//! The committed fixtures are small, fully synthetic containers (see
+//! `fixtures/README.md`) — a base APP, a delta APP, and an OPTION — so this runs
+//! the whole decrypt-and-extract path, including the delta/base VHD merge, with
+//! no proprietary data. You can also drop your own real `.app`/`.opt` files in
+//! `fixtures/` (they are git-ignored) and they'll be exercised too. When
+//! `fixtures/` is empty the test is a no-op, so the suite stays green regardless.
+
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+fn fixture_files() -> Vec<PathBuf> {
+    let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("fixtures");
+    let Ok(entries) = fs::read_dir(dir) else {
+        return Vec::new();
+    };
+    let mut files: Vec<PathBuf> = entries
+        .filter_map(|e| e.ok().map(|e| e.path()))
+        .filter(|p| {
+            p.is_file()
+                && matches!(
+                    p.extension().and_then(|e| e.to_str()),
+                    Some("app") | Some("opt")
+                )
+        })
+        .collect();
+    files.sort();
+    files
+}
+
+#[test]
+fn decrypts_fixture_containers() {
+    let files = fixture_files();
+    if files.is_empty() {
+        eprintln!(
+            "skipping e2e: no .app/.opt fixtures in {}/fixtures — add containers to run this test",
+            env!("CARGO_MANIFEST_DIR")
+        );
+        return;
+    }
+
+    // Work in a temp dir so extraction never touches the fixtures folder. The
+    // tool extracts next to its input, so we copy every fixture in *first* — a
+    // delta APP needs its base APP sitting alongside for the auto-merge.
+    let workdir = std::env::temp_dir().join(format!("fsdecrypt-e2e-{}", std::process::id()));
+    let _ = fs::remove_dir_all(&workdir);
+    fs::create_dir_all(&workdir).expect("create work dir");
+
+    // Copy *every* fixture file in — the containers plus any external
+    // `{game_id}.bin` key files an unlisted game needs.
+    let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("fixtures");
+    if let Ok(entries) = fs::read_dir(&dir) {
+        for e in entries.filter_map(|e| e.ok()) {
+            let p = e.path();
+            if p.is_file() {
+                let _ = fs::copy(&p, workdir.join(p.file_name().unwrap()));
+            }
+        }
+    }
+    let inputs: Vec<PathBuf> = files
+        .iter()
+        .map(|f| workdir.join(f.file_name().unwrap()))
+        .collect();
+
+    for input in &inputs {
+        let name = input.file_name().unwrap().to_os_string();
+        // Run from the work dir so the binary resolves `{game_id}.bin` keys there.
+        let output = Command::new(env!("CARGO_BIN_EXE_fsdecrypt"))
+            .current_dir(&workdir)
+            .arg(input)
+            .output()
+            .expect("failed to run fsdecrypt");
+
+        let stdout = String::from_utf8_lossy(&output.stdout);
+        let stderr = String::from_utf8_lossy(&output.stderr);
+        assert!(
+            output.status.success(),
+            "decrypting {name:?} failed (exit {:?})\n--- stdout ---\n{stdout}\n--- stderr ---\n{stderr}",
+            output.status.code()
+        );
+
+        // A default run extracts into a folder named after the input (no extension).
+        let out_dir = input.with_extension("");
+        let produced = fs::read_dir(&out_dir).map(|rd| rd.count()).unwrap_or(0);
+        assert!(
+            produced > 0,
+            "{name:?}: expected extracted entries in {}\n--- stdout ---\n{stdout}",
+            out_dir.display()
+        );
+        println!("ok: {name:?} -> {produced} top-level entr(y/ies) extracted");
+    }
+
+    let _ = fs::remove_dir_all(&workdir);
+}