Implemented FARC as seen in the App

src/CRE/app_farc.hpp

@@ -0,0 +1,223 @@
+/*
+    by korenkonder
+    GitHub/GitLab: korenkonder
+*/
+
+#pragma once
+
+#include <random>
+#include "../KKdLib/default.hpp"
+#include "../KKdLib/io/stream.hpp"
+#include "../KKdLib/prj/rijndael.hpp"
+#include "../KKdLib/prj/shared_ptr.hpp"
+#include "../KKdLib/farc.hpp"
+
+enum farc_type {
+    FARC_NORMAL = 0,
+    FARC_COMPRESSED,
+    FARC_EXTENDED,
+};
+
+struct farc_aes_key {
+    uint8_t data[prj::Rijndael_Nb * sizeof(uint32_t)];
+};
+
+struct farc_params {
+    size_t align;
+    bool compress;
+    bool encrypt;
+    bool key_set;
+    farc_aes_key key;
+
+    farc_params();
+
+    void set_key(const farc_aes_key* key);
+};
+
+struct farc_read_file {
+    char name[128];
+    uint32_t offset;
+    uint32_t size;
+    uint32_t original_size;
+    bool compressed;
+    bool encrypted;
+
+    farc_read_file();
+};
+
+struct farc_read {
+    const void* data;
+    size_t size;
+    int32_t field_10;
+    int32_t field_14;
+    farc_type type;
+    farc_signature signature;
+    uint32_t header_length;
+    farc_flags flags;
+    uint32_t field_28;
+    uint32_t alignment;
+    uint32_t entry_padding;
+    uint32_t header_padding;
+    std::vector<farc_read_file> files;
+    prj::shared_ptr<prj::Rijndael> rijndael;
+
+    farc_read();
+    ~farc_read();
+
+    const farc_read_file* get_file(int32_t index) const;
+    size_t get_file_size(int32_t index) const;
+    const void* get_file_data(int32_t index) const;
+    int32_t get_file_index(const char* name) const;
+    bool get_file_is_plain(int32_t index) const;
+    const char* get_file_name(int32_t index) const;
+    size_t get_file_offset(int32_t index) const;
+    size_t get_file_original_size(int32_t index) const;
+    size_t get_files_count() const;
+    bool init(const void* data, size_t size, const farc_aes_key* key = 0);
+    bool load_file_data(void* data, size_t size, int32_t index) const;
+    void reset();
+};
+
+struct farc_ft_params {
+    size_t align;
+    bool compress;
+    bool encrypt;
+    bool key_set;
+    farc_aes_key key;
+    bool normal;
+
+    farc_ft_params();
+
+    void set_key(const farc_aes_key* key);
+};
+
+struct farc_ft_read_file {
+    char name[128];
+    uint32_t offset;
+    uint32_t size;
+    uint32_t original_size;
+    farc_flags flags;
+
+    farc_ft_read_file();
+};
+
+struct farc_ft_read {
+    const void* data;
+    size_t size;
+    farc_type type;
+    farc_signature signature;
+    uint32_t header_length;
+    farc_flags flags;
+    uint32_t field_20;
+    uint32_t header_size;
+    uint32_t alignment;
+    uint32_t files_count;
+    uint32_t entry_size;
+    std::vector<farc_ft_read_file> files;
+    prj::shared_ptr<farc_aes_key> aes_key;
+
+    farc_ft_read();
+    ~farc_ft_read();
+
+    const farc_ft_read_file* get_file(int32_t index) const;
+    int32_t get_file_size(int32_t index) const;
+    const void* get_file_data(int32_t index) const;
+    int32_t get_file_index(const char* name) const;
+    const char* get_file_name(int32_t index) const;
+    int32_t get_file_offset(int32_t index) const;
+    int32_t get_file_original_size(int32_t index) const;
+    uint32_t get_files_count() const;
+    bool init(const void* data, size_t size);
+    bool load_file_data(void* data, size_t size, int32_t index) const;
+    void reset();
+};
+
+struct farc_write_file {
+    std::string name;
+    size_t offset;
+    size_t size;
+    size_t original_size;
+
+    farc_write_file();
+    ~farc_write_file();
+};
+
+struct farc_write {
+    std::string path;
+    farc_params params;
+    farc_type type;
+    std::vector<farc_write_file> files;
+    std::string data_path;
+    FILE* file_handle;
+    size_t data_offset;
+    prj::Rijndael rijndael;
+
+    farc_write();
+    ~farc_write();
+
+    bool add_file(const void* data, size_t size, const std::string& name);
+    size_t get_align(size_t size) const;
+    bool open(const std::string& path, const farc_params& params);
+    void reset();
+    bool write_data(const void* data, size_t size);
+    bool write_file();
+    bool write_header(FILE* f);
+};
+
+struct p_farc_write {
+    farc_write* ptr;
+
+    p_farc_write();
+    ~p_farc_write();
+
+    bool add_file(const void* data, size_t size, const std::string& file);
+    bool open(const std::string& path, const farc_params& params);
+    bool open(const std::string& path, bool compress, size_t align);
+    bool write_file();
+};
+
+struct farc_ft_write_file {
+    std::string name;
+    uint32_t offset;
+    uint32_t size;
+    uint32_t original_size;
+    farc_flags flags;
+
+    farc_ft_write_file();
+    ~farc_ft_write_file();
+};
+
+struct farc_ft_write {
+    std::string path;
+    farc_ft_params params;
+    farc_type type;
+    std::vector<farc_ft_write_file> files;
+    std::string data_path;
+    FILE* file_handle;
+    std::mt19937 mt_rand;
+    prj::shared_ptr<farc_aes_key> aes_key;
+
+    farc_ft_write();
+    ~farc_ft_write();
+
+    bool add_file(const void* data, size_t size, const std::string& name);
+    size_t get_align(size_t size) const;
+    bool open(const std::string& path, const farc_ft_params& params);
+    void reset();
+    bool write_data(const void* data, size_t size, size_t& act_size, bool& encrypted);
+    bool write_file();
+    bool write_header(FILE* f);
+    bool write_pad(size_t align, bool random, FILE* f);
+};
+
+struct p_farc_ft_write {
+    farc_ft_write* ptr;
+
+    p_farc_ft_write();
+    ~p_farc_ft_write();
+
+    bool add_file(const void* data, size_t size, const std::string& file);
+    bool open(const std::string& path, const farc_ft_params& params);
+    bool open(const std::string& path, bool compress, size_t align);
+    bool write_file();
+};

src/CRE/rob/rob.cpp

@@ -8,11 +8,11 @@
 #include "../../KKdLib/io/json.hpp"
 #include "../../KKdLib/io/path.hpp"
 #include "../../KKdLib/prj/algorithm.hpp"
-#include "../../KKdLib/farc.hpp"
 #include "../../KKdLib/key_val.hpp"
 #include "../../KKdLib/sort.hpp"
 #include "../../KKdLib/str_utils.hpp"
 #include "../../KKdLib/waitable_timer.hpp"
+#include "../app_farc.hpp"
 #include "../customize_item_table.hpp"
 #include "../module_table.hpp"
 #include "../data.hpp"
@@ -142,53 +142,6 @@ struct opd_vec3_data_vec {
     ~opd_vec3_data_vec();
 };
 
-struct opd_farc_params {
-    size_t align;
-    bool compress;
-    bool encrypt;
-
-    opd_farc_params();
-};
-
-struct opd_farc_file {
-    std::string name;
-    size_t offset;
-    size_t size;
-
-    opd_farc_file();
-    opd_farc_file(const std::string& name, size_t offset, size_t size);
-    ~opd_farc_file();
-};
-
-struct opd_farc {
-    std::string path;
-    opd_farc_params params;
-    std::vector<opd_farc_file> files;
-    std::string data_path;
-    file_stream* stream;
-    size_t farc_size;
-
-    opd_farc();
-    ~opd_farc();
-
-    bool add_file(const void* data, size_t size, const std::string& name);
-    bool open(const std::string& path, const opd_farc_params& params);
-    void reset();
-    bool write_file();
-};
-
-struct p_opd_farc {
-    opd_farc* ptr;
-
-    p_opd_farc();
-    ~p_opd_farc();
-
-    bool add_file(const void* data, size_t size, const std::string& file);
-    bool open(const std::string& path, const opd_farc_params& params);
-    bool open(const std::string& path, bool compress, size_t align);
-    bool write_file();
-};
-
 struct opd_chara_data {
     int32_t chara_id;
     bool init;
@@ -197,8 +150,8 @@ struct opd_chara_data {
     uint32_t motion_id;
     uint64_t field_18;
     std::vector<std::vector<opd_vec3_data_vec>> opd_data[ITEM_OSAGE_COUNT];
-    p_opd_farc opd[ITEM_OSAGE_COUNT];
-    p_opd_farc opdi[ITEM_OSAGE_COUNT];
+    p_farc_write opd[ITEM_OSAGE_COUNT];
+    p_farc_write opdi[ITEM_OSAGE_COUNT];
 
     opd_chara_data();
     ~opd_chara_data();
@@ -17152,171 +17105,6 @@ opd_vec3_data_vec::~opd_vec3_data_vec() {
 
 }
 
-opd_farc_params::opd_farc_params() : compress(), encrypt() {
-    align = 1;
-}
-
-opd_farc_file::opd_farc_file() : offset(), size() {
-
-}
-
-opd_farc_file::opd_farc_file(const std::string& name, size_t offset, size_t size) {
-    this->name.assign(name);
-    this->offset = offset;
-    this->size = size;
-}
-
-opd_farc_file::~opd_farc_file() {
-
-}
-
-opd_farc::opd_farc() : stream(), farc_size() {
-
-}
-
-opd_farc::~opd_farc() {
-
-}
-
-bool opd_farc::add_file(const void* data, size_t size, const std::string& name) {
-    if (!data || !size || !name.size() || !stream)
-        return false;
-
-    size_t offset = stream->get_position();
-    stream->write(data, size);
-    files.push_back({ name, offset, size });
-    return true;
-}
-
-bool opd_farc::open(const std::string& path, const opd_farc_params& params) {
-    reset();
-    if (!path.size() || !params.align || ((params.align - 1) & params.align))
-        return 0;
-
-    this->params = params;
-    this->path.assign(path);
-    data_path.assign(path);
-    data_path.append(".data");
-    stream = new file_stream;
-    stream->open(data_path.c_str(), "wb");
-    return stream->check_not_null();
-}
-
-void opd_farc::reset() {
-    path.clear();
-    params = {};
-    files.clear();
-    data_path.clear();
-    if (stream) {
-        delete stream;
-        stream = 0;
-    }
-    farc_size = 0;
-}
-
-bool opd_farc::write_file() {
-    std::string path_tmp(path);
-    path_tmp.append(".tmp");
-
-    file_stream fs;
-    bool ret = false;
-    if (!stream)
-        goto End;
-
-    stream->close();
-
-    delete stream;
-    stream = 0;
-
-    if (!path_delete_file(path.c_str()))
-        goto End;
-
-    fs.open(data_path.c_str(), "rb");
-    if (fs.check_not_null() && fs.get_length()) {
-        bool compress = params.compress;
-        bool encrypt = params.encrypt;
-        farc f;
-
-        for (opd_farc_file& i : files) {
-            fs.set_position(i.offset, SEEK_SET);
-            uint8_t* data = force_malloc<uint8_t>(i.size);
-            size_t len = fs.read(data, i.size);
-            if (len) {
-                farc_file* ff = f.add_file(i.name.c_str());
-                ff->data = data;
-                ff->size = i.size;
-                ff->compressed = compress;
-                ff->encrypted = encrypt;
-                ff->data_changed = true;
-            }
-        }
-
-        farc_signature signature;
-        farc_flags flags;
-        if (encrypt) {
-            signature = FARC_FARC;
-            flags = FARC_AES;
-
-            if (compress)
-                enum_or(flags, FARC_GZIP);
-        }
-        else if (compress) {
-            signature = FARC_FArC;
-            flags = FARC_NONE;
-        }
-        else {
-            signature = FARC_FArc;
-            flags = FARC_NONE;
-        }
-        f.alignment = (uint32_t)params.align;
-        f.write(path_tmp.c_str(), signature, flags, false, false);
-        ret = true;
-    }
-    fs.close();
-
-    if (ret) {
-        path_delete_file(path.c_str());
-        if (!path_move_file(path_tmp.c_str(), path.c_str()))
-            ret = false;
-    }
-
-End:
-    path_delete_file(data_path.c_str());
-    path_delete_file(path_tmp.c_str());
-    reset();
-    return ret;
-}
-
-p_opd_farc::p_opd_farc() : ptr() {
-    ptr = new opd_farc;
-}
-
-p_opd_farc::~p_opd_farc() {
-    if (ptr) {
-        delete ptr;
-        ptr = 0;
-    }
-}
-
-bool p_opd_farc::add_file(const void* data, size_t size, const std::string& file) {
-    return ptr->add_file(data, size, file);
-}
-
-bool p_opd_farc::open(const std::string& path, const opd_farc_params& params) {
-    return ptr->open(path, params);
-}
-
-bool p_opd_farc::open(const std::string& path, bool compress, size_t align) {
-    opd_farc_params farc_data;
-    farc_data.compress = compress;
-    farc_data.align = align;
-    return open(path, farc_data);
-}
-
-bool p_opd_farc::write_file() {
-    return ptr->write_file();
-}
-
 opd_chara_data::opd_chara_data() : chara_id(), init(), frame_index(), frame_count(), motion_id(), field_18() {
     chara_id = opd_chara_data_counter++;
     reset();
@@ -17388,7 +17176,7 @@ void opd_chara_data::encode_data() {
 
     std::string buf = sprintf_s_string("%s/%d", get_ram_osage_play_data_tmp_dir(), chara_id);
 
-    p_opd_farc* opd = this->opd;
+    p_farc_write* opd = this->opd;
     for (int32_t i = 0; i < ITEM_OSAGE_COUNT; i++, opd++) {
         rob_chara_item_equip_object* itm_eq_obj
             = rob_itm_equip->get_item_equip_object((item_id)(ITEM_OSAGE_FIRST + i));
@@ -17473,7 +17261,7 @@ void opd_chara_data::encode_init_data(uint32_t motion_id) {
 
     rob_chara_item_equip* rob_itm_equip = rob_chr->item_equip;
 
-    p_opd_farc* opdi = this->opdi;
+    p_farc_write* opdi = this->opdi;
     for (int32_t i = 0; i < ITEM_OSAGE_COUNT; i++, opdi++) {
         rob_chara_item_equip_object* itm_eq_obj
             = rob_itm_equip->get_item_equip_object((item_id)(ITEM_OSAGE_FIRST + i));
@@ -17617,7 +17405,7 @@ void opd_chara_data::open_opd_file() {
 
     rob_chara_item_equip* rob_itm_equip = rob_chr->item_equip;
 
-    p_opd_farc* opd = this->opd;
+    p_farc_write* opd = this->opd;
     for (int32_t i = 0; i < ITEM_OSAGE_COUNT; i++, opd++) {
         rob_chara_item_equip_object* itm_eq_obj
             = rob_itm_equip->get_item_equip_object((item_id)(ITEM_OSAGE_FIRST + i));
@@ -17647,7 +17435,7 @@ void opd_chara_data::open_opdi_file() {
 
     rob_chara_item_equip* rob_itm_equip = rob_chr->item_equip;
 
-    p_opd_farc* opdi = this->opdi;
+    p_farc_write* opdi = this->opdi;
     for (int32_t i = 0; i < ITEM_OSAGE_COUNT; i++, opdi++) {
         rob_chara_item_equip_object* itm_eq_obj
             = rob_itm_equip->get_item_equip_object((item_id)(ITEM_OSAGE_FIRST + i));

src/KKdLib/deflate.cpp

@@ -18,12 +18,12 @@ namespace deflate {
     };
 
     static int32_t compress_static(struct libdeflate_compressor* c, const void* src, size_t src_length,
-        void*& dst, size_t& dst_length, int32_t compression_level, mode mode, const allocator& allocator);
+        void*& dst, size_t& dst_length, int32_t compression_level, mode mode, const allocator* allocator);
     static int32_t decompress_static(struct libdeflate_decompressor* d, const void* src, size_t src_length,
-        void*& dst, size_t& dst_length, mode mode, const allocator& allocator);
+        void*& dst, size_t& dst_length, mode mode, const allocator* allocator);
 
     int32_t compress(const void* src, size_t src_length, void*& dst, size_t& dst_length,
-        mode mode, int32_t compression_level, const allocator& allocator) {
+        mode mode, int32_t compression_level, const allocator* allocator) {
         if (!src_length)
             return -1;
         else if (!src)
@@ -43,6 +43,17 @@ namespace deflate {
         return result >= 0 ? result : result - 0x10;
     }
 
+    size_t compress_bound(size_t src_length, mode mode) {
+        switch (mode) {
+        case MODE_GZIP:
+            return libdeflate_gzip_compress_bound(0, src_length);
+        case MODE_ZLIB:
+            return libdeflate_zlib_compress_bound(0, src_length);
+        default:
+            return libdeflate_deflate_compress_bound(0, src_length);
+        }
+    }
+
     int32_t compress_gzip(const void* src, size_t src_length, void*& dst, size_t& dst_length,
         int32_t compression_level, const char* file_name, const allocator& allocator) {
         if (!src_length)
@@ -57,7 +68,7 @@ namespace deflate {
             return -5;
 
         int32_t result = compress_static(c, src, src_length,
-            dst, dst_length, compression_level, MODE_GZIP, allocator);
+            dst, dst_length, compression_level, MODE_GZIP, &allocator);
         libdeflate_free_compressor(c);
         if (result < 0)
             return result - 0x10;
@@ -80,7 +91,7 @@ namespace deflate {
     }
 
     int32_t decompress(const void* src, size_t src_length, void*& dst, size_t& dst_length,
-        mode mode, const allocator& allocator) {
+        mode mode, const allocator* allocator) {
         if (!src_length)
             return -1;
         else if (!src)
@@ -100,7 +111,7 @@ namespace deflate {
     }
 
     static int32_t compress_static(struct libdeflate_compressor* c, const void* src, size_t src_length,
-        void*& dst, size_t& dst_length, int32_t compression_level, mode mode, const allocator& allocator) {
+        void*& dst, size_t& dst_length, int32_t compression_level, mode mode, const allocator* allocator) {
         size_t dst_max_length;
         switch (mode) {
         case MODE_GZIP:
@@ -114,7 +125,8 @@ namespace deflate {
             break;
         }
 
-        dst = allocator.malloc_callback(allocator.data, dst_max_length);
+        if (allocator)
+            dst = allocator->malloc_callback(allocator->data, dst_max_length);
         if (!dst)
             return -1;
 
@@ -136,18 +148,23 @@ namespace deflate {
             return 0;
         }
 
-        void* temp = allocator.malloc_callback(allocator.data, dst_act_length);
-        if (!temp) {
+        if (!allocator) {
+            dst_length = dst_act_length;
+            return -2;
+        }
+
+        void* temp = allocator->malloc_callback(allocator->data, dst_act_length);
+        if (!temp || !allocator) {
             if (dst) {
-                allocator.free_callback(allocator.data, dst);
+                allocator->free_callback(allocator->data, dst);
                 dst = 0;
             }
             return -2;
         }
 
         memcpy(temp, dst, dst_act_length);
-        if (dst) {
-            allocator.free_callback(allocator.data, dst);
+        if (allocator && dst) {
+            allocator->free_callback(allocator->data, dst);
             dst = 0;
         }
         dst = temp;
@@ -156,8 +173,9 @@ namespace deflate {
     }
 
     static int32_t decompress_static(struct libdeflate_decompressor* d, const void* src, size_t src_length,
-        void*& dst, size_t& dst_length, mode mode, const allocator& allocator) {
-        dst = allocator.malloc_callback(allocator.data, dst_length);
+        void*& dst, size_t& dst_length, mode mode, const allocator* allocator) {
+        if (allocator)
+            dst = allocator->malloc_callback(allocator->data, dst_length);
         if (!dst)
             return -1;
 
@@ -177,14 +195,14 @@ namespace deflate {
 
         switch (result) {
         case LIBDEFLATE_BAD_DATA:
-            if (dst) {
-                allocator.free_callback(allocator.data, dst);
+            if (allocator && dst) {
+                allocator->free_callback(allocator->data, dst);
                 dst = 0;
             }
             return -2;
         case LIBDEFLATE_INSUFFICIENT_SPACE:
-            if (dst) {
-                allocator.free_callback(allocator.data, dst);
+            if (allocator && dst) {
+                allocator->free_callback(allocator->data, dst);
                 dst = 0;
             }
             dst_length *= 2;
@@ -193,10 +211,13 @@ namespace deflate {
             if (dst_act_length >= dst_length)
                 break;
 
-            void* temp = allocator.malloc_callback(allocator.data, dst_act_length);
+            if (!allocator)
+                return -3;
+
+            void* temp = allocator->malloc_callback(allocator->data, dst_act_length);
             if (!temp) {
                 if (dst) {
-                    allocator.free_callback(allocator.data, dst);
+                    allocator->free_callback(allocator->data, dst);
                     dst = 0;
                 }
                 return -3;
@@ -204,7 +225,7 @@ namespace deflate {
 
             memcpy(temp, dst, dst_act_length);
             if (dst) {
-                allocator.free_callback(allocator.data, dst);
+                allocator->free_callback(allocator->data, dst);
                 dst = 0;
             }
             dst = temp;

src/KKdLib/deflate.hpp

@@ -23,9 +23,10 @@ namespace deflate {
     extern const allocator default_allocator;
 
     extern int32_t compress(const void* src, size_t src_length, void*& dst, size_t& dst_length,
-        mode mode, int32_t compression_level = 12, const allocator& allocator = default_allocator);
+        mode mode, int32_t compression_level = 12, const allocator* allocator = &default_allocator);
+    extern size_t compress_bound(size_t src_length, mode mode);
     extern int32_t compress_gzip(const void* src, size_t src_length, void*& dst, size_t& dst_length,
         int32_t compression_level, const char* file_name = 0, const allocator& allocator = default_allocator);
     extern int32_t decompress(const void* src, size_t src_length, void*& dst, size_t& dst_length,
-        mode mode, const allocator& allocator = default_allocator);
+        mode mode, const allocator* allocator = &default_allocator);
 }