From 8f38b6141e45ec2287beb2eec707a5a654ca5040 Mon Sep 17 00:00:00 2001 From: Nunuhara Cabbage Date: Sat, 23 May 2020 12:33:40 -0700 Subject: [PATCH] alice-ar: various improvements * Can now extract files from .afa version 3 archives * Can now extract files from .flat files * Includes recursively extracting .flat files contained in other archive formats * Added option to extract images only * Now converts images to PNG format by default * WebP format also supported Because the table of contents is encrypted in .afa version 3 archives, the support is a bit limited. Only known file types can be extracted and they do not retain their original filenames (they are given numbered filenames instead). --- include/system4/ajp.h | 2 +- include/system4/buffer.h | 5 + meson.build | 1 + src/ajp.c | 4 +- src/cg.c | 1 - src/flat.c | 330 ++++++++++++++++++++++++++++ src/meson.build | 3 +- src/tools/ar/afa_extract.c | 433 +++++++++++++++++++++++++++++++++++++ src/tools/ar/alice-ar.c | 336 ++++++++++++++++++++++++---- src/tools/ar/meson.build | 1 + 10 files changed, 1067 insertions(+), 49 deletions(-) create mode 100644 src/flat.c create mode 100644 src/tools/ar/afa_extract.c diff --git a/include/system4/ajp.h b/include/system4/ajp.h index 0ce94a7..fc86642 100644 --- a/include/system4/ajp.h +++ b/include/system4/ajp.h @@ -24,6 +24,6 @@ struct cg; bool ajp_checkfmt(const uint8_t *data); -void ajp_extract(uint8_t *data, size_t size, struct cg *cg); +void ajp_extract(const uint8_t *data, size_t size, struct cg *cg); #endif /* SYSTEM4_AJP_H */ diff --git a/include/system4/buffer.h b/include/system4/buffer.h index 9669b1d..e6504e1 100644 --- a/include/system4/buffer.h +++ b/include/system4/buffer.h @@ -49,4 +49,9 @@ static inline void buffer_seek(struct buffer *r, size_t off) r->index = off; } +static inline void buffer_align(struct buffer *r, int p) +{ + r->index = (r->index + (p-1)) & ~(p-1); +} + #endif /* SYSTEM4_BUFFER_H */ diff --git a/meson.build b/meson.build index 80fc6eb..415a0cc 100644 --- a/meson.build +++ b/meson.build @@ -12,6 +12,7 @@ gl = dependency('GL') glew = dependency('glew') tj = dependency('libturbojpeg') webp = dependency('libwebp') +png = dependency('libpng') chibi = dependency('chibi-scheme', required : get_option('debugger')) flex = find_program('flex') diff --git a/src/ajp.c b/src/ajp.c index f286549..5a9cfd0 100644 --- a/src/ajp.c +++ b/src/ajp.c @@ -54,7 +54,7 @@ struct ajp_header { uint32_t mask_size; }; -static void ajp_extract_header(uint8_t *data, struct ajp_header *ajp) +static void ajp_extract_header(const uint8_t *data, struct ajp_header *ajp) { ajp->width = LittleEndian_getDW(data, 12); ajp->height = LittleEndian_getDW(data, 16); @@ -76,7 +76,7 @@ static void ajp_init_metrics(struct ajp_header *ajp, struct cg_metrics *dst) dst->alpha_pitch = 1; } -void ajp_extract(uint8_t *data, size_t size, struct cg *cg) +void ajp_extract(const uint8_t *data, size_t size, struct cg *cg) { uint8_t *buf = NULL, *jpeg_data = NULL, *mask_data = NULL; int width, height, subsamp; diff --git a/src/cg.c b/src/cg.c index d4b63e6..8eb8611 100644 --- a/src/cg.c +++ b/src/cg.c @@ -22,7 +22,6 @@ #include "system4/cg.h" #include "system4/ajp.h" #include "system4/dcf.h" -#include "system4/flat.h" #include "system4/qnt.h" #include "system4/webp.h" diff --git a/src/flat.c b/src/flat.c new file mode 100644 index 0000000..c1aecd6 --- /dev/null +++ b/src/flat.c @@ -0,0 +1,330 @@ +/* Copyright (C) 2019 Nunuhara Cabbage + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + */ + +#include +#include +#include +#include "little_endian.h" +#include "file.h" +#include "system4.h" +#include "system4/archive.h" +#include "system4/buffer.h" +#include "system4/cg.h" +#include "system4/ajp.h" + +enum flat_data_type { + FLAG_CG = 2, + FLAT_ZLIB = 5, +}; + +struct flat_entry { + size_t off; + size_t size; + int type; +}; + +struct flat_data { + struct archive_data super; + struct flat_entry *entry; + bool allocated; +}; + +struct flat_archive { + struct archive ar; + bool allocated; + size_t libl_off; + size_t talt_off; + size_t filesize; + unsigned nr_libl_files; + struct flat_entry *libl_files; + unsigned nr_talt_files; + struct flat_entry *talt_files; + uint8_t *data; +}; + +static const char *get_file_extension(int type, const char *data) +{ + switch (type) { + case FLAG_CG: + if (!strncmp(data, "AJP", 4)) + return ".ajp"; + if (!strncmp(data, "QNT", 4)) + return ".qnt"; + return ".img"; + case FLAT_ZLIB: + return ".z.dat"; + default: + return ".dat"; + } +} + +static void flat_free_data(struct archive_data *data) +{ + struct flat_data *flat = (struct flat_data*)data; + if (flat->allocated) + free(data->data); + free(data->name); + free(data); +} + +static void flat_free(struct archive *_ar) +{ + struct flat_archive *ar = (struct flat_archive*)_ar; + if (ar->allocated) + free(ar->data); + free(ar->libl_files); + free(ar->talt_files); + free(ar); +} + +static struct flat_entry *flat_get_entry(struct flat_archive *ar, unsigned no) +{ + if (no < ar->nr_libl_files) + return &ar->libl_files[no]; + + no -= ar->nr_libl_files; + if (no < ar->nr_talt_files) + return &ar->talt_files[no]; + + return NULL; +} + +static struct archive_data *flat_get(struct archive *_ar, int no) +{ + struct flat_archive *ar = (struct flat_archive*)_ar; + struct flat_entry *e = flat_get_entry(ar, no); + struct flat_data *data = xcalloc(1, sizeof(struct flat_data)); + + data->super.data = ar->data + e->off; + data->super.size = e->size; + data->super.name = xmalloc(256); + snprintf(data->super.name, 256, "%d%s", no, get_file_extension(e->type, (const char*)ar->data+e->off)); + data->super.no = no; + data->super.archive = _ar; + data->entry = e; + return &data->super; +} + +static bool flat_load_file(possibly_unused struct archive_data *data) +{ + struct flat_archive *ar = (struct flat_archive*)data->archive; + struct flat_data *flatdata = (struct flat_data*)data; + struct flat_entry *e = flatdata->entry; + + // inflate zlib compressed data + if (e->type == FLAT_ZLIB && ar->data[e->off+4] == 0x78) { + unsigned long size = LittleEndian_getDW(ar->data, e->off); + uint8_t *out = xmalloc(size); + if (uncompress(out, &size, ar->data + e->off + 4, e->size - 4) != Z_OK) { + WARNING("uncompress failed"); + free(out); + return false; + } + data->data = out; + data->size = size; + flatdata->allocated = true; + } + + return true; +} + +static void flat_for_each(struct archive *_ar, void (*iter)(struct archive_data *data, void *user), void *user) +{ + struct flat_archive *ar = (struct flat_archive*)_ar; + for (unsigned i = 0; i < ar->nr_libl_files + ar->nr_talt_files; i++) { + struct archive_data *data = flat_get(_ar, i); + if (!data) + continue; + iter(data, user); + flat_free_data(data); + } +} + +static struct archive_ops flat_archive_ops = { + .get = flat_get, + .load_file = flat_load_file, + .for_each = flat_for_each, + .free_data = flat_free_data, + .free = flat_free, +}; + +static bool is_image(const char *data) +{ + if (!strncmp(data, "AJP", 4)) + return true; + if (!strncmp(data, "QNT", 4)) + return true; + return false; +} + +static void read_libl(uint8_t *data, size_t size, struct flat_archive *ar) +{ + struct buffer r; + buffer_init(&r, data, size); + + buffer_skip(&r, 8); + ar->nr_libl_files = buffer_read_int32(&r); + ar->libl_files = xcalloc(ar->nr_libl_files, sizeof(struct flat_entry)); + for (unsigned i = 0; i < ar->nr_libl_files; i++) { + int len = buffer_read_int32(&r); + buffer_skip(&r, len); + buffer_align(&r, 4); + + ar->libl_files[i].type = buffer_read_int32(&r); + ar->libl_files[i].size = buffer_read_int32(&r); + ar->libl_files[i].off = ar->libl_off + r.index; + + if (ar->libl_files[i].type == FLAG_CG && !is_image(buffer_strdata(&r))) { + // XXX: special case: CG usually (not always!) has extra int32 + if (is_image(buffer_strdata(&r)+4)) { + ar->libl_files[i].off += 4; + ar->libl_files[i].size -= 4; + buffer_skip(&r, 4); + } else { + WARNING("Couldn't read CG data in LIBL section"); + } + } + + buffer_skip(&r, ar->libl_files[i].size); + buffer_align(&r, 4); + } + + if (r.index != size) + WARNING("Junk at end of LIBL section"); +} + +static void read_talt(uint8_t *data, size_t size, struct flat_archive *ar) +{ + if (size < 16) + return; + + struct buffer r; + buffer_init(&r, data, size); + + buffer_skip(&r, 8); + ar->nr_talt_files = buffer_read_int32(&r); + ar->talt_files = xcalloc(ar->nr_talt_files, sizeof(struct flat_entry)); + for (unsigned i = 0; i < ar->nr_talt_files; i++) { + ar->talt_files[i].size = buffer_read_int32(&r); + ar->talt_files[i].off = ar->talt_off + r.index; + + if (strncmp(buffer_strdata(&r), "AJP", 4)) + ERROR("NOT AN AJP FILE"); + + buffer_skip(&r, ar->talt_files[i].size); + buffer_align(&r, 4); + + unsigned nr_meta = buffer_read_int32(&r); + for (unsigned j = 0; j < nr_meta; j++) { + unsigned size = buffer_read_int32(&r); + buffer_skip(&r, size); + buffer_align(&r, 4); + buffer_read_int32(&r); + buffer_read_int32(&r); + buffer_read_int32(&r); + buffer_read_int32(&r); + } + } + + if (r.index != size) + WARNING("Junk at end of TALT section"); +} + +struct archive *flat_open(uint8_t *data, size_t size, int *error) +{ + struct flat_archive *ar = NULL; + size_t off, libl_off, talt_off; + struct buffer r; + buffer_init(&r, data, size); + + if (!strncmp(buffer_strdata(&r), "ELNA", 4)) { + buffer_seek(&r, 8); + } + + // FLAT section + if (strncmp(buffer_strdata(&r), "FLAT", 4)) + goto exit_err; + buffer_skip(&r, 4); + off = buffer_read_int32(&r); + buffer_skip(&r, off); + + if (!strncmp(buffer_strdata(&r), "TMNL", 4)) { + buffer_skip(&r, 4); + off = buffer_read_int32(&r); + buffer_skip(&r, off); + } + + // MTLC section + if (strncmp(buffer_strdata(&r), "MTLC", 4)) + goto exit_err; + buffer_skip(&r, 4); + off = buffer_read_int32(&r); + buffer_skip(&r, off); + + // LIBL section + if (strncmp(buffer_strdata(&r), "LIBL", 4)) + goto exit_err; + libl_off = r.index; + buffer_skip(&r, 4); + off = buffer_read_int32(&r); + buffer_skip(&r, off); + + // TALT section + talt_off = r.index; + if (r.index < size) { + if (strncmp(buffer_strdata(&r), "TALT", 4)) + goto exit_err; + buffer_skip(&r, 4); + off = buffer_read_int32(&r); + } + + if (r.index + off < size) + WARNING("Junk at end of FLAT file? %uB/%uB", (unsigned)r.index+off, (unsigned)size); + else if (r.index + off > size) + WARNING("FLAT file truncated? %uB/%uB", (unsigned)size, (unsigned)r.index+off); + + ar = xcalloc(1, sizeof(struct flat_archive)); + ar->libl_off = libl_off; + ar->talt_off = talt_off; + ar->filesize = size; + ar->data = data; + read_libl(data+libl_off, talt_off - libl_off, ar); + read_talt(data+talt_off, size - talt_off, ar); + + ar->ar.ops = &flat_archive_ops; + return &ar->ar; +exit_err: + *error = ARCHIVE_BAD_ARCHIVE_ERROR; + free(ar); + return NULL; +} + +struct archive *flat_open_file(const char *path, possibly_unused int flags, int *error) +{ + size_t size; + uint8_t *data = file_read(path, &size); + if (!data) + return NULL; + + struct flat_archive *ar = (struct flat_archive*)flat_open(data, size, error); + if (!ar) { + free(data); + return NULL; + } + + ar->allocated = true; + return (struct archive*)ar; +} diff --git a/src/meson.build b/src/meson.build index 084bc90..3e7e637 100644 --- a/src/meson.build +++ b/src/meson.build @@ -9,6 +9,7 @@ system4 = ['afa.c', 'dcf.c', 'ex.c', 'file.c', + 'flat.c', 'fnl.c', 'hashtable.c', 'instructions.c', @@ -78,7 +79,7 @@ xsystem4 += flexgen.process('ini_lexer.l') xsystem4 += bisongen.process('ini_parser.y') libsys4 = static_library('sys4', system4, - dependencies : [libm, zlib, tj, webp], + dependencies : [libm, zlib, tj, webp, png], include_directories : incdir) executable('xsystem4', xsystem4, diff --git a/src/tools/ar/afa_extract.c b/src/tools/ar/afa_extract.c new file mode 100644 index 0000000..d693de4 --- /dev/null +++ b/src/tools/ar/afa_extract.c @@ -0,0 +1,433 @@ +/* Copyright (C) 2019 Nunuhara Cabbage + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + */ + +#include +#include +#include +#include +#include +#include +#include "little_endian.h" +#include "system4.h" +#include "system4/archive.h" +#include "system4/buffer.h" +#include "system4/cg.h" +#include "system4/qnt.h" +#include "system4/ajp.h" + +/* + * Makeshift AFA extraction for new AFA format with encrypted TOC. + */ + +struct file_magic { + const char * const ext; + const char * const magic; + size_t len; + bool (*skip)(struct buffer *r); +}; + +static bool skip_ajp(struct buffer *r); +static bool skip_qnt(struct buffer *r); +static bool skip_dcf(struct buffer *r); +static bool skip_flat(struct buffer *r); +static bool skip_zlb(struct buffer *r); +static bool skip_ex(struct buffer *r); + +static struct file_magic file_magic[] = { + { ".ajp", "AJP", 4, skip_ajp }, + { ".qnt", "QNT", 4, skip_qnt }, + { ".dcf", "dcf\x20", 4, skip_dcf }, + { ".flat", "ELNA", 4, skip_flat }, + { ".zlb", "ZLB", 4, skip_zlb }, + { ".ex", "HEAD", 4, skip_ex }, + // TODO: ogg +}; + +struct afa3_entry { + size_t off; + size_t size; +}; + +struct afa3_archive { + struct archive ar; + char *filename; + size_t file_size; + uint32_t data_off; + unsigned nr_files; + struct afa3_entry *files; + void *mmap_ptr; +}; + +static bool afa3_exists(struct archive *ar, int no); +static bool afa3_exists_by_name(struct archive *ar, const char *name); +static struct archive_data *afa3_get(struct archive *_ar, int no); +static struct archive_data *afa3_get_by_name(struct archive *_ar, const char *name); +static bool afa3_load_file(struct archive_data *data); +static void afa3_for_each(struct archive *_ar, void (*iter)(struct archive_data *data, void *user), void *user); +static void afa3_free_data(struct archive_data *ar); +static void afa3_free(struct archive *ar); + +struct archive_ops afa3_archive_ops = { + .exists = afa3_exists, + .exists_by_name = afa3_exists_by_name, + .get = afa3_get, + .get_by_name = afa3_get_by_name, + .load_file = afa3_load_file, + .for_each = afa3_for_each, + .free_data = afa3_free_data, + .free = afa3_free, +}; + +static const char *get_file_extension(char *data) +{ + for (size_t i = 0; i < sizeof(file_magic)/sizeof(*file_magic); i++) { + if (!strncmp(data, file_magic[i].magic, file_magic[i].len)) + return file_magic[i].ext; + } + return ".dat"; +} + +static int _afa3_get_by_name(struct afa3_archive *ar, const char *name) +{ + char filename[512]; + for (unsigned i = 0; i < ar->nr_files; i++) { + snprintf(filename, 512, "%u%s", i, get_file_extension(ar->mmap_ptr+ar->files[i].off)); + if (!strcmp(name, filename)) + return i; + } + return -1; +} + +static bool afa3_exists(struct archive *ar, int no) +{ + return (unsigned)no < ((struct afa3_archive*)ar)->nr_files; +} + +static bool afa3_exists_by_name(struct archive *ar, const char *name) +{ + return _afa3_get_by_name((struct afa3_archive*)ar, name) >= 0; +} + +static struct archive_data *afa3_make_data(struct afa3_archive *ar, unsigned no) +{ + struct afa3_entry *e = &ar->files[no]; + struct archive_data *data = xcalloc(1, sizeof(struct archive_data)); + data->data = ar->mmap_ptr+e->off; + data->size = e->size; + data->name = xmalloc(256); + snprintf(data->name, 256, "%d%s", no, get_file_extension((char*)data->data)); + data->no = no; + data->archive = (struct archive*)ar; + return data; +} + +static struct archive_data *afa3_get(struct archive *_ar, int no) +{ + struct afa3_archive *ar = (struct afa3_archive*)_ar; + if ((unsigned)no >= ar->nr_files) + return NULL; + return afa3_make_data(ar, no); +} + +static struct archive_data *afa3_get_by_name(struct archive *_ar, const char *name) +{ + struct afa3_archive *ar = (struct afa3_archive*)_ar; + int no = _afa3_get_by_name(ar, name); + if (no < 0) + return NULL; + return afa3_make_data(ar, no); +} + +static bool afa3_load_file(possibly_unused struct archive_data *data) +{ + return true; +} + +static void afa3_for_each(struct archive *_ar, void (*iter)(struct archive_data *data, void *user), void *user) +{ + struct afa3_archive *ar = (struct afa3_archive*)_ar; + for (unsigned i = 0; i < ar->nr_files; i++) { + struct archive_data *data = afa3_get(_ar, i); + if (!data) + continue; + iter(data, user); + afa3_free_data(data); + } +} + +static bool afa3_read_header(FILE *f, struct afa3_archive *ar, int *error) +{ + char buf[12]; + if (fread(buf, 12, 1, f) != 1) { + *error = ARCHIVE_FILE_ERROR; + return false; + } + + fseek(f, 0, SEEK_END); + ar->file_size = ftell(f); + fseek(f, 0, SEEK_SET); + + if (strncmp(buf, "AFAH", 4) || LittleEndian_getDW((uint8_t*)buf, 8) != 0x3) { + *error = ARCHIVE_BAD_ARCHIVE_ERROR; + return false; + } + + ar->data_off = LittleEndian_getDW((uint8_t*)buf, 4); + return true; +} + +static bool skip_ajp(struct buffer *r) +{ + // FIXME: validate + size_t off = r->index; + buffer_skip(r, 20); + uint32_t jpeg_off = buffer_read_int32(r); + uint32_t jpeg_size = buffer_read_int32(r); + uint32_t mask_off = buffer_read_int32(r); + uint32_t mask_size = buffer_read_int32(r); + uint32_t size = jpeg_off + jpeg_size; + if (mask_size && mask_off > jpeg_off) { + size = mask_off + mask_size; + } + buffer_seek(r, off+size); + return true; +} + +static bool skip_qnt(struct buffer *r) +{ + // FIXME: validate + struct qnt_header qnt; + qnt_extract_header(r->buf+r->index, &qnt); + buffer_seek(r, r->index + qnt.hdr_size + qnt.pixel_size + qnt.alpha_size); + return true; +} + +static bool skip_dcf(struct buffer *r) +{ + // FIXME: validate + buffer_skip(r, 4); + uint32_t size = buffer_read_int32(r); + buffer_skip(r, size); + + // dfdl header + if (strncmp(buffer_strdata(r), "dfdl", 4)) { + WARNING("DCF magic2 incorrect"); + return false; + } + buffer_skip(r, 4); + size = buffer_read_int32(r); + buffer_skip(r, size); + + // dcgd header + if (strncmp(buffer_strdata(r), "dcgd", 4)) { + WARNING("DCF magic3 incorrect"); + return false; + } + buffer_skip(r, 4); + size = buffer_read_int32(r); + buffer_skip(r, size); + return true; +} + +static bool is_flat_section(const char *data) +{ + if (!strncmp(data, "FLAT", 4)) + return true; + if (!strncmp(data, "TMNL", 4)) + return true; + if (!strncmp(data, "MTLC", 4)) + return true; + if (!strncmp(data, "LIBL", 4)) + return true; + if (!strncmp(data, "TALT", 4)) + return true; + return false; +} + +static bool skip_flat(struct buffer *r) +{ + // FIXME: validate + size_t size; + buffer_skip(r, 8); + + while (is_flat_section(buffer_strdata(r))) { + buffer_skip(r, 4); + size = buffer_read_int32(r); + buffer_skip(r, size); + } + return true; +} + +static bool skip_zlb(struct buffer *r) +{ + // FIXME: validate + buffer_skip(r, 12); + unsigned size = buffer_read_int32(r); + buffer_skip(r, size); + return true; +} + +static bool skip_ex(struct buffer *r) +{ + // FIXME: validate + buffer_skip(r, 8); + if (strncmp(buffer_strdata(r), "EXTF", 4)) { + WARNING("Missing ex file EXTF header at 0x%x", (unsigned)r->index); + return false; + } + buffer_skip(r, 12); + if (strncmp(buffer_strdata(r), "DATA", 4)) { + WARNING("Missing ex file DATA header at 0x%x", (unsigned)r->index); + return false; + } + buffer_skip(r, 4); + + unsigned size = buffer_read_int32(r); + buffer_skip(r, size+4); + return true; +} + +static bool afa3_read_file(struct buffer *r, struct afa3_entry *file) +{ + file->off = r->index; + char *magic = buffer_strdata(r); + for (size_t i = 0; i < sizeof(file_magic)/sizeof(*file_magic); i++) { + if (!strncmp(magic, file_magic[i].magic, file_magic[i].len)) { + file_magic[i].skip(r); + file->size = r->index - file->off; + return true; + } + } + return false; +} + +static bool buffer_seek_next_file(struct buffer *r) +{ + while (r->index < r->size - 4) { + for (size_t i = 0; i < sizeof(file_magic)/sizeof(*file_magic); i++) { + if (!strncmp((char*)r->buf+r->index, file_magic[i].magic, file_magic[i].len)) { + return true; + } + } + r->index++; + } + return false; +} + +static void afa3_read_files(struct afa3_archive *ar) +{ + struct buffer r; + buffer_init(&r, ar->mmap_ptr, ar->file_size); + buffer_seek(&r, ar->data_off+8); + + int nr_alloc = 256; + int file_no = 0; + ar->files = xcalloc(nr_alloc, sizeof(struct afa3_entry)); + + while (r.index+4 < ar->file_size) { + if (file_no >= nr_alloc) { + ar->files = xrealloc_array(ar->files, nr_alloc, nr_alloc+256, sizeof(struct afa3_entry)); + nr_alloc += 256; + } + if (afa3_read_file(&r, &ar->files[file_no])) { + file_no++; + } else { + unsigned old_index = r.index; + WARNING("UNKNOWN FILE TYPE at 0x%x (magic: %02x %02x %02x %02x)", (unsigned)r.index, + r.buf[r.index], r.buf[r.index+1], r.buf[r.index+2], r.buf[r.index+3]); + // try to find next file (brute force) + if (!buffer_seek_next_file(&r)) + break; + WARNING("Skipped %u bytes", r.index - old_index); + } + } + + ar->nr_files = file_no; +} + +struct afa3_archive *afa3_open(const char *file, int flags, int *error) +{ +#ifdef _WIN32 + flags &= ~ARCHIVE_MMAP; +#endif + FILE *fp = NULL; + struct afa3_archive *ar = xcalloc(1, sizeof(struct afa3_archive)); + + if (!(fp = fopen(file, "rb"))) { + WARNING("fopen failed: %s", strerror(errno)); + *error = ARCHIVE_FILE_ERROR; + goto exit_err; + } + if (!afa3_read_header(fp, ar, error)) { + WARNING("afa3_read_header failed"); + fclose(fp); + goto exit_err; + } + + // read archive into memory + if (flags & ARCHIVE_MMAP) { + int fd = open(file, O_RDONLY); + if (fd < 0) { + WARNING("open failed: %s", strerror(errno)); + *error = ARCHIVE_FILE_ERROR; + goto exit_err; + } + ar->mmap_ptr = mmap(0, ar->file_size, PROT_READ, MAP_SHARED, fd, 0); + close(fd); + if (ar->mmap_ptr == MAP_FAILED) { + WARNING("mmap failed: %s", strerror(errno)); + *error = ARCHIVE_FILE_ERROR; + goto exit_err; + } + ar->ar.mmapped = true; + } else { + ar->mmap_ptr = xmalloc(ar->file_size); + if (fread(ar->mmap_ptr, ar->file_size, 1, fp) != 1) { + WARNING("fread failed: %s", strerror(errno)); + *error = ARCHIVE_FILE_ERROR; + goto exit_err; + } + } + + afa3_read_files(ar); + ar->filename = strdup(file); + ar->ar.ops = &afa3_archive_ops; + return ar; +exit_err: + if (fp) + fclose(fp); + free(ar); + return NULL; +} + +static void afa3_free_data(struct archive_data *data) +{ + free(data->name); + free(data); +} + +static void afa3_free(struct archive *_ar) +{ + struct afa3_archive *ar = (struct afa3_archive*)_ar; + if (_ar->mmapped) { + munmap(ar->mmap_ptr, ar->file_size); + } else { + free(ar->mmap_ptr); + } + free(ar->filename); + free(ar->files); + free(ar); +} diff --git a/src/tools/ar/alice-ar.c b/src/tools/ar/alice-ar.c index 55f162c..892c2a2 100644 --- a/src/tools/ar/alice-ar.c +++ b/src/tools/ar/alice-ar.c @@ -24,10 +24,13 @@ #include #include #include +#include +#include +#include "file.h" #include "system4.h" #include "system4/afa.h" #include "system4/ald.h" -#include "file.h" +#include "system4/cg.h" #include "system4/string.h" #include "system4/utfsjis.h" @@ -38,13 +41,15 @@ static void usage(void) puts(""); puts(" -h, --help Display this message and exit"); puts(" -x, --extract Extract archive"); - puts(" -c, --create Create an archive"); - puts(" -a, --add Add to archive"); - puts(" -d, --delete Delete a file from the archive"); + //puts(" -c, --create Create an archive"); + //puts(" -a, --add Add to archive"); + //puts(" -d, --delete Delete a file from the archive"); puts(" -o, --output Specify output file/directory"); puts(" -i, --index Specify file index"); puts(" -n, --name Specify file name"); - puts(" -f, --force Allow replacing existing files when using the --add option"); + puts(" -f, --force Allow overwriting existing files"); + puts(" --images-only Only extract image files (recommended for flat archives)"); + puts(" --raw Don't convert files formats"); } enum { @@ -58,8 +63,15 @@ enum { LOPT_INDEX, LOPT_NAME, LOPT_FORCE, + LOPT_IMAGE_FORMAT, + LOPT_IMAGES_ONLY, + LOPT_RAW, }; +static bool raw = false; +static bool force = false; +static bool images_only = false; + // dirname is allowed to return a pointer to static memory OR modify its input. // This works around the braindamage by ALWAYS returning a pointer to static // memory, at the cost of a string copy. @@ -77,6 +89,23 @@ char *xbasename(const char *path) return basename(buf); } +// Add a trailing slash to a path +char *output_file_dir(const char *path) +{ + if (!path || *path == '\0') + return strdup("./"); // default is cwd + + size_t size = strlen(path); + if (path[size-1] == '/') + return strdup(path); + + char *s = xmalloc(size + 2); + strcpy(s, path); + s[size] = '/'; + s[size+1] = '\0'; + return s; +} + static struct archive *open_ald_archive(const char *path, int *error) { int count = 0; @@ -123,6 +152,10 @@ static struct archive *open_ald_archive(const char *path, int *error) return ar; } +struct afa3_archive *afa3_open(const char *file, int flags, int *error); +struct archive *flat_open_file(const char *path, possibly_unused int flags, int *error); +struct archive *flat_open(uint8_t *data, size_t size, int *error); + static struct archive *open_archive(const char *path, int *error) { size_t len = strlen(path); @@ -133,7 +166,13 @@ static struct archive *open_archive(const char *path, int *error) if (!strcasecmp(ext, ".ald")) { return open_ald_archive(path, error); } else if (!strcasecmp(ext, ".afa")) { - return (struct archive*)afa_open(path, ARCHIVE_MMAP, error); + struct archive *ar = (struct archive*)afa_open(path, ARCHIVE_MMAP, error); + if (!ar && *error == ARCHIVE_BAD_ARCHIVE_ERROR) { + ar = (struct archive*)afa3_open(path, ARCHIVE_MMAP, error); + } + return ar; + } else if (!strcasecmp(ext, "flat")) { + return flat_open_file(path, 0, error); } err: @@ -149,46 +188,227 @@ static void mkdir_for_file(const char *filename) free(tmp); } -static void write_file(struct archive_data *data, const char *output_file) +static void write_webp(struct cg *cg, FILE *f) { - FILE *f = NULL; + uint8_t *out; + size_t len = WebPEncodeLosslessRGBA(cg->pixels, cg->metrics.w, cg->metrics.h, cg->metrics.w*4, &out); + if (!fwrite(out, len, 1, f)) + ERROR("fwrite failed: %s", strerror(errno)); + WebPFree(out); +} - if (!output_file) { - char *u = sjis2utf(data->name, strlen(data->name)); - mkdir_for_file(u); - if (!(f = fopen(u, "wb"))) - ERROR("fopen failed: %s", strerror(errno)); - free(u); - } else if (strcmp(output_file, "-")) { - if (!(f = fopen(output_file, "wb"))) - ERROR("fopen failed: %s", strerror(errno)); +static void write_png(struct cg *cg, FILE *f) +{ + png_structp png_ptr = NULL; + png_infop info_ptr = NULL; + png_byte **row_pointers = NULL; + + png_ptr = png_create_write_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL); + if (!png_ptr) { + WARNING("png_create_write_struct failed"); + goto cleanup; } - if (fwrite(data->data, data->size, 1, f ? f : stdout) != 1) + info_ptr = png_create_info_struct(png_ptr); + if (!info_ptr) { + WARNING("png_create_info_struct failed"); + goto cleanup; + } + + if (setjmp(png_jmpbuf(png_ptr))) { + WARNING("png_init_io failed"); + goto cleanup; + } + + png_init_io(png_ptr, f); + + if (setjmp(png_jmpbuf(png_ptr))) { + WARNING("png_write_header failed"); + goto cleanup; + } + + png_set_IHDR(png_ptr, info_ptr, cg->metrics.w, cg->metrics.h, + 8, PNG_COLOR_TYPE_RGBA, PNG_INTERLACE_NONE, + PNG_COMPRESSION_TYPE_BASE, PNG_FILTER_TYPE_BASE); + png_write_info(png_ptr, info_ptr); + + png_uint_32 stride = cg->metrics.w * 4; + row_pointers = png_malloc(png_ptr, cg->metrics.h * sizeof(png_byte*)); + for (int i = 0; i < cg->metrics.h; i++) { + row_pointers[i] = cg->pixels + i*stride; + } + + if (setjmp(png_jmpbuf(png_ptr))) { + WARNING("png_write_image failed"); + goto cleanup; + } + + png_write_image(png_ptr, row_pointers); + + if (setjmp(png_jmpbuf(png_ptr))) { + WARNING("png_write_end failed"); + goto cleanup; + } + + png_write_end(png_ptr, NULL); +cleanup: + if (row_pointers) + png_free(png_ptr, row_pointers); + if (png_ptr) + png_destroy_write_struct(&png_ptr, info_ptr ? &info_ptr : NULL); + return; +} + +enum imgenc { + IMGENC_PNG, + IMGENC_WEBP, +}; + +struct image_encoder { + void (*write)(struct cg *cg, FILE *f); + const char * const ext; +}; + +static struct image_encoder image_encoders[] = { + [IMGENC_PNG] = { write_png, ".png" }, + [IMGENC_WEBP] = { write_webp, ".webp" }, +}; + +static enum imgenc imgenc = IMGENC_PNG; + +static bool is_image_file(struct archive_data *data) +{ + return cg_check_format(data->data) != ALCG_UNKNOWN; +} + +static char *get_default_filename(struct archive_data *data, const char *ext) +{ + char *u = sjis2utf(data->name, 0); + if (ext) { + size_t ulen = strlen(u); + size_t extlen = strlen(ext); + u = xrealloc(u, ulen + extlen + 1); + memcpy(u+ulen, ext, extlen + 1); + } + return u; +} + +static FILE *checked_fopen(const char *path, const char *mode) +{ + if (file_exists(path) && !force) { + errno = EEXIST; + return NULL; + } + FILE *f = fopen(path, mode); + if (!f) + ERROR("fopen failed: %s", strerror(errno)); + return f; +} + +static bool write_file(struct archive_data *data, const char *output_file) +{ + FILE *f = NULL; + bool output_img = !raw && is_image_file(data); + + if (!output_file) { + char *u = get_default_filename(data, output_img ? image_encoders[imgenc].ext : NULL); + mkdir_for_file(u); + if (!(f = checked_fopen(u, "wb"))) { + free(u); + return false; + } + free(u); + } else if (strcmp(output_file, "-")) { + if (!(f = checked_fopen(output_file, "wb"))) { + return false; + } + } + + if (output_img) { + struct cg *cg = cg_load_data(data); + if (cg) { + image_encoders[imgenc].write(cg, f); + cg_free(cg); + } else { + WARNING("Failed to load CG"); + } + } else if (!images_only && fwrite(data->data, data->size, 1, f ? f : stdout) != 1) { ERROR("fwrite failed: %s", strerror(errno)); + } if (f) fclose(f); + return true; } -static void extract_all_iter(struct archive_data *data, void *_output_dir) -{ - char *output_dir = _output_dir ? _output_dir : "."; - char *file_name = sjis2utf(data->name, 0); - char output_file[PATH_MAX]; +static void extract_all_iter(struct archive_data *data, void *_output_dir); - snprintf(output_file, PATH_MAX, "%s/%s", output_dir, file_name); +static void extract_flat(struct archive_data *data, char *output_dir) +{ + int error; + struct archive *ar = flat_open(data->data, data->size, &error); + if (!ar) { + WARNING("Error opening FLAT archive: %s", archive_strerror(error)); + return; + } + + // generate filename prefix + char *uname = sjis2utf(data->name, 0); + size_t dir_len = strlen(output_dir); + size_t name_len = strlen(uname); + char *prefix = xmalloc(dir_len + name_len + 2); + strcpy(prefix, output_dir); + strcpy(prefix+dir_len, uname); + strcpy(prefix+dir_len+name_len, "."); + //NOTICE("Extracting %s...", uname); + + archive_for_each(ar, extract_all_iter, prefix); + archive_free(ar); + free(prefix); + free(uname); +} + +static bool is_flat_file(const char *data) +{ + if (!strncmp(data, "ELNA", 4)) + data += 8; + return !strncmp(data, "FLAT", 4); +} + +static void extract_all_iter(struct archive_data *data, void *_prefix) +{ + if (!archive_load_file(data)) { + char *u = sjis2utf(data->name, 0); + WARNING("Error loading file: %s", u); + free(u); + return; + } + + if (!raw && is_flat_file((char*)data->data)) { + char *u = sjis2utf(data->name, 0); + NOTICE("Extracting %s...", u); + free(u); + extract_flat(data, _prefix); + return; + } + + char *prefix = _prefix; + bool is_image = is_image_file(data); + char *file_name = get_default_filename(data, !raw && is_image ? image_encoders[imgenc].ext : NULL); + char output_file[PATH_MAX]; + snprintf(output_file, PATH_MAX, "%s%s", prefix, file_name); free(file_name); + if (!is_image && images_only) { + NOTICE("Skipping non-image file: %s", output_file); + return; + } mkdir_for_file(output_file); - printf("%s\n", output_file); - - if (!archive_load_file(data)) { - WARNING("Error loading file: %s", output_file); - return; - } - write_file(data, output_file); + if (write_file(data, output_file)) + NOTICE("%s", output_file); + else + NOTICE("Skipping existing file: %s", output_file); } static void list_all_iter(struct archive_data *data, possibly_unused void *_) @@ -201,7 +421,7 @@ static void list_all_iter(struct archive_data *data, possibly_unused void *_) int main(int argc, char *argv[]) { char *output_file = NULL; - char *input_file = NULL; + possibly_unused char *input_file = NULL; char *file_name = NULL; int file_index = -1; @@ -210,23 +430,25 @@ int main(int argc, char *argv[]) bool add = false; bool delete = false; bool list = false; - //bool force = false; while (1) { static struct option long_options[] = { - { "help", no_argument, 0, LOPT_HELP }, - { "extract", no_argument, 0, LOPT_EXTRACT }, - { "create", required_argument, 0, LOPT_CREATE }, - { "add", required_argument, 0, LOPT_ADD }, - { "delete", no_argument, 0, LOPT_DELETE }, - { "list", no_argument, 0, LOPT_LIST }, - { "output", required_argument, 0, LOPT_OUTPUT }, - { "index", required_argument, 0, LOPT_INDEX }, - { "name", required_argument, 0, LOPT_NAME }, - { "force", no_argument, 0, LOPT_FORCE }, + { "help", no_argument, 0, LOPT_HELP }, + { "extract", no_argument, 0, LOPT_EXTRACT }, + { "create", required_argument, 0, LOPT_CREATE }, + { "add", required_argument, 0, LOPT_ADD }, + { "delete", no_argument, 0, LOPT_DELETE }, + { "list", no_argument, 0, LOPT_LIST }, + { "output", required_argument, 0, LOPT_OUTPUT }, + { "index", required_argument, 0, LOPT_INDEX }, + { "name", required_argument, 0, LOPT_NAME }, + { "force", no_argument, 0, LOPT_FORCE }, + { "image-format", required_argument, 0, LOPT_IMAGE_FORMAT }, + { "images-only", no_argument, 0, LOPT_IMAGES_ONLY }, + { "raw", no_argument, 0, LOPT_RAW }, }; int option_index = 0; - int c = getopt_long(argc, argv, "hxc:a:dlo:i:n:", long_options, &option_index); + int c = getopt_long(argc, argv, "hxc:a:dlo:i:n:f", long_options, &option_index); if (c == -1) break; @@ -245,6 +467,7 @@ int main(int argc, char *argv[]) break; case 'a': case LOPT_ADD: + add = true; input_file = optarg; break; case 'd': @@ -267,6 +490,24 @@ int main(int argc, char *argv[]) case 'n': file_name = optarg; break; + case 'f': + case LOPT_FORCE: + force = true; + break; + case LOPT_IMAGE_FORMAT: + if (!strcasecmp(optarg, "png")) + imgenc = IMGENC_PNG; + else if (!strcasecmp(optarg, "webp")) + imgenc = IMGENC_WEBP; + else + ERROR("Unrecognized image format: \"%s\"", optarg); + break; + case LOPT_IMAGES_ONLY: + images_only = true; + break; + case LOPT_RAW: + raw = true; + break; case '?': ERROR("Unrecognized command line argument"); } @@ -303,12 +544,19 @@ int main(int argc, char *argv[]) if (!d) ERROR("No file with index %d", file_index); write_file(d, output_file); + archive_free_data(d); } else if (file_name) { char *u = utf2sjis(file_name, strlen(file_name)); struct archive_data *d = archive_get_by_name(ar, u); + if (!d) + ERROR("No file with name \"%s\"", u); write_file(d, output_file); + archive_free_data(d); + free(u); } else { + output_file = output_file_dir(output_file); archive_for_each(ar, extract_all_iter, output_file); + free(output_file); } } else if (add) { WARNING("adding files to archive not yet implemented"); diff --git a/src/tools/ar/meson.build b/src/tools/ar/meson.build index cd8fd29..06886b7 100644 --- a/src/tools/ar/meson.build +++ b/src/tools/ar/meson.build @@ -1,4 +1,5 @@ alicear = ['alice-ar.c', + 'afa_extract.c', ] executable('alice-ar', alicear,