alice-ar: various improvements

* Can now extract files from .afa version 3 archives
* Can now extract files from .flat files
  * Includes recursively extracting .flat files contained in other
    archive formats
* Added option to extract images only
* Now converts images to PNG format by default
  * WebP format also supported

Because the table of contents is encrypted in .afa version 3 archives,
the support is a bit limited. Only known file types can be extracted and
they do not retain their original filenames (they are given numbered
filenames instead).

include/system4/ajp.h

@@ -24,6 +24,6 @@
 struct cg;
 
 bool ajp_checkfmt(const uint8_t *data);
-void ajp_extract(uint8_t *data, size_t size, struct cg *cg);
+void ajp_extract(const uint8_t *data, size_t size, struct cg *cg);
 
 #endif /* SYSTEM4_AJP_H */

include/system4/buffer.h

@@ -49,4 +49,9 @@ static inline void buffer_seek(struct buffer *r, size_t off)
 	r->index = off;
 }
 
+static inline void buffer_align(struct buffer *r, int p)
+{
+	r->index = (r->index + (p-1)) & ~(p-1);
+}
+
 #endif /* SYSTEM4_BUFFER_H */

meson.build

@@ -12,6 +12,7 @@ gl = dependency('GL')
 glew = dependency('glew')
 tj = dependency('libturbojpeg')
 webp = dependency('libwebp')
+png = dependency('libpng')
 chibi = dependency('chibi-scheme', required : get_option('debugger'))
 
 flex = find_program('flex')

src/ajp.c

@@ -54,7 +54,7 @@ struct ajp_header {
 	uint32_t mask_size;
 };
 
-static void ajp_extract_header(uint8_t *data, struct ajp_header *ajp)
+static void ajp_extract_header(const uint8_t *data, struct ajp_header *ajp)
 {
 	ajp->width     = LittleEndian_getDW(data, 12);
 	ajp->height    = LittleEndian_getDW(data, 16);
@@ -76,7 +76,7 @@ static void ajp_init_metrics(struct ajp_header *ajp, struct cg_metrics *dst)
 	dst->alpha_pitch = 1;
 }
 
-void ajp_extract(uint8_t *data, size_t size, struct cg *cg)
+void ajp_extract(const uint8_t *data, size_t size, struct cg *cg)
 {
 	uint8_t *buf = NULL, *jpeg_data = NULL, *mask_data = NULL;
 	int width, height, subsamp;

src/cg.c

@@ -22,7 +22,6 @@
 #include "system4/cg.h"
 #include "system4/ajp.h"
 #include "system4/dcf.h"
-#include "system4/flat.h"
 #include "system4/qnt.h"
 #include "system4/webp.h"
 

src/flat.c

@@ -0,0 +1,330 @@
+/* Copyright (C) 2019 Nunuhara Cabbage <nunuhara@haniwa.technology>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://gnu.org/licenses/>.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <zlib.h>
+#include "little_endian.h"
+#include "file.h"
+#include "system4.h"
+#include "system4/archive.h"
+#include "system4/buffer.h"
+#include "system4/cg.h"
+#include "system4/ajp.h"
+
+enum flat_data_type {
+	FLAG_CG  = 2,
+	FLAT_ZLIB = 5,
+};
+
+struct flat_entry {
+	size_t off;
+	size_t size;
+	int type;
+};
+
+struct flat_data {
+	struct archive_data super;
+	struct flat_entry *entry;
+	bool allocated;
+};
+
+struct flat_archive {
+	struct archive ar;
+	bool allocated;
+	size_t libl_off;
+	size_t talt_off;
+	size_t filesize;
+	unsigned nr_libl_files;
+	struct flat_entry *libl_files;
+	unsigned nr_talt_files;
+	struct flat_entry *talt_files;
+	uint8_t *data;
+};
+
+static const char *get_file_extension(int type, const char *data)
+{
+	switch (type) {
+	case FLAG_CG:
+		if (!strncmp(data, "AJP", 4))
+			return ".ajp";
+		if (!strncmp(data, "QNT", 4))
+			return ".qnt";
+		return ".img";
+	case FLAT_ZLIB:
+		return ".z.dat";
+	default:
+		return ".dat";
+	}
+}
+
+static void flat_free_data(struct archive_data *data)
+{
+	struct flat_data *flat = (struct flat_data*)data;
+	if (flat->allocated)
+		free(data->data);
+	free(data->name);
+	free(data);
+}
+
+static void flat_free(struct archive *_ar)
+{
+	struct flat_archive *ar = (struct flat_archive*)_ar;
+	if (ar->allocated)
+		free(ar->data);
+	free(ar->libl_files);
+	free(ar->talt_files);
+	free(ar);
+}
+
+static struct flat_entry *flat_get_entry(struct flat_archive *ar, unsigned no)
+{
+	if (no < ar->nr_libl_files)
+		return &ar->libl_files[no];
+
+	no -= ar->nr_libl_files;
+	if (no < ar->nr_talt_files)
+		return &ar->talt_files[no];
+
+	return NULL;
+}
+
+static struct archive_data *flat_get(struct archive *_ar, int no)
+{
+	struct flat_archive *ar = (struct flat_archive*)_ar;
+	struct flat_entry *e = flat_get_entry(ar, no);
+	struct flat_data *data = xcalloc(1, sizeof(struct flat_data));
+
+	data->super.data = ar->data + e->off;
+	data->super.size = e->size;
+	data->super.name = xmalloc(256);
+	snprintf(data->super.name, 256, "%d%s", no, get_file_extension(e->type, (const char*)ar->data+e->off));
+	data->super.no = no;
+	data->super.archive = _ar;
+	data->entry = e;
+	return &data->super;
+}
+
+static bool flat_load_file(possibly_unused struct archive_data *data)
+{
+	struct flat_archive *ar = (struct flat_archive*)data->archive;
+	struct flat_data *flatdata = (struct flat_data*)data;
+	struct flat_entry *e = flatdata->entry;
+
+	// inflate zlib compressed data
+	if (e->type == FLAT_ZLIB && ar->data[e->off+4] == 0x78) {
+		unsigned long size = LittleEndian_getDW(ar->data, e->off);
+		uint8_t *out = xmalloc(size);
+		if (uncompress(out, &size, ar->data + e->off + 4, e->size - 4) != Z_OK) {
+			WARNING("uncompress failed");
+			free(out);
+			return false;
+		}
+		data->data = out;
+		data->size = size;
+		flatdata->allocated = true;
+	}
+
+	return true;
+}
+
+static void flat_for_each(struct archive *_ar, void (*iter)(struct archive_data *data, void *user), void *user)
+{
+	struct flat_archive *ar = (struct flat_archive*)_ar;
+	for (unsigned i = 0; i < ar->nr_libl_files + ar->nr_talt_files; i++) {
+		struct archive_data *data = flat_get(_ar, i);
+		if (!data)
+			continue;
+		iter(data, user);
+		flat_free_data(data);
+	}
+}
+
+static struct archive_ops flat_archive_ops = {
+	.get = flat_get,
+	.load_file = flat_load_file,
+	.for_each = flat_for_each,
+	.free_data = flat_free_data,
+	.free = flat_free,
+};
+
+static bool is_image(const char *data)
+{
+	if (!strncmp(data, "AJP", 4))
+		return true;
+	if (!strncmp(data, "QNT", 4))
+		return true;
+	return false;
+}
+
+static void read_libl(uint8_t *data, size_t size, struct flat_archive *ar)
+{
+	struct buffer r;
+	buffer_init(&r, data, size);
+
+	buffer_skip(&r, 8);
+	ar->nr_libl_files = buffer_read_int32(&r);
+	ar->libl_files = xcalloc(ar->nr_libl_files, sizeof(struct flat_entry));
+	for (unsigned i = 0; i < ar->nr_libl_files; i++) {
+		int len = buffer_read_int32(&r);
+		buffer_skip(&r, len);
+		buffer_align(&r, 4);
+
+		ar->libl_files[i].type = buffer_read_int32(&r);
+		ar->libl_files[i].size = buffer_read_int32(&r);
+		ar->libl_files[i].off = ar->libl_off + r.index;
+
+		if (ar->libl_files[i].type == FLAG_CG && !is_image(buffer_strdata(&r))) {
+			// XXX: special case: CG usually (not always!) has extra int32
+			if (is_image(buffer_strdata(&r)+4)) {
+				ar->libl_files[i].off += 4;
+				ar->libl_files[i].size -= 4;
+				buffer_skip(&r, 4);
+			} else {
+				WARNING("Couldn't read CG data in LIBL section");
+			}
+		}
+
+		buffer_skip(&r, ar->libl_files[i].size);
+		buffer_align(&r, 4);
+	}
+
+	if (r.index != size)
+		WARNING("Junk at end of LIBL section");
+}
+
+static void read_talt(uint8_t *data, size_t size, struct flat_archive *ar)
+{
+	if (size < 16)
+		return;
+
+	struct buffer r;
+	buffer_init(&r, data, size);
+
+	buffer_skip(&r, 8);
+	ar->nr_talt_files = buffer_read_int32(&r);
+	ar->talt_files = xcalloc(ar->nr_talt_files, sizeof(struct flat_entry));
+	for (unsigned i = 0; i < ar->nr_talt_files; i++) {
+		ar->talt_files[i].size = buffer_read_int32(&r);
+		ar->talt_files[i].off = ar->talt_off + r.index;
+
+		if (strncmp(buffer_strdata(&r), "AJP", 4))
+			ERROR("NOT AN AJP FILE");
+
+		buffer_skip(&r, ar->talt_files[i].size);
+		buffer_align(&r, 4);
+
+		unsigned nr_meta = buffer_read_int32(&r);
+		for (unsigned j = 0; j < nr_meta; j++) {
+			unsigned size = buffer_read_int32(&r);
+			buffer_skip(&r, size);
+			buffer_align(&r, 4);
+			buffer_read_int32(&r);
+			buffer_read_int32(&r);
+			buffer_read_int32(&r);
+			buffer_read_int32(&r);
+		}
+	}
+
+	if (r.index != size)
+		WARNING("Junk at end of TALT section");
+}
+
+struct archive *flat_open(uint8_t *data, size_t size, int *error)
+{
+	struct flat_archive *ar = NULL;
+	size_t off, libl_off, talt_off;
+	struct buffer r;
+	buffer_init(&r, data, size);
+
+	if (!strncmp(buffer_strdata(&r), "ELNA", 4)) {
+		buffer_seek(&r, 8);
+	}
+
+	// FLAT section
+	if (strncmp(buffer_strdata(&r), "FLAT", 4))
+		goto exit_err;
+	buffer_skip(&r, 4);
+	off = buffer_read_int32(&r);
+	buffer_skip(&r, off);
+
+	if (!strncmp(buffer_strdata(&r), "TMNL", 4)) {
+		buffer_skip(&r, 4);
+		off = buffer_read_int32(&r);
+		buffer_skip(&r, off);
+	}
+
+	// MTLC section
+	if (strncmp(buffer_strdata(&r), "MTLC", 4))
+		goto exit_err;
+	buffer_skip(&r, 4);
+	off = buffer_read_int32(&r);
+	buffer_skip(&r, off);
+
+	// LIBL section
+	if (strncmp(buffer_strdata(&r), "LIBL", 4))
+		goto exit_err;
+	libl_off = r.index;
+	buffer_skip(&r, 4);
+	off = buffer_read_int32(&r);
+	buffer_skip(&r, off);
+
+	// TALT section
+	talt_off = r.index;
+	if (r.index < size) {
+		if (strncmp(buffer_strdata(&r), "TALT", 4))
+			goto exit_err;
+		buffer_skip(&r, 4);
+		off = buffer_read_int32(&r);
+	}
+
+	if (r.index + off < size)
+		WARNING("Junk at end of FLAT file? %uB/%uB", (unsigned)r.index+off, (unsigned)size);
+	else if (r.index + off > size)
+		WARNING("FLAT file truncated? %uB/%uB", (unsigned)size, (unsigned)r.index+off);
+
+	ar = xcalloc(1, sizeof(struct flat_archive));
+	ar->libl_off = libl_off;
+	ar->talt_off = talt_off;
+	ar->filesize = size;
+	ar->data = data;
+	read_libl(data+libl_off, talt_off - libl_off, ar);
+	read_talt(data+talt_off, size - talt_off, ar);
+
+	ar->ar.ops = &flat_archive_ops;
+	return &ar->ar;
+exit_err:
+	*error = ARCHIVE_BAD_ARCHIVE_ERROR;
+	free(ar);
+	return NULL;
+}
+
+struct archive *flat_open_file(const char *path, possibly_unused int flags, int *error)
+{
+	size_t size;
+	uint8_t *data = file_read(path, &size);
+	if (!data)
+		return NULL;
+
+	struct flat_archive *ar = (struct flat_archive*)flat_open(data, size, error);
+	if (!ar) {
+		free(data);
+		return NULL;
+	}
+
+	ar->allocated = true;
+	return (struct archive*)ar;
+}

src/meson.build

@@ -9,6 +9,7 @@ system4 = ['afa.c',
            'dcf.c',
            'ex.c',
            'file.c',
+           'flat.c',
            'fnl.c',
            'hashtable.c',
            'instructions.c',
@@ -78,7 +79,7 @@ xsystem4 += flexgen.process('ini_lexer.l')
 xsystem4 += bisongen.process('ini_parser.y')
 
 libsys4 = static_library('sys4', system4,
-                         dependencies : [libm, zlib, tj, webp],
+                         dependencies : [libm, zlib, tj, webp, png],
                          include_directories : incdir)
 
 executable('xsystem4', xsystem4,

src/tools/ar/afa_extract.c

@@ -0,0 +1,433 @@
+/* Copyright (C) 2019 Nunuhara Cabbage <nunuhara@haniwa.technology>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://gnu.org/licenses/>.
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include "little_endian.h"
+#include "system4.h"
+#include "system4/archive.h"
+#include "system4/buffer.h"
+#include "system4/cg.h"
+#include "system4/qnt.h"
+#include "system4/ajp.h"
+
+/*
+ * Makeshift AFA extraction for new AFA format with encrypted TOC.
+ */
+
+struct file_magic {
+	const char * const ext;
+	const char * const magic;
+	size_t len;
+	bool (*skip)(struct buffer *r);
+};
+
+static bool skip_ajp(struct buffer *r);
+static bool skip_qnt(struct buffer *r);
+static bool skip_dcf(struct buffer *r);
+static bool skip_flat(struct buffer *r);
+static bool skip_zlb(struct buffer *r);
+static bool skip_ex(struct buffer *r);
+
+static struct file_magic file_magic[] = {
+	{ ".ajp",  "AJP",     4, skip_ajp  },
+	{ ".qnt",  "QNT",     4, skip_qnt  },
+	{ ".dcf",  "dcf\x20", 4, skip_dcf  },
+	{ ".flat", "ELNA",    4, skip_flat },
+	{ ".zlb",  "ZLB",     4, skip_zlb  },
+	{ ".ex",   "HEAD",    4, skip_ex   },
+	// TODO: ogg
+};
+
+struct afa3_entry {
+	size_t off;
+	size_t size;
+};
+
+struct afa3_archive {
+	struct archive ar;
+	char *filename;
+	size_t file_size;
+	uint32_t data_off;
+	unsigned nr_files;
+	struct afa3_entry *files;
+	void *mmap_ptr;
+};
+
+static bool afa3_exists(struct archive *ar, int no);
+static bool afa3_exists_by_name(struct archive *ar, const char *name);
+static struct archive_data *afa3_get(struct archive *_ar, int no);
+static struct archive_data *afa3_get_by_name(struct archive *_ar, const char *name);
+static bool afa3_load_file(struct archive_data *data);
+static void afa3_for_each(struct archive *_ar, void (*iter)(struct archive_data *data, void *user), void *user);
+static void afa3_free_data(struct archive_data *ar);
+static void afa3_free(struct archive *ar);
+
+struct archive_ops afa3_archive_ops = {
+	.exists = afa3_exists,
+	.exists_by_name = afa3_exists_by_name,
+	.get = afa3_get,
+	.get_by_name = afa3_get_by_name,
+	.load_file = afa3_load_file,
+	.for_each = afa3_for_each,
+	.free_data = afa3_free_data,
+	.free = afa3_free,
+};
+
+static const char *get_file_extension(char *data)
+{
+	for (size_t i = 0; i < sizeof(file_magic)/sizeof(*file_magic); i++) {
+		if (!strncmp(data, file_magic[i].magic, file_magic[i].len))
+			return file_magic[i].ext;
+	}
+	return ".dat";
+}
+
+static int _afa3_get_by_name(struct afa3_archive *ar, const char *name)
+{
+	char filename[512];
+	for (unsigned i = 0; i < ar->nr_files; i++) {
+		snprintf(filename, 512, "%u%s", i, get_file_extension(ar->mmap_ptr+ar->files[i].off));
+		if (!strcmp(name, filename))
+			return i;
+	}
+	return -1;
+}
+
+static bool afa3_exists(struct archive *ar, int no)
+{
+	return (unsigned)no < ((struct afa3_archive*)ar)->nr_files;
+}
+
+static bool afa3_exists_by_name(struct archive *ar, const char *name)
+{
+	return _afa3_get_by_name((struct afa3_archive*)ar, name) >= 0;
+}
+
+static struct archive_data *afa3_make_data(struct afa3_archive *ar, unsigned no)
+{
+	struct afa3_entry *e = &ar->files[no];
+	struct archive_data *data = xcalloc(1, sizeof(struct archive_data));
+	data->data = ar->mmap_ptr+e->off;
+	data->size = e->size;
+	data->name = xmalloc(256);
+	snprintf(data->name, 256, "%d%s", no, get_file_extension((char*)data->data));
+	data->no = no;
+	data->archive = (struct archive*)ar;
+	return data;
+}
+
+static struct archive_data *afa3_get(struct archive *_ar, int no)
+{
+	struct afa3_archive *ar = (struct afa3_archive*)_ar;
+	if ((unsigned)no >= ar->nr_files)
+		return NULL;
+	return afa3_make_data(ar, no);
+}
+
+static struct archive_data *afa3_get_by_name(struct archive *_ar, const char *name)
+{
+	struct afa3_archive *ar = (struct afa3_archive*)_ar;
+	int no = _afa3_get_by_name(ar, name);
+	if (no < 0)
+		return NULL;
+	return afa3_make_data(ar, no);
+}
+
+static bool afa3_load_file(possibly_unused struct archive_data *data)
+{
+	return true;
+}
+
+static void afa3_for_each(struct archive *_ar, void (*iter)(struct archive_data *data, void *user), void *user)
+{
+	struct afa3_archive *ar = (struct afa3_archive*)_ar;
+	for (unsigned i = 0; i < ar->nr_files; i++) {
+		struct archive_data *data = afa3_get(_ar, i);
+		if (!data)
+			continue;
+		iter(data, user);
+		afa3_free_data(data);
+	}
+}
+
+static bool afa3_read_header(FILE *f, struct afa3_archive *ar, int *error)
+{
+	char buf[12];
+	if (fread(buf, 12, 1, f) != 1) {
+		*error = ARCHIVE_FILE_ERROR;
+		return false;
+	}
+
+	fseek(f, 0, SEEK_END);
+	ar->file_size = ftell(f);
+	fseek(f, 0, SEEK_SET);
+
+	if (strncmp(buf, "AFAH", 4) || LittleEndian_getDW((uint8_t*)buf, 8) != 0x3) {
+		*error = ARCHIVE_BAD_ARCHIVE_ERROR;
+		return false;
+	}
+
+	ar->data_off = LittleEndian_getDW((uint8_t*)buf, 4);
+	return true;
+}
+
+static bool skip_ajp(struct buffer *r)
+{
+	// FIXME: validate
+	size_t off = r->index;
+	buffer_skip(r, 20);
+	uint32_t jpeg_off = buffer_read_int32(r);
+	uint32_t jpeg_size = buffer_read_int32(r);
+	uint32_t mask_off = buffer_read_int32(r);
+	uint32_t mask_size = buffer_read_int32(r);
+	uint32_t size = jpeg_off + jpeg_size;
+	if (mask_size && mask_off > jpeg_off) {
+		size = mask_off + mask_size;
+	}
+	buffer_seek(r, off+size);
+	return true;
+}
+
+static bool skip_qnt(struct buffer *r)
+{
+	// FIXME: validate
+	struct qnt_header qnt;
+	qnt_extract_header(r->buf+r->index, &qnt);
+	buffer_seek(r, r->index + qnt.hdr_size + qnt.pixel_size + qnt.alpha_size);
+	return true;
+}
+
+static bool skip_dcf(struct buffer *r)
+{
+	// FIXME: validate
+	buffer_skip(r, 4);
+	uint32_t size = buffer_read_int32(r);
+	buffer_skip(r, size);
+
+	// dfdl header
+	if (strncmp(buffer_strdata(r), "dfdl", 4)) {
+		WARNING("DCF magic2 incorrect");
+		return false;
+	}
+	buffer_skip(r, 4);
+	size = buffer_read_int32(r);
+	buffer_skip(r, size);
+
+	// dcgd header
+	if (strncmp(buffer_strdata(r), "dcgd", 4)) {
+		WARNING("DCF magic3 incorrect");
+		return false;
+	}
+	buffer_skip(r, 4);
+	size = buffer_read_int32(r);
+	buffer_skip(r, size);
+	return true;
+}
+
+static bool is_flat_section(const char *data)
+{
+	if (!strncmp(data, "FLAT", 4))
+		return true;
+	if (!strncmp(data, "TMNL", 4))
+		return true;
+	if (!strncmp(data, "MTLC", 4))
+		return true;
+	if (!strncmp(data, "LIBL", 4))
+		return true;
+	if (!strncmp(data, "TALT", 4))
+		return true;
+	return false;
+}
+
+static bool skip_flat(struct buffer *r)
+{
+	// FIXME: validate
+	size_t size;
+	buffer_skip(r, 8);
+
+	while (is_flat_section(buffer_strdata(r))) {
+		buffer_skip(r, 4);
+		size = buffer_read_int32(r);
+		buffer_skip(r, size);
+	}
+	return true;
+}
+
+static bool skip_zlb(struct buffer *r)
+{
+	// FIXME: validate
+	buffer_skip(r, 12);
+	unsigned size = buffer_read_int32(r);
+	buffer_skip(r, size);
+	return true;
+}
+
+static bool skip_ex(struct buffer *r)
+{
+	// FIXME: validate
+	buffer_skip(r, 8);
+	if (strncmp(buffer_strdata(r), "EXTF", 4)) {
+		WARNING("Missing ex file EXTF header at 0x%x", (unsigned)r->index);
+		return false;
+	}
+	buffer_skip(r, 12);
+	if (strncmp(buffer_strdata(r), "DATA", 4)) {
+		WARNING("Missing ex file DATA header at 0x%x", (unsigned)r->index);
+		return false;
+	}
+	buffer_skip(r, 4);
+
+	unsigned size = buffer_read_int32(r);
+	buffer_skip(r, size+4);
+	return true;
+}
+
+static bool afa3_read_file(struct buffer *r, struct afa3_entry *file)
+{
+	file->off = r->index;
+	char *magic = buffer_strdata(r);
+	for (size_t i = 0; i < sizeof(file_magic)/sizeof(*file_magic); i++) {
+		if (!strncmp(magic, file_magic[i].magic, file_magic[i].len)) {
+			file_magic[i].skip(r);
+			file->size = r->index - file->off;
+			return true;
+		}
+	}
+	return false;
+}
+
+static bool buffer_seek_next_file(struct buffer *r)
+{
+	while (r->index < r->size - 4) {
+		for (size_t i = 0; i < sizeof(file_magic)/sizeof(*file_magic); i++) {
+			if (!strncmp((char*)r->buf+r->index, file_magic[i].magic, file_magic[i].len)) {
+				return true;
+			}
+		}
+		r->index++;
+	}
+	return false;
+}
+
+static void afa3_read_files(struct afa3_archive *ar)
+{
+	struct buffer r;
+	buffer_init(&r, ar->mmap_ptr, ar->file_size);
+	buffer_seek(&r, ar->data_off+8);
+
+	int nr_alloc = 256;
+	int file_no = 0;
+	ar->files = xcalloc(nr_alloc, sizeof(struct afa3_entry));
+
+	while (r.index+4 < ar->file_size) {
+		if (file_no >= nr_alloc) {
+			ar->files = xrealloc_array(ar->files, nr_alloc, nr_alloc+256, sizeof(struct afa3_entry));
+			nr_alloc += 256;
+		}
+		if (afa3_read_file(&r, &ar->files[file_no])) {
+			file_no++;
+		} else {
+			unsigned old_index = r.index;
+			WARNING("UNKNOWN FILE TYPE at 0x%x (magic: %02x %02x %02x %02x)", (unsigned)r.index,
+				r.buf[r.index], r.buf[r.index+1], r.buf[r.index+2], r.buf[r.index+3]);
+			// try to find next file (brute force)
+			if (!buffer_seek_next_file(&r))
+				break;
+			WARNING("Skipped %u bytes", r.index - old_index);
+		}
+	}
+
+	ar->nr_files = file_no;
+}
+
+struct afa3_archive *afa3_open(const char *file, int flags, int *error)
+{
+#ifdef _WIN32
+	flags &= ~ARCHIVE_MMAP;
+#endif
+	FILE *fp = NULL;
+	struct afa3_archive *ar = xcalloc(1, sizeof(struct afa3_archive));
+
+	if (!(fp = fopen(file, "rb"))) {
+		WARNING("fopen failed: %s", strerror(errno));
+		*error = ARCHIVE_FILE_ERROR;
+		goto exit_err;
+	}
+	if (!afa3_read_header(fp, ar, error)) {
+		WARNING("afa3_read_header failed");
+		fclose(fp);
+		goto exit_err;
+	}
+
+	// read archive into memory
+	if (flags & ARCHIVE_MMAP) {
+		int fd = open(file, O_RDONLY);
+		if (fd < 0) {
+			WARNING("open failed: %s", strerror(errno));
+			*error = ARCHIVE_FILE_ERROR;
+			goto exit_err;
+		}
+		ar->mmap_ptr = mmap(0, ar->file_size, PROT_READ, MAP_SHARED, fd, 0);
+		close(fd);
+		if (ar->mmap_ptr == MAP_FAILED) {
+			WARNING("mmap failed: %s", strerror(errno));
+			*error = ARCHIVE_FILE_ERROR;
+			goto exit_err;
+		}
+		ar->ar.mmapped = true;
+	} else {
+		ar->mmap_ptr = xmalloc(ar->file_size);
+		if (fread(ar->mmap_ptr, ar->file_size, 1, fp) != 1) {
+			WARNING("fread failed: %s", strerror(errno));
+			*error = ARCHIVE_FILE_ERROR;
+			goto exit_err;
+		}
+	}
+
+	afa3_read_files(ar);
+	ar->filename = strdup(file);
+	ar->ar.ops = &afa3_archive_ops;
+	return ar;
+exit_err:
+	if (fp)
+		fclose(fp);
+	free(ar);
+	return NULL;
+}
+
+static void afa3_free_data(struct archive_data *data)
+{
+	free(data->name);
+	free(data);
+}
+
+static void afa3_free(struct archive *_ar)
+{
+	struct afa3_archive *ar = (struct afa3_archive*)_ar;
+	if (_ar->mmapped) {
+		munmap(ar->mmap_ptr, ar->file_size);
+	} else {
+		free(ar->mmap_ptr);
+	}
+	free(ar->filename);
+	free(ar->files);
+	free(ar);
+}

src/tools/ar/alice-ar.c

@@ -24,10 +24,13 @@
 #include <getopt.h>
 #include <limits.h>
 #include <libgen.h>
+#include <png.h>
+#include <webp/encode.h>
+#include "file.h"
 #include "system4.h"
 #include "system4/afa.h"
 #include "system4/ald.h"
-#include "file.h"
+#include "system4/cg.h"
 #include "system4/string.h"
 #include "system4/utfsjis.h"
 
@@ -38,13 +41,15 @@ static void usage(void)
 	puts("");
 	puts("    -h, --help                 Display this message and exit");
 	puts("    -x, --extract              Extract archive");
-	puts("    -c, --create <manifest>    Create an archive");
-	puts("    -a, --add <file>           Add <file> to archive");
-	puts("    -d, --delete               Delete a file from the archive");
+	//puts("    -c, --create <manifest>    Create an archive");
+	//puts("    -a, --add <file>           Add <file> to archive");
+	//puts("    -d, --delete               Delete a file from the archive");
 	puts("    -o, --output               Specify output file/directory");
 	puts("    -i, --index <index>        Specify file index");
 	puts("    -n, --name <name>          Specify file name");
-	puts("    -f, --force                Allow replacing existing files when using the --add option");
+	puts("    -f, --force                Allow overwriting existing files");
+	puts("        --images-only          Only extract image files (recommended for flat archives)");
+	puts("        --raw                  Don't convert files formats");
 }
 
 enum {
@@ -58,8 +63,15 @@ enum {
 	LOPT_INDEX,
 	LOPT_NAME,
 	LOPT_FORCE,
+	LOPT_IMAGE_FORMAT,
+	LOPT_IMAGES_ONLY,
+	LOPT_RAW,
 };
 
+static bool raw = false;
+static bool force = false;
+static bool images_only = false;
+
 // dirname is allowed to return a pointer to static memory OR modify its input.
 // This works around the braindamage by ALWAYS returning a pointer to static
 // memory, at the cost of a string copy.
@@ -77,6 +89,23 @@ char *xbasename(const char *path)
 	return basename(buf);
 }
 
+// Add a trailing slash to a path
+char *output_file_dir(const char *path)
+{
+	if (!path || *path == '\0')
+		return strdup("./"); // default is cwd
+
+	size_t size = strlen(path);
+	if (path[size-1] == '/')
+		return strdup(path);
+
+	char *s = xmalloc(size + 2);
+	strcpy(s, path);
+	s[size] = '/';
+	s[size+1] = '\0';
+	return s;
+}
+
 static struct archive *open_ald_archive(const char *path, int *error)
 {
 	int count = 0;
@@ -123,6 +152,10 @@ static struct archive *open_ald_archive(const char *path, int *error)
 	return ar;
 }
 
+struct afa3_archive *afa3_open(const char *file, int flags, int *error);
+struct archive *flat_open_file(const char *path, possibly_unused int flags, int *error);
+struct archive *flat_open(uint8_t *data, size_t size, int *error);
+
 static struct archive *open_archive(const char *path, int *error)
 {
 	size_t len = strlen(path);
@@ -133,7 +166,13 @@ static struct archive *open_archive(const char *path, int *error)
 	if (!strcasecmp(ext, ".ald")) {
 		return open_ald_archive(path, error);
 	} else if (!strcasecmp(ext, ".afa")) {
-		return (struct archive*)afa_open(path, ARCHIVE_MMAP, error);
+		struct archive *ar = (struct archive*)afa_open(path, ARCHIVE_MMAP, error);
+		if (!ar && *error == ARCHIVE_BAD_ARCHIVE_ERROR) {
+			ar = (struct archive*)afa3_open(path, ARCHIVE_MMAP, error);
+		}
+		return ar;
+	} else if (!strcasecmp(ext, "flat")) {
+		return flat_open_file(path, 0, error);
 	}
 
 err:
@@ -149,46 +188,227 @@ static void mkdir_for_file(const char *filename)
 	free(tmp);
 }
 
-static void write_file(struct archive_data *data, const char *output_file)
+static void write_webp(struct cg *cg, FILE *f)
 {
-	FILE *f = NULL;
+	uint8_t *out;
+	size_t len = WebPEncodeLosslessRGBA(cg->pixels, cg->metrics.w, cg->metrics.h, cg->metrics.w*4, &out);
+	if (!fwrite(out, len, 1, f))
+		ERROR("fwrite failed: %s", strerror(errno));
+	WebPFree(out);
+}
 
-	if (!output_file) {
-		char *u = sjis2utf(data->name, strlen(data->name));
-		mkdir_for_file(u);
-		if (!(f = fopen(u, "wb")))
-			ERROR("fopen failed: %s", strerror(errno));
-		free(u);
-	} else if (strcmp(output_file, "-")) {
-		if (!(f = fopen(output_file, "wb")))
-			ERROR("fopen failed: %s", strerror(errno));
+static void write_png(struct cg *cg, FILE *f)
+{
+	png_structp png_ptr = NULL;
+	png_infop info_ptr = NULL;
+	png_byte **row_pointers = NULL;
+
+	png_ptr = png_create_write_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL);
+	if (!png_ptr) {
+		WARNING("png_create_write_struct failed");
+		goto cleanup;
 	}
 
-	if (fwrite(data->data, data->size, 1, f ? f : stdout) != 1)
+	info_ptr = png_create_info_struct(png_ptr);
+	if (!info_ptr) {
+		WARNING("png_create_info_struct failed");
+		goto cleanup;
+	}
+
+	if (setjmp(png_jmpbuf(png_ptr))) {
+		WARNING("png_init_io failed");
+		goto cleanup;
+	}
+
+	png_init_io(png_ptr, f);
+
+	if (setjmp(png_jmpbuf(png_ptr))) {
+		WARNING("png_write_header failed");
+		goto cleanup;
+	}
+
+	png_set_IHDR(png_ptr, info_ptr, cg->metrics.w, cg->metrics.h,
+		     8, PNG_COLOR_TYPE_RGBA, PNG_INTERLACE_NONE,
+		     PNG_COMPRESSION_TYPE_BASE, PNG_FILTER_TYPE_BASE);
+	png_write_info(png_ptr, info_ptr);
+
+	png_uint_32 stride = cg->metrics.w * 4;
+	row_pointers = png_malloc(png_ptr, cg->metrics.h * sizeof(png_byte*));
+	for (int i = 0; i < cg->metrics.h; i++) {
+		row_pointers[i] = cg->pixels + i*stride;
+	}
+
+	if (setjmp(png_jmpbuf(png_ptr))) {
+		WARNING("png_write_image failed");
+		goto cleanup;
+	}
+
+	png_write_image(png_ptr, row_pointers);
+
+	if (setjmp(png_jmpbuf(png_ptr))) {
+		WARNING("png_write_end failed");
+		goto cleanup;
+	}
+
+	png_write_end(png_ptr, NULL);
+cleanup:
+	if (row_pointers)
+		png_free(png_ptr, row_pointers);
+	if (png_ptr)
+		png_destroy_write_struct(&png_ptr, info_ptr ? &info_ptr : NULL);
+	return;
+}
+
+enum imgenc {
+	IMGENC_PNG,
+	IMGENC_WEBP,
+};
+
+struct image_encoder {
+	void (*write)(struct cg *cg, FILE *f);
+	const char * const ext;
+};
+
+static struct image_encoder image_encoders[] = {
+	[IMGENC_PNG]  = { write_png,  ".png"  },
+	[IMGENC_WEBP] = { write_webp, ".webp" },
+};
+
+static enum imgenc imgenc = IMGENC_PNG;
+
+static bool is_image_file(struct archive_data *data)
+{
+	return cg_check_format(data->data) != ALCG_UNKNOWN;
+}
+
+static char *get_default_filename(struct archive_data *data, const char *ext)
+{
+	char *u = sjis2utf(data->name, 0);
+	if (ext) {
+		size_t ulen = strlen(u);
+		size_t extlen = strlen(ext);
+		u = xrealloc(u, ulen + extlen + 1);
+		memcpy(u+ulen, ext, extlen + 1);
+	}
+	return u;
+}
+
+static FILE *checked_fopen(const char *path, const char *mode)
+{
+	if (file_exists(path) && !force) {
+		errno = EEXIST;
+		return NULL;
+	}
+	FILE *f = fopen(path, mode);
+	if (!f)
+		ERROR("fopen failed: %s", strerror(errno));
+	return f;
+}
+
+static bool write_file(struct archive_data *data, const char *output_file)
+{
+	FILE *f = NULL;
+	bool output_img = !raw && is_image_file(data);
+
+	if (!output_file) {
+		char *u = get_default_filename(data, output_img ? image_encoders[imgenc].ext : NULL);
+		mkdir_for_file(u);
+		if (!(f = checked_fopen(u, "wb"))) {
+			free(u);
+			return false;
+		}
+		free(u);
+	} else if (strcmp(output_file, "-")) {
+		if (!(f = checked_fopen(output_file, "wb"))) {
+			return false;
+		}
+	}
+
+	if (output_img) {
+		struct cg *cg = cg_load_data(data);
+		if (cg) {
+			image_encoders[imgenc].write(cg, f);
+			cg_free(cg);
+		} else {
+			WARNING("Failed to load CG");
+		}
+	} else if (!images_only && fwrite(data->data, data->size, 1, f ? f : stdout) != 1) {
 		ERROR("fwrite failed: %s", strerror(errno));
+	}
 
 	if (f)
 		fclose(f);
+	return true;
 }
 
-static void extract_all_iter(struct archive_data *data, void *_output_dir)
-{
-	char *output_dir = _output_dir ? _output_dir : ".";
-	char *file_name = sjis2utf(data->name, 0);
-	char output_file[PATH_MAX];
+static void extract_all_iter(struct archive_data *data, void *_output_dir);
 
-	snprintf(output_file, PATH_MAX, "%s/%s", output_dir, file_name);
+static void extract_flat(struct archive_data *data, char *output_dir)
+{
+	int error;
+	struct archive *ar = flat_open(data->data, data->size, &error);
+	if (!ar) {
+		WARNING("Error opening FLAT archive: %s", archive_strerror(error));
+		return;
+	}
+
+	// generate filename prefix
+	char *uname = sjis2utf(data->name, 0);
+	size_t dir_len = strlen(output_dir);
+	size_t name_len = strlen(uname);
+	char *prefix = xmalloc(dir_len + name_len + 2);
+	strcpy(prefix, output_dir);
+	strcpy(prefix+dir_len, uname);
+	strcpy(prefix+dir_len+name_len, ".");
+	//NOTICE("Extracting %s...", uname);
+
+	archive_for_each(ar, extract_all_iter, prefix);
+	archive_free(ar);
+	free(prefix);
+	free(uname);
+}
+
+static bool is_flat_file(const char *data)
+{
+	if (!strncmp(data, "ELNA", 4))
+		data += 8;
+	return !strncmp(data, "FLAT", 4);
+}
+
+static void extract_all_iter(struct archive_data *data, void *_prefix)
+{
+	if (!archive_load_file(data)) {
+		char *u = sjis2utf(data->name, 0);
+		WARNING("Error loading file: %s", u);
+		free(u);
+		return;
+	}
+
+	if (!raw && is_flat_file((char*)data->data)) {
+		char *u = sjis2utf(data->name, 0);
+		NOTICE("Extracting %s...", u);
+		free(u);
+		extract_flat(data, _prefix);
+		return;
+	}
+
+	char *prefix = _prefix;
+	bool is_image = is_image_file(data);
+	char *file_name = get_default_filename(data, !raw && is_image ? image_encoders[imgenc].ext : NULL);
+	char output_file[PATH_MAX];
+	snprintf(output_file, PATH_MAX, "%s%s", prefix, file_name);
 	free(file_name);
+	if (!is_image && images_only) {
+		NOTICE("Skipping non-image file: %s", output_file);
+		return;
+	}
 
 	mkdir_for_file(output_file);
 
-	printf("%s\n", output_file);
-
-	if (!archive_load_file(data)) {
-		WARNING("Error loading file: %s", output_file);
-		return;
-	}
-	write_file(data, output_file);
+	if (write_file(data, output_file))
+		NOTICE("%s", output_file);
+	else
+		NOTICE("Skipping existing file: %s", output_file);
 }
 
 static void list_all_iter(struct archive_data *data, possibly_unused void *_)
@@ -201,7 +421,7 @@ static void list_all_iter(struct archive_data *data, possibly_unused void *_)
 int main(int argc, char *argv[])
 {
 	char *output_file = NULL;
-	char *input_file = NULL;
+	possibly_unused char *input_file = NULL;
 	char *file_name = NULL;
 	int file_index = -1;
 
@@ -210,23 +430,25 @@ int main(int argc, char *argv[])
 	bool add = false;
 	bool delete = false;
 	bool list = false;
-	//bool force = false;
 
 	while (1) {
 		static struct option long_options[] = {
-			{ "help",    no_argument,       0, LOPT_HELP },
-			{ "extract", no_argument,       0, LOPT_EXTRACT },
-			{ "create",  required_argument, 0, LOPT_CREATE },
-			{ "add",     required_argument, 0, LOPT_ADD },
-			{ "delete",  no_argument,       0, LOPT_DELETE },
-			{ "list",    no_argument,       0, LOPT_LIST },
-			{ "output",  required_argument, 0, LOPT_OUTPUT },
-			{ "index",   required_argument, 0, LOPT_INDEX },
-			{ "name",    required_argument, 0, LOPT_NAME },
-			{ "force",   no_argument,       0, LOPT_FORCE },
+			{ "help",         no_argument,       0, LOPT_HELP },
+			{ "extract",      no_argument,       0, LOPT_EXTRACT },
+			{ "create",       required_argument, 0, LOPT_CREATE },
+			{ "add",          required_argument, 0, LOPT_ADD },
+			{ "delete",       no_argument,       0, LOPT_DELETE },
+			{ "list",         no_argument,       0, LOPT_LIST },
+			{ "output",       required_argument, 0, LOPT_OUTPUT },
+			{ "index",        required_argument, 0, LOPT_INDEX },
+			{ "name",         required_argument, 0, LOPT_NAME },
+			{ "force",        no_argument,       0, LOPT_FORCE },
+			{ "image-format", required_argument, 0, LOPT_IMAGE_FORMAT },
+			{ "images-only",  no_argument,       0, LOPT_IMAGES_ONLY },
+			{ "raw",          no_argument,       0, LOPT_RAW },
 		};
 		int option_index = 0;
-		int c = getopt_long(argc, argv, "hxc:a:dlo:i:n:", long_options, &option_index);
+		int c = getopt_long(argc, argv, "hxc:a:dlo:i:n:f", long_options, &option_index);
 		if (c == -1)
 			break;
 
@@ -245,6 +467,7 @@ int main(int argc, char *argv[])
 			break;
 		case 'a':
 		case LOPT_ADD:
+			add = true;
 			input_file = optarg;
 			break;
 		case 'd':
@@ -267,6 +490,24 @@ int main(int argc, char *argv[])
 		case 'n':
 			file_name = optarg;
 			break;
+		case 'f':
+		case LOPT_FORCE:
+			force = true;
+			break;
+		case LOPT_IMAGE_FORMAT:
+			if (!strcasecmp(optarg, "png"))
+				imgenc = IMGENC_PNG;
+			else if (!strcasecmp(optarg, "webp"))
+				imgenc = IMGENC_WEBP;
+			else
+				ERROR("Unrecognized image format: \"%s\"", optarg);
+			break;
+		case LOPT_IMAGES_ONLY:
+			images_only = true;
+			break;
+		case LOPT_RAW:
+			raw = true;
+			break;
 		case '?':
 			ERROR("Unrecognized command line argument");
 		}
@@ -303,12 +544,19 @@ int main(int argc, char *argv[])
 			if (!d)
 				ERROR("No file with index %d", file_index);
 			write_file(d, output_file);
+			archive_free_data(d);
 		} else if (file_name) {
 			char *u = utf2sjis(file_name, strlen(file_name));
 			struct archive_data *d = archive_get_by_name(ar, u);
+			if (!d)
+				ERROR("No file with name \"%s\"", u);
 			write_file(d, output_file);
+			archive_free_data(d);
+			free(u);
 		} else {
+			output_file = output_file_dir(output_file);
 			archive_for_each(ar, extract_all_iter, output_file);
+			free(output_file);
 		}
 	} else if (add) {
 		WARNING("adding files to archive not yet implemented");

src/tools/ar/meson.build

@@ -1,4 +1,5 @@
 alicear = ['alice-ar.c',
+           'afa_extract.c',
 ]
 
 executable('alice-ar', alicear,