alice-ar: support extracting ALD and AFA files

Support whole archive, by-index and by-name extraction of files from ALD
and AFA archives.

include/system4/afa.h

@@ -48,6 +48,6 @@ struct afa_archive {
 	uint8_t *data;
 };
 
-struct afa_archive *afa_open(char *file, int flags, int *error);
+struct afa_archive *afa_open(const char *file, int flags, int *error);
 
 #endif /* SYSTEM4_AFA_H */

include/system4/ald.h

@@ -24,6 +24,7 @@
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
+#include <stdio.h>
 #include "system4/archive.h"
 
 #define ALD_FILEMAX 255
@@ -36,6 +37,7 @@ struct ald_archive {
 		uint8_t *data;
 		size_t size;
 		char *name;
+		FILE *fp;
 	} files[ALD_FILEMAX];
 	// upper limit on how many files could be referenced by this archive
 	int maxfile;
@@ -47,6 +49,13 @@ struct ald_archive {
 	int *fileptr[ALD_FILEMAX];
 };
 
+struct ald_archive_data {
+	struct archive_data data;
+	int disk;
+	int dataptr;
+	int hdr_size;
+};
+
 enum {
 	ALDFILE_BGM,
 	ALDFILE_CG,

include/system4/archive.h

@@ -39,7 +39,11 @@ struct archive {
 
 struct archive_ops {
 	bool (*exists)(struct archive *ar, int no);
+	bool (*exists_by_name)(struct archive *ar, const char *name);
 	struct archive_data *(*get)(struct archive *ar, int no);
+	struct archive_data *(*get_by_name)(struct archive *ar, const char *name);
+	bool (*load_file)(struct archive_data *data);
+	void (*for_each)(struct archive *ar, void (*iter)(struct archive_data *data, void *user), void *user);
 	void (*free_data)(struct archive_data *data);
 	void (*free)(struct archive *ar);
 };
@@ -47,8 +51,9 @@ struct archive_ops {
 struct archive_data {
 	size_t size;
 	uint8_t *data;
-	uint8_t *data_raw;
 	char *name;
+	int no;
+	void *private;
 	struct archive *archive;
 };
 
@@ -62,15 +67,43 @@ const char *archive_strerror(int error);
  */
 static inline bool archive_exists(struct archive *ar, int no)
 {
-	return ar->ops->exists(ar, no);
+	return ar->ops->exists ? ar->ops->exists(ar, no) : false;
+}
+
+static inline bool archive_exists_by_name(struct archive *ar, const char *name)
+{
+	return ar->ops->exists_by_name ? ar->ops->exists_by_name(ar, name) : false;
 }
 
 /*
- * Retrieve a piece of data from an ALD archive.
+ * Retrieve a file from an archive by ID.
  */
 static inline struct archive_data *archive_get(struct archive *ar, int no)
 {
-	return ar->ops->get(ar, no);
+	return ar->ops->get ? ar->ops->get(ar, no) : NULL;
+}
+
+/*
+ * Retrieve a file from an archive by name.
+ */
+static inline struct archive_data *archive_get_by_name(struct archive *ar, const char *name)
+{
+	return ar->ops->get_by_name ? ar->ops->get_by_name(ar, name) : NULL;
+}
+
+/*
+ * Load a file into memory, given an uninitialized descriptor.
+ * This should be used in conjunction with archive_for_each.
+ */
+static inline bool archive_load_file(struct archive_data *data)
+{
+	return data->archive->ops->load_file ? data->archive->ops->load_file(data) : false;
+}
+
+static inline void archive_for_each(struct archive *ar, void (*iter)(struct archive_data *data, void *user), void *user)
+{
+	if (ar->ops->for_each)
+		ar->ops->for_each(ar, iter, user);
 }
 
 /*

src/afa.c

@@ -18,6 +18,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <errno.h>
 #include <unistd.h>
 #include <fcntl.h>
 #include <zlib.h>
@@ -30,12 +31,18 @@
 
 static bool afa_exists(struct archive *ar, int no);
 static struct archive_data *afa_get(struct archive *ar, int no);
+static struct archive_data *afa_get_by_name(struct archive *ar, const char *name);
+static bool afa_load_file(struct archive_data *data);
+static void afa_for_each(struct archive *ar, void (*iter)(struct archive_data *data, void *user), void *user);
 static void afa_free_data(struct archive_data *data);
 static void afa_free(struct archive *ar);
 
 struct archive_ops afa_archive_ops = {
 	.exists = afa_exists,
 	.get = afa_get,
+	.get_by_name = afa_get_by_name,
+	.load_file = afa_load_file,
+	.for_each = afa_for_each,
 	.free_data = afa_free_data,
 	.free = afa_free,
 };
@@ -46,18 +53,94 @@ static bool afa_exists(struct archive *_ar, int no)
 	return (uint32_t)no < ar->nr_files;
 }
 
-static struct archive_data *afa_get(struct archive *_ar, int no)
+static bool afa_load_file(struct archive_data *data)
+{
+	if (data->data)
+		return true;
+
+	struct afa_archive *ar = (struct afa_archive*)data->archive;
+	struct afa_entry *e = &ar->files[data->no];
+
+	if (ar->ar.mmapped) {
+		data->data = (uint8_t*)ar->mmap_ptr + ar->data_start + e->off;
+		return true;
+	}
+
+	FILE *f = fopen(ar->filename, "rb");
+	if (!f) {
+		WARNING("Failed to open '%s': %s", ar->filename, strerror(errno));
+		free(data);
+		return false;
+	}
+
+	fseek(f, ar->data_start + e->off, SEEK_SET);
+
+	data->data = xmalloc(e->size);
+	if (fread(data->data, e->size, 1, f) != e->size) {
+		WARNING("Failed to read '%s': %s", ar->filename, strerror(errno));
+		free(data->data);
+		free(data);
+		return false;
+	}
+
+	return true;
+}
+
+static struct archive_data *afa_get_descriptor(struct archive *_ar, int no)
 {
 	struct afa_archive *ar = (struct afa_archive*)_ar;
 	if ((uint32_t)no >= ar->nr_files)
 		return NULL;
-	// ...
+	struct afa_entry *e = &ar->files[no];
+	struct archive_data *data = xcalloc(1, sizeof(struct archive_data));
+	data->size = e->size;
+	data->name = e->name->text;
+	data->no = no;
+	data->archive = _ar;
+	return data;
+}
+
+static struct archive_data *afa_get(struct archive *_ar, int no)
+{
+	struct archive_data *data = afa_get_descriptor(_ar, no);
+	if (!data)
+		return NULL;
+	if (!afa_load_file(data)) {
+		afa_free_data(data);
+		return NULL;
+	}
+	return data;
+}
+
+static struct archive_data *afa_get_by_name(struct archive *_ar, const char *name)
+{
+	// TODO: index filenames
+	struct afa_archive *ar = (struct afa_archive*)_ar;
+	for (uint32_t i = 0; i < ar->nr_files; i++) {
+		if (!strcmp(name, ar->files[i].name->text)) {
+			return afa_get(_ar, i);
+		}
+	}
 	return NULL;
 }
 
+static void afa_for_each(struct archive *_ar, void (*iter)(struct archive_data *data, void *user), void *user)
+{
+	struct afa_archive *ar = (struct afa_archive*)_ar;
+	for (uint32_t i = 0; i < ar->nr_files; i++) {
+		struct archive_data *data = afa_get_descriptor(_ar, i);
+		if (!data)
+			continue;
+		iter(data, user);
+		afa_free_data(data);
+	}
+}
+
 static void afa_free_data(struct archive_data *data)
 {
-	
+	if (data->data && !data->archive->mmapped)
+		free(data->data);
+	free(data);
 }
 
 static void afa_free(struct archive *_ar)
@@ -175,7 +258,7 @@ exit_err:
 	return false;
 }
 
-struct afa_archive *afa_open(char *file, int flags, int *error)
+struct afa_archive *afa_open(const char *file, int flags, int *error)
 {
 #ifdef _WIN32
 	flags &= ~ARCHIVE_MMAP;
@@ -184,18 +267,22 @@ struct afa_archive *afa_open(char *file, int flags, int *error)
 	struct afa_archive *ar = xcalloc(1, sizeof(struct afa_archive));
 
 	if (!(fp = fopen(file, "rb"))) {
+		WARNING("fopen failed: %s", strerror(errno));
 		*error = ARCHIVE_FILE_ERROR;
 		goto exit_err;
 	}
 	if (!afa_read_header(fp, ar, error)) {
+		WARNING("afa_read_header failed");
 		fclose(fp);
 		goto exit_err;
 	}
 	if (!afa_read_file_table(fp, ar, error)) {
+		WARNING("afa_read_file_table failed");
 		fclose(fp);
 		goto exit_err;
 	}
 	if (fclose(fp)) {
+		WARNING("fclose failed: %s", strerror(errno));
 		*error = ARCHIVE_FILE_ERROR;
 		goto exit_err;
 	}
@@ -203,12 +290,14 @@ struct afa_archive *afa_open(char *file, int flags, int *error)
 	if (flags & ARCHIVE_MMAP) {
 		int fd = open(file, O_RDONLY);
 		if (fd < 0) {
+			WARNING("open failed: %s", strerror(errno));
 			*error = ARCHIVE_FILE_ERROR;
 			goto exit_err;
 		}
 		ar->mmap_ptr = mmap(0, ar->file_size, PROT_READ, MAP_SHARED, fd, 0);
 		close(fd);
 		if (ar->mmap_ptr == MAP_FAILED) {
+			WARNING("mmap failed: %s", strerror(errno));
 			*error = ARCHIVE_FILE_ERROR;
 			goto exit_err;
 		}

src/ald.c

@@ -30,12 +30,18 @@
 
 static bool ald_exists(struct archive *ar, int no);
 static struct archive_data *ald_get(struct archive *ar, int no);
+static struct archive_data *ald_get_by_name(struct archive *_ar, const char *name);
+static bool ald_load_file(struct archive_data *data);
+static void ald_for_each(struct archive *_ar, void (*iter)(struct archive_data *data, void *user), void *user);
 static void ald_free_data(struct archive_data *data);
 static void ald_free(struct archive *ar);
 
 struct archive_ops ald_archive_ops = {
 	.exists = ald_exists,
 	.get = ald_get,
+	.get_by_name = ald_get_by_name,
+	.load_file = ald_load_file,
+	.for_each = ald_for_each,
 	.free_data = ald_free_data,
 	.free = ald_free,
 };
@@ -184,43 +190,108 @@ static bool ald_exists(struct archive *ar, int no)
 	return ar && !!_ald_get((struct ald_archive*)ar, no, &disk, &dataptr);
 }
 
-/* Get a piece of data from an ALD archive. */
-struct archive_data *ald_get(struct archive *_ar, int no)
+/* Get a descriptor for a file in an ALD archive. */
+struct archive_data *ald_get_descriptor(struct archive *_ar, int no)
 {
 	if (!_ar)
 		return NULL;
 
-	uint8_t *data;
-	struct archive_data *dfile;
-	int disk, dataptr, ptr, size;
 	struct ald_archive *ar = (struct ald_archive*)_ar;
-	int readsize = _ald_get(ar, no, &disk, &dataptr);
+	struct ald_archive_data *dfile = calloc(1, sizeof(struct ald_archive_data));
 
-	// get data top
-	if (ar->ar.mmapped) {
-		data = ar->files[disk].data + dataptr;
-	} else {
-		//int readsize = dataptr2 - dataptr;
-		FILE *fp;
-		// FIXME: check return values
-		data = malloc(sizeof(char) * readsize);
-		fp = fopen(ar->files[disk].name, "r");
-		fseek(fp, dataptr, SEEK_SET);
-		fread(data, 1, readsize, fp);
-		fclose(fp);
+	if (!_ald_get(ar, no, &dfile->disk, &dfile->dataptr)) {
+		free(dfile);
+		return NULL;
 	}
 
-	// get real data and size
-	ptr  = LittleEndian_getDW(data, 0);
-	size = LittleEndian_getDW(data, 4);
+	// read header
+	if (ar->ar.mmapped) {
+		uint8_t *hdr  = ar->files[dfile->disk].data + dfile->dataptr;
+		dfile->hdr_size = LittleEndian_getDW(hdr, 0);
+		dfile->data.size = LittleEndian_getDW(hdr, 4);
+		dfile->data.name = strdup((char*)hdr + 16);
+	} else {
+		uint8_t *hdr = xmalloc(8);
+		FILE *fp = ar->files[dfile->disk].fp;
 
-	dfile = calloc(1, sizeof(struct archive_data));
-	dfile->data_raw = data;
-	dfile->data = data + ptr;
-	dfile->size = size;
-	dfile->name = strdup((char*)data + 16);
-	dfile->archive = &ar->ar;
-	return dfile;
+		// read header size, file size
+		fseek(fp, dfile->dataptr, SEEK_SET);
+		fread(hdr, 8, 1, fp);
+		dfile->hdr_size = LittleEndian_getDW(hdr, 0);
+		dfile->data.size = LittleEndian_getDW(hdr, 4);
+
+		// read name
+		dfile->data.name = xcalloc(dfile->hdr_size-8, 1);
+		fread(dfile->data.name, dfile->hdr_size-8, 1, fp);
+
+		free(hdr);
+	}
+
+	dfile->data.no = no;
+	dfile->data.archive = &ar->ar;
+	return &dfile->data;
+}
+
+/* Get a file from an ALD archive. */
+struct archive_data *ald_get(struct archive *ar, int no)
+{
+	if (!ar)
+		return NULL;
+
+	struct archive_data *data = ald_get_descriptor(ar, no);
+	if (!data)
+		return NULL;
+
+	if (!ald_load_file(data)) {
+		ald_free_data(data);
+		return NULL;
+	}
+
+	return data;
+}
+
+static struct archive_data *ald_get_by_name(struct archive *_ar, const char *name)
+{
+	struct ald_archive *ar = (struct ald_archive*)_ar;
+	for (int i = 0; i < ar->maxfile; i++) {
+		struct archive_data *data = ald_get_descriptor(_ar, i);
+		if (!data)
+			continue;
+		if (!strcmp(data->name, name)) {
+			ald_load_file(data);
+			return data;
+		}
+		ald_free_data(data);
+	}
+	return NULL;
+}
+
+static bool ald_load_file(struct archive_data *data)
+{
+	struct ald_archive *ar = (struct ald_archive*)data->archive;
+	struct ald_archive_data *dfile = (struct ald_archive_data*)data;
+	if (data->archive->mmapped) {
+		data->data = ar->files[dfile->disk].data + dfile->dataptr + dfile->hdr_size;
+	} else {
+		FILE *fp = ar->files[dfile->disk].fp;
+		data->data = xmalloc(data->size);
+
+		fseek(fp, dfile->dataptr + dfile->hdr_size, SEEK_SET);
+		fread(data->data, data->size, 1, fp);
+	}
+	return data;
+}
+
+static void ald_for_each(struct archive *_ar, void (*iter)(struct archive_data *data, void *user), void *user)
+{
+	struct ald_archive *ar = (struct ald_archive*)_ar;
+	for (int i = 0; i < ar->maxfile; i++) {
+		struct archive_data *data = ald_get_descriptor(_ar, i);
+		if (!data)
+			continue;
+		iter(data, user);
+		ald_free_data(data);
+	}
 }
 
 /* Free an ald_data strcture returned by `ald_get`. */
@@ -228,7 +299,7 @@ static void ald_free_data(struct archive_data *data)
 {
 	if (!data)
 		return;
-	if (!data->archive->mmapped)
+	if (!data->archive->mmapped && data->data)
 		free(data->data);
 	free(data->name);
 	free(data);
@@ -242,13 +313,16 @@ static void ald_free(struct archive *_ar)
 
 	struct ald_archive *ar = (struct ald_archive*)_ar;
 
-	// unmap mmap files
-	if (ar->ar.mmapped) {
-		for (int i = 0; i < ALD_FILEMAX; i++) {
-			if (ar->files[i].data)
-				munmap(ar->files[i].data, ar->files[i].size);
+	// unmap mmap files/close file descriptors
+	for (int i = 0; i < ALD_FILEMAX; i++) {
+		if (ar->ar.mmapped && ar->files[i].data) {
+			munmap(ar->files[i].data, ar->files[i].size);
+		} else if (ar->files[i].fp) {
+			fclose(ar->files[i].fp);
 		}
+		free(ar->files[i].name);
 	}
+
 	free(ar);
 }
 
@@ -265,9 +339,9 @@ struct archive *ald_open(char **files, int count, int flags, int *error)
 #endif
 
 	for (int i = 0; i < count; i++) {
-		// XXX: why allow this?
 		if (!files[i])
 			continue;
+		printf("OPENING FILE %s\n", files[i]);
 		if (!(fp = fopen(files[i], "r"))) {
 			*error = ARCHIVE_FILE_ERROR;
 			goto exit_err;
@@ -304,6 +378,12 @@ struct archive *ald_open(char **files, int count, int flags, int *error)
 				goto exit_err;
 			}
 			ar->files[i].size = filesize;
+		} else {
+			// get a file descriptor for each file
+			if (!(ar->files[i].fp = fopen(files[i], "rb"))) {
+				*error = ARCHIVE_FILE_ERROR;
+				goto exit_err;
+			}
 		}
 	}
 	int c = 0;

src/tools/ar/alice-ar.c

@@ -17,9 +17,17 @@
 #include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <ctype.h>
+#include <dirent.h>
 #include <getopt.h>
+#include <limits.h>
+#include <libgen.h>
 #include "system4.h"
 #include "system4/afa.h"
+#include "system4/ald.h"
+#include "file.h"
 #include "system4/string.h"
 #include "system4/utfsjis.h"
 
@@ -52,6 +60,144 @@ enum {
 	LOPT_FORCE,
 };
 
+// dirname is allowed to return a pointer to static memory OR modify its input.
+// This works around the braindamage by ALWAYS returning a pointer to static
+// memory, at the cost of a string copy.
+char *xdirname(const char *path)
+{
+	static char buf[PATH_MAX];
+	strncpy(buf, path, PATH_MAX-1);
+	return dirname(buf);
+}
+
+char *xbasename(const char *path)
+{
+	static char buf[PATH_MAX];
+	strncpy(buf, path, PATH_MAX-1);
+	return basename(buf);
+}
+
+static struct archive *open_ald_archive(const char *path, int *error)
+{
+	int count = 0;
+	char *dir_name = xdirname(path);
+	char *base_name = xbasename(path);
+	char *ald_filenames[ALD_FILEMAX];
+	int prefix_len = strlen(base_name) - 5;
+	if (prefix_len <= 0)
+		return NULL;
+
+	memset(ald_filenames, 0, sizeof(char*) * ALD_FILEMAX);
+
+	DIR *dir;
+	struct dirent *d;
+	char filepath[PATH_MAX];
+
+	if (!(dir = opendir(dir_name))) {
+		ERROR("Failed to open directory: %s", path);
+	}
+
+	while ((d = readdir(dir))) {
+		printf("checking %s\n", d->d_name);
+		int len = strlen(d->d_name);
+		if (len < prefix_len + 5 || strcasecmp(d->d_name+len-4, ".ald"))
+			continue;
+		if (strncasecmp(d->d_name, base_name, prefix_len))
+			continue;
+
+		int dno = toupper(*(d->d_name+len-5)) - 'A';
+		if (dno < 0 || dno >= ALD_FILEMAX)
+			continue;
+
+		snprintf(filepath, PATH_MAX-1, "%s/%s", dir_name, d->d_name);
+		ald_filenames[dno] = strdup(filepath);
+		count = max(count, dno+1);
+	}
+
+	struct archive *ar = ald_open(ald_filenames, count, ARCHIVE_MMAP, error);
+
+	for (int i = 0; i < ALD_FILEMAX; i++) {
+		free(ald_filenames[i]);
+	}
+
+	return ar;
+}
+
+static struct archive *open_archive(const char *path, int *error)
+{
+	size_t len = strlen(path);
+	if (len < 4)
+		goto err;
+
+	const char *ext = path + len - 4;
+	if (!strcasecmp(ext, ".ald")) {
+		return open_ald_archive(path, error);
+	} else if (!strcasecmp(ext, ".afa")) {
+		return (struct archive*)afa_open(path, ARCHIVE_MMAP, error);
+	}
+
+err:
+	usage();
+	ERROR("Couldn't determine archive type for '%s'", path);
+}
+
+static void mkdir_for_file(const char *filename)
+{
+	char *tmp = strdup(filename);
+	char *dir = dirname(tmp);
+	mkdir_p(dir);
+	free(tmp);
+}
+
+static void write_file(struct archive_data *data, const char *output_file)
+{
+	FILE *f = NULL;
+
+	if (!output_file) {
+		char *u = sjis2utf(data->name, strlen(data->name));
+		mkdir_for_file(u);
+		if (!(f = fopen(u, "wb")))
+			ERROR("fopen failed: %s", strerror(errno));
+		free(u);
+	} else if (strcmp(output_file, "-")) {
+		if (!(f = fopen(output_file, "wb")))
+			ERROR("fopen failed: %s", strerror(errno));
+	}
+
+	if (fwrite(data->data, data->size, 1, f ? f : stdout) != 1)
+		ERROR("fwrite failed: %s", strerror(errno));
+
+	if (f)
+		fclose(f);
+}
+
+static void extract_all_iter(struct archive_data *data, void *_output_dir)
+{
+	char *output_dir = _output_dir ? _output_dir : ".";
+	char *file_name = sjis2utf(data->name, 0);
+	char output_file[PATH_MAX];
+
+	snprintf(output_file, PATH_MAX, "%s/%s", output_dir, file_name);
+	free(file_name);
+
+	mkdir_for_file(output_file);
+
+	printf("%s\n", output_file);
+
+	if (!archive_load_file(data)) {
+		WARNING("Error loading file: %s", output_file);
+		return;
+	}
+	write_file(data, output_file);
+}
+
+static void list_all_iter(struct archive_data *data, possibly_unused void *_)
+{
+	char *name = sjis2utf(data->name, 0);
+	printf("%d: %s\n", data->no, name);
+	free(name);
+}
+
 int main(int argc, char *argv[])
 {
 	char *output_file = NULL;
@@ -64,7 +210,7 @@ int main(int argc, char *argv[])
 	bool add = false;
 	bool delete = false;
 	bool list = false;
-	bool force = false;
+	//bool force = false;
 
 	while (1) {
 		static struct option long_options[] = {
@@ -146,25 +292,32 @@ int main(int argc, char *argv[])
 	}
 
 	int error;
-	struct afa_archive *ar = afa_open(argv[0], ARCHIVE_MMAP, &error);
+	struct archive *ar = open_archive(argv[0], &error);
 	if (!ar) {
 		ERROR("Opening archive: %s", archive_strerror(error));
 	}
 
 	if (extract) {
-		WARNING("archive extraction not yet implemented");
+		if (file_index >= 0) {
+			struct archive_data *d = archive_get(ar, file_index);
+			if (!d)
+				ERROR("No file with index %d", file_index);
+			write_file(d, output_file);
+		} else if (file_name) {
+			char *u = utf2sjis(file_name, strlen(file_name));
+			struct archive_data *d = archive_get_by_name(ar, u);
+			write_file(d, output_file);
+		} else {
+			archive_for_each(ar, extract_all_iter, output_file);
+		}
 	} else if (add) {
 		WARNING("adding files to archive not yet implemented");
 	} else if (delete) {
 		WARNING("deleting files from archive not yet implemented");
 	} else if (list) {
-		for (uint32_t i = 0; i < ar->nr_files; i++) {
-			char *u = sjis2utf(ar->files[i].name->text, ar->files[i].name->size);
-			printf("%u: %s\n", i, u);
-			free(u);
-		}
+		archive_for_each(ar, list_all_iter, NULL);
 	}
 
-	archive_free(&ar->ar);
+	archive_free(ar);
 	return 0;
 }