Fix encoding issue with macro generation

Fixes #2.

The problem here was that init_member_functions assumed ASCII or SJIS
encoding of function/struct names in the ain file. Now it will convert
to UTF-8 when analyzing member functions.

This commit also removes the --ain-encoding option to ainedit.
--output-encoding should always match the encoding of the input ain
file. --transcode can be used to change the encoding before running
running further ainedit commands.

include/system4/ain.h

@@ -329,6 +329,7 @@ struct ain *ain_open(const char *path, int *error);
 struct ain *ain_new(int version);
 void ain_decrypt(uint8_t *buf, size_t len);
 
+void ain_init_member_functions(struct ain *ain, char *(*to_ascii)(const char*));
 struct ain_function *ain_get_function(struct ain *ain, char *name);
 int ain_get_function_no(struct ain *ain, char *name);
 int ain_get_function_index(struct ain *ain, struct ain_function *f);

src/ain.c

@@ -112,58 +112,58 @@ static bool function_is_member_of(char *func_name, char *struct_name)
 	return !*struct_name && *func_name == '@';
 }
 
-/*
- * Returns true if func_name contains the '@' character.
- * We can't use strchr here because '@' is valid as the second byte of a SJIS character.
- */
-static bool is_member_function(char *func_name)
-{
-	while (*func_name) {
-		if (*func_name == '@')
-			return true;
-		if (SJIS_2BYTE(*func_name)) {
-			func_name++;
-			if (*func_name)
-				func_name++;
-		} else {
-			func_name++;
-		}
-	}
-	return false;
-}
-
 /*
  * Infer struct member functions from function names.
  */
-static void init_member_functions(struct ain *ain)
+void ain_init_member_functions(struct ain *ain, char *(*to_ascii)(const char*))
 {
+	// XXX: we convert all struct names up front to avoid repeated conversions below
+	char **struct_names = xcalloc(ain->nr_structures, sizeof(char*));
+	for (int i = 0; i < ain->nr_structures; i++) {
+		struct_names[i] = to_ascii(ain->structures[i].name);
+	}
+	char **enum_names = xcalloc(ain->nr_enums, sizeof(char*));
+	for (int i = 0; i < ain->nr_enums; i++) {
+		enum_names[i] = to_ascii(ain->enums[i].name);
+	}
+
 	for (int f = 0; f < ain->nr_functions; f++) {
 		ain->functions[f].struct_type = -1;
 		ain->functions[f].enum_type = -1;
-		char *name = ain->functions[f].name;
-		if (!is_member_function(name))
+		char *name = to_ascii(ain->functions[f].name);
+		if (!strchr(name, '@')) {
+			free(name);
 			continue;
+		}
 		for (int s = 0; s < ain->nr_structures; s++) {
-			if (function_is_member_of(name, ain->structures[s].name)) {
+			if (function_is_member_of(name, struct_names[s])) {
 				ain->functions[f].struct_type = s;
 				break;
 			}
 		}
-		if (ain->functions[f].struct_type != -1)
+		if (ain->functions[f].struct_type != -1) {
+			free(name);
 			continue;
+		}
 		// check enums
 		for (int e = 0; e < ain->nr_enums; e++) {
-			if (function_is_member_of(name, ain->enums[e].name)) {
+			if (function_is_member_of(name, enum_names[e])) {
 				ain->functions[f].enum_type = e;
 				break;
 			}
 		}
-		if (ain->functions[f].enum_type == -1) {
-			char *u = sjis2utf(name, 0);
-			WARNING("Failed to find struct type for function \"%s\"", u);
-			free(u);
-		}
+		if (ain->functions[f].enum_type == -1)
+			WARNING("Failed to find struct type for function \"%s\"", name);
+		free(name);
 	}
+	for (int i = 0; i < ain->nr_structures; i++) {
+		free(struct_names[i]);
+	}
+	free(struct_names);
+	for (int i = 0; i < ain->nr_enums; i++) {
+		free(enum_names[i]);
+	}
+	free(enum_names);
 }
 
 static struct func_list *get_function(struct ain *ain, const char *name)
@@ -1175,7 +1175,6 @@ struct ain *ain_open(const char *path, int *error)
 		goto err;
 	}
 	distribute_initvals(ain);
-	init_member_functions(ain);
 
 	free(buf);
 	*error = AIN_SUCCESS;

src/tools/aincmp/aincmp.c

@@ -135,7 +135,10 @@ static bool ain_compare_code(struct ain *_a, struct ain *_b)
 			} else {
 				if (ia != ib) {
 					NOTICE("argument differs at 0x%08x (%d vs %d)", (uint32_t)a.addr, ia, ib);
-					return false;
+					// NOTE: If there's duplicate strings in the string table, string arguments
+					//       can change when rebuilding. This shouldn't matter (?).
+					if (a.instr->opcode != S_PUSH || strcmp(_a->strings[ia]->text, _b->strings[ib]->text))
+						return false;
 				}
 			}
 		}
@@ -192,8 +195,11 @@ static bool ain_compare_functions(struct ain *a, struct ain *b)
 			return false;
 		}
 		if (fa->address != fb->address) {
-			NOTICE("function address differs for %s (0x%08x vs 0x%08x)", fa->name, fa->address, fb->address);
-			return false;
+			// NOTE: address of NULL function doesn't matter
+			if (strcmp(fa->name, "NULL")) {
+				NOTICE("function address differs for %s (0x%08x vs 0x%08x)", fa->name, fa->address, fb->address);
+				return false;
+			}
 		}
 		if (fa->is_label != fb->is_label) {
 			NOTICE("function is_label differs for %s (%d vs %d)", fa->name, fa->is_label, fb->is_label);

src/tools/aindump/aindump.c

@@ -703,6 +703,7 @@ int main(int argc, char *argv[])
 		ERROR("Failed to open ain file: %s\n", ain_strerror(err));
 		return 1;
 	}
+	ain_init_member_functions(ain, encode_text_utf8);
 
 	for (int i = 0; i < dump_ptr; i++) {
 		switch (dump_targets[i]) {

src/tools/ainedit/ainedit.c

@@ -61,14 +61,13 @@ enum {
 	LOPT_RAW,
 	LOPT_INPUT_ENCODING,
 	LOPT_OUTPUT_ENCODING,
-	LOPT_AIN_ENCODING,
 	LOPT_AIN_VERSION,
 	LOPT_SILENT,
 };
 
 iconv_t ain_conv;
 iconv_t print_conv;
-iconv_t ain_input_conv;
+iconv_t ain_utf8_conv;
 
 char *convert_text(iconv_t cd, const char *str);
 
@@ -78,16 +77,16 @@ char *encode_text(const char *str)
 	return convert_text(ain_conv, str);
 }
 
-char *encode_text_to_input_format(char *str)
-{
-	return convert_text(ain_input_conv, str);
-}
-
-char *encode_text_for_print(char *str)
+char *encode_text_for_print(const char *str)
 {
 	return convert_text(print_conv, str);
 }
 
+char *encode_ain_to_utf8(const char *str)
+{
+	return convert_text(ain_utf8_conv, str);
+}
+
 int main(int argc, char *argv[])
 {
 	initialize_instructions();
@@ -102,7 +101,6 @@ int main(int argc, char *argv[])
 	const char *output_file = NULL;
 	const char *input_encoding = "UTF-8";
 	const char *output_encoding = "CP932";
-	const char *ain_encoding = "CP932";
 	int ain_version = 4;
 	bool transcode = false;
 	uint32_t flags = 0;
@@ -119,7 +117,6 @@ int main(int argc, char *argv[])
 			{ "raw",             no_argument,       0, LOPT_RAW },
 			{ "input-encoding",  required_argument, 0, LOPT_INPUT_ENCODING },
 			{ "output-encoding", required_argument, 0, LOPT_OUTPUT_ENCODING },
-			{ "ain-encoding",    required_argument, 0, LOPT_AIN_ENCODING },
 			{ "ain-version",     required_argument, 0, LOPT_AIN_VERSION },
 			{ "silent",          no_argument,       0, LOPT_SILENT },
 		};
@@ -173,9 +170,6 @@ int main(int argc, char *argv[])
 		case LOPT_OUTPUT_ENCODING:
 			output_encoding = optarg;
 			break;
-		case LOPT_AIN_ENCODING:
-			ain_encoding = optarg;
-			break;
 		case LOPT_AIN_VERSION:
 			ain_version = atoi(optarg);
 			if (ain_version < 4 || ain_version > 12)
@@ -203,7 +197,7 @@ int main(int argc, char *argv[])
 	if ((print_conv = iconv_open("UTF-8", input_encoding)) == (iconv_t)-1) {
 		ERROR("iconv_open: %s", strerror(errno));
 	}
-	if ((ain_input_conv = iconv_open(ain_encoding, input_encoding)) == (iconv_t)-1) {
+	if ((ain_utf8_conv = iconv_open("UTF-8", output_encoding)) == (iconv_t)-1) {
 		ERROR("iconv_open: %s", strerror(errno));
 	}
 
@@ -223,6 +217,7 @@ int main(int argc, char *argv[])
 			ERROR("Failed to open ain file: %s", ain_strerror(err));
 		}
 	}
+	ain_init_member_functions(ain, encode_ain_to_utf8);
 
 	if (transcode) {
 		ain_transcode(ain);

src/tools/ainedit/ainedit.h

@@ -87,7 +87,6 @@ void read_text(const char *filename, struct ain *ain);
 void ain_transcode(struct ain *ain);
 
 char *encode_text(const char *str);
-char *encode_text_to_input_format(char *str);
-char *encode_text_for_print(char *str);
+char *encode_text_for_print(const char *str);
 
 #endif /* AINEDIT_AINEDIT_H */

src/tools/ainedit/asm.c

@@ -441,7 +441,7 @@ static void decompose_switch_index(struct asm_state *state, char *in, int *switc
 static int find_member(struct ain_struct *s, char *_member_name)
 {
 	int member_no = -1;
-	char *member_name = encode_text_to_input_format(_member_name);
+	char *member_name = encode_text(_member_name);
 
 	for (int i = 0; i < s->nr_members; i++) {
 		if (!strcmp(member_name, s->members[i].name)) {

src/tools/ainedit/jaf_compile.c

@@ -67,7 +67,7 @@ static void end_loop(struct compiler_state *state)
 
 static int get_string_no(struct compiler_state *state, struct string *s)
 {
-	char *u = encode_text_to_input_format(s->text);
+	char *u = encode_text(s->text);
 	int i = ain_add_string(state->ain, u);
 	free(u);
 	return i;

src/tools/ainedit/jaf_parser.y

@@ -91,7 +91,7 @@ struct jaf_block *jaf_parse(struct ain *ain, const char **files, unsigned nr_fil
 
 int sym_type(char *name)
 {
-    char *u = encode_text_to_input_format(name);
+    char *u = encode_text(name);
     if (ain_get_struct(jaf_ain_out, u)) {
 	free(u);
 	return TYPEDEF_NAME;

src/tools/ainedit/jaf_static_analysis.c

@@ -33,7 +33,7 @@ void jaf_define_struct(struct ain *ain, struct jaf_block_item *def)
 	if (!def->struc.name)
 		ERROR("Anonymous structs not supported");
 
-	char *u = encode_text_to_input_format(def->struc.name->text);
+	char *u = encode_text(def->struc.name->text);
 	if (ain_get_struct(ain, u))
 		ERROR("Redefining structs not supported");
 	def->struc.struct_no = ain_add_struct(ain, u);
@@ -43,7 +43,7 @@ void jaf_define_struct(struct ain *ain, struct jaf_block_item *def)
 void jaf_define_functype(struct ain *ain, struct jaf_fundecl *decl)
 {
 	struct ain_function_type f = {0};
-	f.name = encode_text_to_input_format(decl->name->text);
+	f.name = encode_text(decl->name->text);
 	if (ain_get_functype(ain, f.name) >= 0)
 		ERROR("Multiple definitions of function type '%s'", decl->name->text);
 	decl->func_no = ain_add_functype(ain, &f);
@@ -115,7 +115,7 @@ static void jaf_to_ain_type(possibly_unused struct ain *ain, struct ain_type *ou
 static void resolve_typedef(struct ain *ain, struct jaf_type_specifier *type)
 {
 	int no;
-	char *u = encode_text_to_input_format(type->name->text);
+	char *u = encode_text(type->name->text);
 	if ((no = ain_get_struct_no(ain, u)) >= 0) {
 		type->type = JAF_STRUCT;
 		type->struct_no = no;
@@ -230,7 +230,7 @@ static void analyze_message(struct jaf_env *env, struct jaf_block_item *item)
 		return;
 	}
 
-	char *u = encode_text_to_input_format(item->msg.func->text);
+	char *u = encode_text(item->msg.func->text);
 	if ((item->msg.func_no = ain_get_function_no(env->ain, u)) < 0)
 		ERROR("Undefined function: %s", item->msg.func->text);
 	free(u);
@@ -343,7 +343,7 @@ static void resolve_structdef_types(struct ain *ain, struct jaf_block_item *item
 	for (size_t i = 0; i < item->struc.members->nr_items; i++) {
 		if (jaf_members->items[i]->kind != JAF_DECL_VAR)
 			continue;
-		members[i].name = encode_text_to_input_format(jaf_members->items[i]->var.name->text);
+		members[i].name = encode_text(jaf_members->items[i]->var.name->text);
 		if (ain->version >= 12)
 			members[i].name2 = strdup("");
 		jaf_to_ain_type(ain, &members[i].type, jaf_members->items[i]->var.type);
@@ -420,7 +420,7 @@ static void jaf_resolve_types(struct ain *ain, struct jaf_block *block)
 
 static void init_variable(struct ain *ain, struct ain_variable *vars, int *var_no, struct jaf_vardecl *decl)
 {
-	vars[*var_no].name = encode_text_to_input_format(decl->name->text);
+	vars[*var_no].name = encode_text(decl->name->text);
 	if (ain->version >= 12)
 		vars[*var_no].name2 = strdup("");
 	jaf_to_ain_type(ain, &vars[*var_no].type, decl->type);
@@ -565,7 +565,7 @@ static void add_functype(struct ain *ain, struct jaf_fundecl *decl)
 
 static void add_global(struct ain *ain, struct jaf_vardecl *decl)
 {
-	char *u = encode_text_to_input_format(decl->name->text);
+	char *u = encode_text(decl->name->text);
 	struct ain_variable *v = ain_add_global(ain, u);
 	jaf_to_ain_type(ain, &v->type, decl->type);
 	decl->var_no = v - ain->globals;

src/tools/ainedit/jaf_types.c

@@ -324,7 +324,7 @@ static void jaf_check_types_identifier(struct jaf_env *env, struct jaf_expressio
 {
 	int no;
 	struct ain_variable *v;
-	char *u = encode_text_to_input_format(expr->s->text);
+	char *u = encode_text(expr->s->text);
 	if ((v = jaf_env_lookup(env, u, &no))) {
 		expr->valuetype = v->type;
 		expr->ident.var_type = v->var_type;
@@ -456,7 +456,7 @@ static void jaf_check_types_member(struct jaf_env *env, struct jaf_expression *e
 		TYPE_ERROR(expr->member.struc, AIN_STRUCT);
 
 	expr->member.member_no = -1;
-	char *u = encode_text_to_input_format(expr->member.name->text);
+	char *u = encode_text(expr->member.name->text);
 	struct ain_struct *s = &env->ain->structures[expr->member.struc->valuetype.struc];
 	for (int i = 0; i < s->nr_members; i++) {
 		if (!strcmp(s->members[i].name, u)) {