Add support for UTF-8 encoded scenario files

If `encoding = utf8` is specified in system3.ini, system3-sdl2 uses
UTF-8 as its multibyte character encoding. It will be used to run games
translated into non-(Japanese|English) languages.

src/common.h

@@ -48,6 +48,10 @@ inline void strcpy_s(char* dst, size_t n, const char* src)
 
 #endif // !WIN32
 
+#ifdef _MSC_VER
+#define strcasecmp _stricmp
+#endif
+
 // resource.cpp
 SDL_RWops* open_resource(const char* name, const char* type);
 SDL_RWops* open_file(const char* name);

src/config.cpp

@@ -9,10 +9,6 @@
 #include <unistd.h>
 #endif
 
-#ifdef _MSC_VER
-#define strcasecmp _stricmp
-#endif
-
 #define INIFILENAME "system3.ini"
 
 namespace {
@@ -64,6 +60,8 @@ Config::Config(int argc, char *argv[])
 			use_fm = true;
 		else if (strcmp(argv[i], "-game") == 0)
 			game_id = argv[++i];
+		else if (strcmp(argv[i], "-encoding") == 0)
+			encoding = argv[++i];
 	}
 }
 
@@ -103,6 +101,8 @@ void Config::load_ini()
 				use_fm = to_bool(val, lineno);
 			else if (sscanf(line, "game = %s", val))
 				game_id = val;
+			else if (sscanf(line, "encoding = %s", val))
+				encoding = val;
 			else if (!is_empty_line(line))
 				WARNING(INIFILENAME ":%d parse error", lineno);
 		} else if (!is_empty_line(line)) {

src/config.h

@@ -7,6 +7,7 @@ struct Config {
  public:
 	std::string font_file;
 	std::string game_id;
+	std::string encoding;
 	std::string save_dir;
 	std::string playlist;
 	bool use_fm = false;

src/encoding.cpp

@@ -22,6 +22,50 @@ private:
 	}
 };
 
+class Utf8Encoding : public Encoding {
+public:
+	int mblen(const unsigned char* s)
+	{
+		if (*s <= 0xbf)
+			return 1;
+		if (*s <= 0xdf)
+			return 2;
+		if (*s <= 0xef)
+			return 3;
+		return 4;
+	}
+
+	int next_codepoint(const unsigned char** str)
+	{
+		int code;
+		const unsigned char *s = *str;
+
+		if (*s <= 0x7f) {
+			code = *s++;
+		} else if (*s <= 0xbf) {
+			// Invalid UTF-8 sequence
+			code = '?';
+			s++;
+		} else if (*s <= 0xdf) {
+			code = (s[0] & 0x1f) << 6 | (s[1] & 0x3f);
+			s += 2;
+		} else if (*s <= 0xef) {
+			code = (s[0] & 0xf) << 12 | (s[1] & 0x3f) << 6 | (s[2] & 0x3f);
+			s += 3;
+		} else if (*s <= 0xf7) {
+			code = (s[0] & 0x7) << 18 | (s[1] & 0x3f) << 12 | (s[2] & 0x3f) << 6 | (s[3] & 0x3f);
+			s += 4;
+		} else {
+			code = 0xfffd;  // REPLACEMENT CHARACTER
+			s++;
+			while (0x80 <= *s && *s <= 0xbf)
+				s++;
+		}
+		*str = s;
+		return code;
+	}
+};
+
 int Encoding::mbslen(const unsigned char* s)
 {
 	int len = 0;
@@ -32,7 +76,18 @@ int Encoding::mbslen(const unsigned char* s)
 	return len;
 }
 
-std::unique_ptr<Encoding> Encoding::create()
+std::unique_ptr<Encoding> Encoding::create(const char* name)
 {
+	if (!strcasecmp(name, "Shift_JIS") ||
+		!strcasecmp(name, "Shift-JIS") ||
+		!strcasecmp(name, "SJIS") ||
+		!strcasecmp(name, "CP932"))
+		return std::make_unique<SjisEncoding>();
+
+	if (!strcasecmp(name, "UTF-8") ||
+		!strcasecmp(name, "UTF8"))
+		return std::make_unique<Utf8Encoding>();
+
+	WARNING("Unrecognized encoding: \"%s\"", name);
 	return std::make_unique<SjisEncoding>();
 }

src/encoding.h

@@ -5,7 +5,7 @@
 
 class Encoding {
  public:
-	static std::unique_ptr<Encoding> create();
+	static std::unique_ptr<Encoding> create(const char* name);
 
 	virtual ~Encoding() = default;
 

src/sys/nact.cpp

@@ -24,7 +24,7 @@ NACT::NACT(int sys_ver, uint32 crc32_a, uint32 crc32_b, const Config& config)
 	  crc32_a(crc32_a),
 	  crc32_b(crc32_b),
 	  lang(get_language()),
-	  encoding(Encoding::create())
+	  encoding(Encoding::create(get_encoding_name(config)))
 {
 	platform_initialize();
 

src/sys/nact.h

@@ -258,6 +258,7 @@ public:
 	static const int get_sys_ver(uint32 crc32_a, uint32 crc32_b);
 	const char* get_title();
 	Language get_language();
+	const char* get_encoding_name(const Config& config);
 	void text_wait();
 	void set_skip_menu_state(bool enabled, bool checked);
 

src/sys/nact_crc32.cpp

@@ -8,6 +8,7 @@
 #include <string.h>
 #include "crc32.h"
 #include "../fileio.h"
+#include "config.h"
 
 namespace {
 
@@ -156,3 +157,12 @@ Language NACT::get_language()
 		return entry->language;
 	return JAPANESE;
 }
+
+const char* NACT::get_encoding_name(const Config& config)
+{
+	if (!config.encoding.empty())
+		return config.encoding.c_str();
+
+	// TODO: Determine encoding from crc32
+	return "Shift_JIS";
+}