diff --git a/appveyor.yml b/appveyor.yml index 1bd2f52..faf8b4d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -27,6 +27,12 @@ after_build: md %TMP%\update\plugins\ md %TMP%\update\plugins\pv_equip\ + + md %TMP%\amd-pack\ + + md %TMP%\amd-pack\plugins\ + + md %TMP%\amd-pack\tools\ @@ -34,6 +40,8 @@ after_build: copy %APPVEYOR_BUILD_FOLDER%\source-code\x64\%CONFIGURATION%\*.dva %TMP%\update\plugins\ + del %TMP%\update\plugins\Novidia.dva + copy "%APPVEYOR_BUILD_FOLDER%\source-code\data\README - PD Loader.url" %TMP%\update\ copy %APPVEYOR_BUILD_FOLDER%\source-code\data\verify_base_game.sfv %TMP%\update\ @@ -44,23 +52,15 @@ after_build: copy %APPVEYOR_BUILD_FOLDER%\source-code\data\verify_mount_data.md5 %TMP%\update\ + copy %APPVEYOR_BUILD_FOLDER%\source-code\data\plugins\ShaderPatch.ini %TMP%\update\plugins\ + copy %APPVEYOR_BUILD_FOLDER%\source-code\data\patches\custom_freeplay_text_example.p %TMP%\update\patches\ - copy %APPVEYOR_BUILD_FOLDER%\source-code\data\plugins\ShaderPatch.ini %TMP%\update\plugins\ - - copy %APPVEYOR_BUILD_FOLDER%\source-code\data\plugins\keyconfig.ini %TMP%\update\plugins\ - - copy %APPVEYOR_BUILD_FOLDER%\source-code\data\plugins\pv_equip\modules.ini %TMP%\update\plugins\pv_equip\ - - copy %APPVEYOR_BUILD_FOLDER%\source-code\data\plugins\pv_equip\skins.ini %TMP%\update\plugins\pv_equip\ - - copy %APPVEYOR_BUILD_FOLDER%\source-code\data\plugins\pv_equip\sfx.ini %TMP%\update\plugins\pv_equip\ - copy %APPVEYOR_BUILD_FOLDER%\source-code\x64\%CONFIGURATION%\dinput8.dll %TMP%\full\ - - copy %APPVEYOR_BUILD_FOLDER%\source-code\data\patches\custom_freeplay_text_example.p %TMP%\full\patches\ - + copy %APPVEYOR_BUILD_FOLDER%\source-code\x64\%CONFIGURATION%\*.dva %TMP%\full\plugins\ + + del %TMP%\update\plugins\Novidia.dva copy "%APPVEYOR_BUILD_FOLDER%\source-code\data\README - PD Loader.url" %TMP%\full\ @@ -76,11 +76,17 @@ after_build: copy %APPVEYOR_BUILD_FOLDER%\source-code\data\plugins\*.* %TMP%\full\plugins\ - copy %APPVEYOR_BUILD_FOLDER%\source-code\data\plugins\*.* %TMP%\full\plugins\pv_equip\ + copy %APPVEYOR_BUILD_FOLDER%\source-code\data\plugins\pv_equip\*.* %TMP%\full\plugins\pv_equip\ copy %APPVEYOR_BUILD_FOLDER%\source-code\data\ram\databank\*.dat %TMP%\full\ram\databank\ + copy %APPVEYOR_BUILD_FOLDER%\source-code\data\patches\custom_freeplay_text_example.p %TMP%\full\patches\ + + copy %APPVEYOR_BUILD_FOLDER%\source-code\x64\%CONFIGURATION%\Novidia.dva %TMP%\amd-pack\plugins\ + + copy %APPVEYOR_BUILD_FOLDER%\source-code\data\amd-tools\*.* %TMP%\amd-pack\tools\ + cd %TMP%\full\ @@ -105,6 +111,20 @@ after_build: del "DELETE DNSAPI.DLL.bat" 7z a %APPVEYOR_BUILD_FOLDER%\PD-Loader-%CONFIGURATION%-Update-AppVeyor-Vista7.zip * + + cd %TMP%\amd-pack\tools\ + + C:\Python38\python.exe -m pip install cx_freeze + + C:\Python38\Scripts\cxfreeze.exe "ARB Patcher Source\main.py" --target-dir "ARB Patcher" + + copy "ARB Patcher Source\gamesettings" "ARB Patcher\gamesettings" + + rm "ARB Patcher Source" + + cd .. + + 7z a %APPVEYOR_BUILD_FOLDER%\PD-Loader-%CONFIGURATION%-AMDPack-AppVeyor.zip * artifacts: diff --git a/source-code/data/amd-tools/ARB Patcher Source/gamesettings/divaaft.py b/source-code/data/amd-tools/ARB Patcher Source/gamesettings/divaaft.py new file mode 100644 index 0000000..d0b10ed --- /dev/null +++ b/source-code/data/amd-tools/ARB Patcher Source/gamesettings/divaaft.py @@ -0,0 +1,106 @@ +# gane-specific settings for diva arcade future tone +GAME_NAME = "Project DIVA Arcade: Future Tone" + +from re import compile as recompile + +# some skinning can't be done due to using NV parameter buffers, so use an otherwise unused texture instead. +VP_SKINNING_REGEX = recompile(r"INT TEMP _adr;\s*?(BUFFER4[^;]*?;)([\s\S]*?_mtxpal)") +VP_SKINNING_SUB = "TEMP _adr;\n#\\1 \nTEMP mtx_tmp, mtx_tmp1, mtx_tmp2;\\2" + +# load the address temp with a texture coordinate instead +# hardcoded to assume a 768*1 pixel texture +VP_SKINNING_REGEX_2 = recompile(r"(BUFFER4[\s\S]*?)CVT.S32.F32\s*?(.*?),\s*?(.*?);") +VP_SKINNING_SUB_2 = "\\1MUL \\2, \\3, 0.0013037809647979;" #1/767 + +VP_SKINNING_REGEX_3 = recompile(r"(BUFFER4[\s\S]*?)(DP4 ([^;]*?), _mtxpal\[([^\]]*?\.x) \])") +VP_SKINNING_REGEX_3b = recompile(r"(BUFFER4[\s\S]*?)(DP4 ([^;]*?), _mtxpal\[([^\]]*?\.y) \])") +VP_SKINNING_REGEX_3c = recompile(r"(BUFFER4[\s\S]*?)(DP4 ([^;]*?), _mtxpal\[([^\]]*?\.z) \])") +VP_SKINNING_REGEX_3d = recompile(r"(BUFFER4[\s\S]*?)(DP4 ([^;]*?), _mtxpal\[([^\]]*?\.w) \])") +VP_SKINNING_SUB_3 = "\\1TEX mtx_tmp, \\4, texture[8], 1D; ADD \\4, \\4, 0.0013037809647979; TEX mtx_tmp1, \\4, texture[8], 1D; ADD \\4, \\4, 0.0013037809647979; TEX mtx_tmp2, \\4, texture[8], 1D; DP4 \\3, _mtxpal[\\4+ ]" + +VP_SKINNING_REGEX_4 = recompile(r"(BUFFER4[\s\S]*?)_mtxpal\[([^\]]*?\.\S)\+?(\s|\d)\]") +VP_SKINNING_SUB_4 = "\\1mtx_tmp\\3" + + +# some kind of effect (scrunching up or stretching?) in tights vp shaders starting with 11 that partially uses skinning of normals... +# works much the same as the regular vp skinning +# make these run before VP_SKINNING_REGEX_4 +#TIGHTS_SKINNING_EFFECT_REGEX = recompile(r"(BUFFER4[\s\S]*?)SUBC .*?, vertex.attrib\[15\], .*?;\s*?SUBC1 tmp, vertex.attrib\[15\], .*?;[\s\S]*?XPD .*?, .*?, .*?;\s*?DP3_SAT (.*?), .*?, .*?;") +#TIGHTS_SKINNING_EFFECT_SUB = "\\1MOV \\2, 0;" +TIGHTS_SKINNING_EFFECT_REGEX = recompile(r"(BUFFER4[\s\S]*?)CVT.S32.F32\s*?(.*?),\s*?(.*?);") +TIGHTS_SKINNING_EFFECT_SUB = "\\1MUL \\2, \\3, 0.0013037809647979;" +#TIGHTS_SKINNING_EFFECT_REGEX_2 = recompile(r"(BUFFER4[\s\S]*?)(IF GT1.y;[\s\S]*?|)(DP3 ([^;]*?), _mtxpal\[([^\]]*?\.x) \])(?![\s\S]*MUL _adr)") +#TIGHTS_SKINNING_EFFECT_REGEX_2b = recompile(r"(BUFFER4[\s\S]*?)(IF GT1.y;[\s\S]*?|)(DP3 ([^;]*?), _mtxpal\[([^\]]*?\.y) \])(?![\s\S]*MUL _adr)") +#TIGHTS_SKINNING_EFFECT_SUB_2 = "\\1TEX mtx_tmp, \\5, texture[8], 1D; ADD \\5, \\5, 0.0013037809647979; TEX mtx_tmp1, \\5, texture[8], 1D; ADD \\5, \\5, 0.0013037809647979; TEX mtx_tmp2, \\5, texture[8], 1D; \\2 DP3 \\4, _mtxpal[\\5+ ]" +TIGHTS_SKINNING_EFFECT_REGEX_2 = recompile(r"(BUFFER4[\s\S]*?)(DP3 ([^;]*?), _mtxpal\[([^\]]*?\.x) \])(?![\s\S]*MUL _adr)") +TIGHTS_SKINNING_EFFECT_SUB_2 = "\\1TEX mtx_tmp, \\4, texture[8], 1D; ADD \\4, \\4, 0.0013037809647979; TEX mtx_tmp1, \\4, texture[8], 1D; ADD \\4, \\4, 0.0013037809647979; TEX mtx_tmp2, \\4, texture[8], 1D; DP3 \\3, _mtxpal[\\4+ ]" +TIGHTS_SKINNING_EFFECT_REGEX_3 = recompile(r"(BUFFER4[\s\S]*?)(DP3 ([^;]*?), _mtxpal\[([^\]]*?\.y) \])(?![\s\S]*MUL _adr)") +TIGHTS_SKINNING_EFFECT_SUB_3 = "\\1TEX mtx_tmp, \\4, texture[8], 1D; DP3 \\3, _mtxpal[\\4+ ]" +TIGHTS_SKINNING_EFFECT_REGEX_4 = recompile(r"(BUFFER4[\s\S]*?)(DP3 ([^;]*?), _mtxpal\[([^\]]*?\.y)\+1\])(?![\s\S]*MUL _adr)") +TIGHTS_SKINNING_EFFECT_SUB_4 = "\\1ADD \\4, \\4, 0.0013037809647979; TEX mtx_tmp1, \\4, texture[8], 1D; ADD \\4, \\4, 0.0013037809647979; TEX mtx_tmp2, \\4, texture[8], 1D; DP3 \\3, _mtxpal[\\4+1]" + + + +# some skinning just needs some branching fixup for AMD BRA working different (needs a target per branch instruction apparently) +SKIN_BRA_REGEX = recompile(r"BRA skinning_(?:\S*)end (\(.*?\.y\))*?\s*?;") +SKIN_BRA_SUB = "BRA sk_end1 \\1;" +SKIN_BRA_REGEX_2 = recompile(r"BRA skinning_(?:\S*)end (\(.*?\.z\))*?\s*?;") +SKIN_BRA_SUB_2 = "BRA sk_end2 \\1;" +SKIN_BRA_REGEX_3 = recompile(r"BRA skinning_(?:\S*)end (\(.*?\.w\))*?\s*?;") +SKIN_BRA_SUB_3 = "BRA sk_end3 \\1;" + +SKIN_BRA_TGT_REGEX = recompile(r"skinning_(?:\S*)end:") +SKIN_BRA_TGT_SUB = "\nsk_end1:\nsk_end2:\nsk_end3:\n" + + +MATCH_FP_FNAME = recompile(r".*fp") +MATCH_VP_FNAME = recompile(r".*vp") +MATCH_ALL_FNAME = recompile(r".*") +MATCH_FONT_FNAME = recompile(r"font.*fp") +MATCH_TIGHTS_FNAME = recompile(r"tights.*vp") + +# fourth param: apply continuously until no more matches are found +fix_repls = [ + (MATCH_VP_FNAME, VP_SKINNING_REGEX, VP_SKINNING_SUB, False), + (MATCH_VP_FNAME, VP_SKINNING_REGEX_2, VP_SKINNING_SUB_2, False), + (MATCH_VP_FNAME, VP_SKINNING_REGEX_3, VP_SKINNING_SUB_3, False), + (MATCH_VP_FNAME, VP_SKINNING_REGEX_3b, VP_SKINNING_SUB_3, False), + (MATCH_VP_FNAME, VP_SKINNING_REGEX_3c, VP_SKINNING_SUB_3, False), + (MATCH_VP_FNAME, VP_SKINNING_REGEX_3d, VP_SKINNING_SUB_3, False), + (MATCH_TIGHTS_FNAME, TIGHTS_SKINNING_EFFECT_REGEX, TIGHTS_SKINNING_EFFECT_SUB, False), + (MATCH_TIGHTS_FNAME, TIGHTS_SKINNING_EFFECT_REGEX_2, TIGHTS_SKINNING_EFFECT_SUB_2, False), + (MATCH_TIGHTS_FNAME, TIGHTS_SKINNING_EFFECT_REGEX_3, TIGHTS_SKINNING_EFFECT_SUB_3, False), + (MATCH_TIGHTS_FNAME, TIGHTS_SKINNING_EFFECT_REGEX_4, TIGHTS_SKINNING_EFFECT_SUB_4, False), + (MATCH_VP_FNAME, VP_SKINNING_REGEX_4, VP_SKINNING_SUB_4, True), + (MATCH_VP_FNAME, SKIN_BRA_REGEX, SKIN_BRA_SUB, False), + (MATCH_VP_FNAME, SKIN_BRA_REGEX_2, SKIN_BRA_SUB_2, False), + (MATCH_VP_FNAME, SKIN_BRA_REGEX_3, SKIN_BRA_SUB_3, False), + (MATCH_VP_FNAME, SKIN_BRA_TGT_REGEX, SKIN_BRA_TGT_SUB, False), +] + + +# use to open a differnt file instead of the original shader (eg. to replace with an easier-to-patch variant +def filename_filter(fname): + if fname == 'sss_filter.130.fp': + return 'sss_filter.120.fp' + elif fname == 'sss_filter.131.fp': + return 'sss_filter.121.fp' + else: + return fname + + +# use to tweak reasults after patching +def post_filter(fname, f_full): + # ideally this could do with a generic version in the main script... + f_full = f_full.replace('MOV.F.CC1 ', 'MOV ') + + f_full = f_full.replace('\nBUFFER4', '\n#BUFFER4') + + # fix tex sampler offsets + # diva has something we can use for pixel size (program.local[0]), but it isn't in fonts so they get an approximation instead + if MATCH_FONT_FNAME.match(fname): + f_full = f_full.replace('{ 0.00078, 0.0014 }; ADD tex_offset_coord', '0.00035; ADD tex_offset_coord') + else: + f_full = f_full.replace('{ 0.00078, 0.0014 }; ADD tex_offset_coord', 'program.local[0]; ADD tex_offset_coord') + + return f_full \ No newline at end of file diff --git a/source-code/data/amd-tools/ARB Patcher Source/main.py b/source-code/data/amd-tools/ARB Patcher Source/main.py new file mode 100644 index 0000000..f6f7d1d --- /dev/null +++ b/source-code/data/amd-tools/ARB Patcher Source/main.py @@ -0,0 +1,187 @@ +from rules import * + +def patch_shader(fname, f_lines, game_settings_module=None, enable_print_warnings=True): + f_lines[0] += "#" + fname +"\n" + + + # do attrib, output, param simplification and replace header tags + attribs = {} + outputs = {} + params = {} + + for i in range(0, len(f_lines)): + no_repl = False + + m = ATTRIB_REGEX.match(f_lines[i]) + while m: + attribs[m.group(3)] = m.group(4) + f_lines[i] = ATTRIB_REGEX.sub('#\\1 \n', f_lines[i], count=1) + no_repl = True + m = ATTRIB_REGEX.match(f_lines[i]) + + m = OUTPUT_REGEX.match(f_lines[i]) + while m: + outputs[m.group(3)] = m.group(4) + f_lines[i] = OUTPUT_REGEX.sub('#\\1 \n', f_lines[i], count=1) + no_repl = True + m = OUTPUT_REGEX.match(f_lines[i]) + + m = PARAM_REGEX.match(f_lines[i]) + while m: + params[m.group(3)] = m.group(4) + f_lines[i] = PARAM_REGEX.sub('#\\1 \n', f_lines[i], count=1) + no_repl = True + m = PARAM_REGEX.match(f_lines[i]) + + if no_repl: + continue + + replace_tokens = attribs.copy() + replace_tokens.update(outputs) + replace_tokens.update(params) + + # sort from longest to shortest to minimize risk of names that are substrings of others causing issues + for k in sorted(replace_tokens.keys(), reverse=True, key=lambda s: len(s)): + # look for specific preceding characters to not replace in the middle of a name + f_lines[i] = f_lines[i].replace(' ' + k, ' ' + replace_tokens[k]) + f_lines[i] = f_lines[i].replace('-' + k, '-' + replace_tokens[k]) + f_lines[i] = f_lines[i].replace('{' + k, '{' + replace_tokens[k]) + + # options are required for conditional stuff... + # tysm nezarn for teaching me this because otherwise the patch would never be anywhere near as complete + f_lines[i] = NVFP_HEADER_REGEX.sub("!!ARBfp1.0\nOPTION NV_fragment_program2;", f_lines[i]) + f_lines[i] = NVVP_HEADER_REGEX.sub("!!ARBvp1.0\nOPTION NV_vertex_program3;", f_lines[i]) + + f_full = ''.join(f_lines) + + f_full = UNUSED_PARAM_REGEX.sub("#\\1 \n", f_full) + + + # add a temp vars for NRM and SSG replacements before their first use + f_full = f_full.replace('NRM ', 'TEMP nrm_tmp; NRM ', 1) + f_full = f_full.replace('SSG ', 'TEMP ssg_tmp1, ssg_tmp2; SSG ', 1) + # this needs to use the regex to match because it isn't just a single instruction + f_full = SAMPLER_OFFSET_REGEX.sub('TEMP tex_offset_coord; \\g<0>', f_full, count=1) + + for r in fix_repls: + if r[0].match(fname): + if r[3]: # r[3]: apply until no matches + while r[1].search(f_full): + f_full = r[1].sub(r[2], f_full) + else: + f_full = r[1].sub(r[2], f_full) + + if game_settings_module and game_settings_module.fix_repls: + for r in game_settings_module.fix_repls: + if r[0].match(fname): + if r[3]: # r[3]: apply until no matches + while r[1].search(f_full): + f_full = r[1].sub(r[2], f_full) + else: + f_full = r[1].sub(r[2], f_full) + + + f_full = f_full.replace('INT TEMP ', 'TEMP ') + + if game_settings_module and game_settings_module.post_filter: + f_full = game_settings_module.post_filter(fname, f_full) + + + bad_instr = BAD_INSTR_REGEX.search(f_full) + if bad_instr: + if enable_print_warnings: + print ('Warning -- unpatched known bad instruction in {}: {}'.format(fname, bad_instr.group(1))) + f_full = BAD_INSTR_REGEX.sub("#\\1 \n", f_full) + + f_full = f_full.replace('\r', '') + + return f_full + +if __name__ == '__main__': + from os import listdir, makedirs + from os.path import join as joinpath, splitext, isfile, exists + from re import compile as recompile + import sys + import importlib + + def get_args(): + # make a list of game settings modules + game_modules_list = [f[:-3] for f in listdir('gamesettings') if isfile(joinpath('gamesettings', f)) and splitext(f)[1] == '.py'] + + import argparse + parser = argparse.ArgumentParser(description='An attempt at converting Nvidia-only ARB shaders to work on AMD.') + parser.add_argument('-i', '--in_dir', default='shader in', help='input directory containing .vp and .fp files') + parser.add_argument('-o', '--out_dir', default='shader patched', help='output directory') + parser.add_argument('-f', '--file_filter', default=None, help='only process files that match this filter (regex)') + parser.add_argument('-g', '--game_settings', default=None, choices=game_modules_list, help='game-specific settings to load') + + parser.add_argument('--list_games', action='store_true', help='list game names for game-specific settings options instead of patching shaders') + + return parser.parse_args() + + def print_game_modules(): + # make a list of game settings modules + game_modules_list = [f[:-3] for f in listdir('gamesettings') if isfile(joinpath('gamesettings', f)) and splitext(f)[1] == '.py'] + + for game in game_modules_list: + game_settings_module = importlib.import_module('gamesettings.{}'.format(game)) + + if game_settings_module.GAME_NAME: + full_name = game_settings_module.GAME_NAME + else: + full_name = 'Unknown' + + print ('{}: {}'.format(game, full_name)) + + + args = get_args() + + if args.list_games: + print_game_modules() + sys.exit() + + if args.game_settings: + game_settings_module = importlib.import_module('gamesettings.{}'.format(args.game_settings)) + else: + game_settings_module = None + + IN_DIR = args.in_dir + OUT_DIR = args.out_dir + + if not exists(IN_DIR): + print ("Directory '{}' does not exist. Aborting".format(IN_DIR)) + sys.exit() + elif isfile(IN_DIR): + print ("'{}' is not a directory. Aborting".format(IN_DIR)) + sys.exit() + + if exists(OUT_DIR): + if isfile(OUT_DIR): + print ("'{}' already exists but is not a directory. Aborting".format(OUT_DIR)) + sys.exit() + else: + makedirs(OUT_DIR) + + if args.file_filter: + file_filter = recompile(args.file_filter) + else: + file_filter = None + + for fname in listdir(IN_DIR): + if file_filter and not file_filter.match(fname): + continue + + if game_settings_module and game_settings_module.filename_filter: + openname = game_settings_module.filename_filter(fname) + else: + openname = fname + + with open(joinpath(IN_DIR, openname), 'r') as f: + f_lines = f.readlines() + + f_full = patch_shader(fname, f_lines, game_settings_module, True) + + print ('Patched {}'.format(fname)) + + with open(joinpath(OUT_DIR, fname), 'wb') as f: + f.write(f_full.encode('utf-8')) diff --git a/source-code/data/amd-tools/ARB Patcher Source/rules.py b/source-code/data/amd-tools/ARB Patcher Source/rules.py new file mode 100644 index 0000000..007d792 --- /dev/null +++ b/source-code/data/amd-tools/ARB Patcher Source/rules.py @@ -0,0 +1,62 @@ +# regexes and config info is stored here to reduce clutter in main script +from re import compile as recompile + +# inline these because... idk exactly +ATTRIB_REGEX = recompile(r"(?