add amd tools, cleanup some config files that shouldn't be in update zips

This commit is contained in:
somewhatlurker
2020-07-25 23:44:38 +10:00
parent faf460be9c
commit 0bd1cd1d50
6 changed files with 419 additions and 14 deletions
@@ -0,0 +1,106 @@
# gane-specific settings for diva arcade future tone
GAME_NAME = "Project DIVA Arcade: Future Tone"
from re import compile as recompile
# some skinning can't be done due to using NV parameter buffers, so use an otherwise unused texture instead.
VP_SKINNING_REGEX = recompile(r"INT TEMP _adr;\s*?(BUFFER4[^;]*?;)([\s\S]*?_mtxpal)")
VP_SKINNING_SUB = "TEMP _adr;\n#\\1 \nTEMP mtx_tmp, mtx_tmp1, mtx_tmp2;\\2"
# load the address temp with a texture coordinate instead
# hardcoded to assume a 768*1 pixel texture
VP_SKINNING_REGEX_2 = recompile(r"(BUFFER4[\s\S]*?)CVT.S32.F32\s*?(.*?),\s*?(.*?);")
VP_SKINNING_SUB_2 = "\\1MUL \\2, \\3, 0.0013037809647979;" #1/767
VP_SKINNING_REGEX_3 = recompile(r"(BUFFER4[\s\S]*?)(DP4 ([^;]*?), _mtxpal\[([^\]]*?\.x) \])")
VP_SKINNING_REGEX_3b = recompile(r"(BUFFER4[\s\S]*?)(DP4 ([^;]*?), _mtxpal\[([^\]]*?\.y) \])")
VP_SKINNING_REGEX_3c = recompile(r"(BUFFER4[\s\S]*?)(DP4 ([^;]*?), _mtxpal\[([^\]]*?\.z) \])")
VP_SKINNING_REGEX_3d = recompile(r"(BUFFER4[\s\S]*?)(DP4 ([^;]*?), _mtxpal\[([^\]]*?\.w) \])")
VP_SKINNING_SUB_3 = "\\1TEX mtx_tmp, \\4, texture[8], 1D; ADD \\4, \\4, 0.0013037809647979; TEX mtx_tmp1, \\4, texture[8], 1D; ADD \\4, \\4, 0.0013037809647979; TEX mtx_tmp2, \\4, texture[8], 1D; DP4 \\3, _mtxpal[\\4+ ]"
VP_SKINNING_REGEX_4 = recompile(r"(BUFFER4[\s\S]*?)_mtxpal\[([^\]]*?\.\S)\+?(\s|\d)\]")
VP_SKINNING_SUB_4 = "\\1mtx_tmp\\3"
# some kind of effect (scrunching up or stretching?) in tights vp shaders starting with 11 that partially uses skinning of normals...
# works much the same as the regular vp skinning
# make these run before VP_SKINNING_REGEX_4
#TIGHTS_SKINNING_EFFECT_REGEX = recompile(r"(BUFFER4[\s\S]*?)SUBC .*?, vertex.attrib\[15\], .*?;\s*?SUBC1 tmp, vertex.attrib\[15\], .*?;[\s\S]*?XPD .*?, .*?, .*?;\s*?DP3_SAT (.*?), .*?, .*?;")
#TIGHTS_SKINNING_EFFECT_SUB = "\\1MOV \\2, 0;"
TIGHTS_SKINNING_EFFECT_REGEX = recompile(r"(BUFFER4[\s\S]*?)CVT.S32.F32\s*?(.*?),\s*?(.*?);")
TIGHTS_SKINNING_EFFECT_SUB = "\\1MUL \\2, \\3, 0.0013037809647979;"
#TIGHTS_SKINNING_EFFECT_REGEX_2 = recompile(r"(BUFFER4[\s\S]*?)(IF GT1.y;[\s\S]*?|)(DP3 ([^;]*?), _mtxpal\[([^\]]*?\.x) \])(?![\s\S]*MUL _adr)")
#TIGHTS_SKINNING_EFFECT_REGEX_2b = recompile(r"(BUFFER4[\s\S]*?)(IF GT1.y;[\s\S]*?|)(DP3 ([^;]*?), _mtxpal\[([^\]]*?\.y) \])(?![\s\S]*MUL _adr)")
#TIGHTS_SKINNING_EFFECT_SUB_2 = "\\1TEX mtx_tmp, \\5, texture[8], 1D; ADD \\5, \\5, 0.0013037809647979; TEX mtx_tmp1, \\5, texture[8], 1D; ADD \\5, \\5, 0.0013037809647979; TEX mtx_tmp2, \\5, texture[8], 1D; \\2 DP3 \\4, _mtxpal[\\5+ ]"
TIGHTS_SKINNING_EFFECT_REGEX_2 = recompile(r"(BUFFER4[\s\S]*?)(DP3 ([^;]*?), _mtxpal\[([^\]]*?\.x) \])(?![\s\S]*MUL _adr)")
TIGHTS_SKINNING_EFFECT_SUB_2 = "\\1TEX mtx_tmp, \\4, texture[8], 1D; ADD \\4, \\4, 0.0013037809647979; TEX mtx_tmp1, \\4, texture[8], 1D; ADD \\4, \\4, 0.0013037809647979; TEX mtx_tmp2, \\4, texture[8], 1D; DP3 \\3, _mtxpal[\\4+ ]"
TIGHTS_SKINNING_EFFECT_REGEX_3 = recompile(r"(BUFFER4[\s\S]*?)(DP3 ([^;]*?), _mtxpal\[([^\]]*?\.y) \])(?![\s\S]*MUL _adr)")
TIGHTS_SKINNING_EFFECT_SUB_3 = "\\1TEX mtx_tmp, \\4, texture[8], 1D; DP3 \\3, _mtxpal[\\4+ ]"
TIGHTS_SKINNING_EFFECT_REGEX_4 = recompile(r"(BUFFER4[\s\S]*?)(DP3 ([^;]*?), _mtxpal\[([^\]]*?\.y)\+1\])(?![\s\S]*MUL _adr)")
TIGHTS_SKINNING_EFFECT_SUB_4 = "\\1ADD \\4, \\4, 0.0013037809647979; TEX mtx_tmp1, \\4, texture[8], 1D; ADD \\4, \\4, 0.0013037809647979; TEX mtx_tmp2, \\4, texture[8], 1D; DP3 \\3, _mtxpal[\\4+1]"
# some skinning just needs some branching fixup for AMD BRA working different (needs a target per branch instruction apparently)
SKIN_BRA_REGEX = recompile(r"BRA skinning_(?:\S*)end (\(.*?\.y\))*?\s*?;")
SKIN_BRA_SUB = "BRA sk_end1 \\1;"
SKIN_BRA_REGEX_2 = recompile(r"BRA skinning_(?:\S*)end (\(.*?\.z\))*?\s*?;")
SKIN_BRA_SUB_2 = "BRA sk_end2 \\1;"
SKIN_BRA_REGEX_3 = recompile(r"BRA skinning_(?:\S*)end (\(.*?\.w\))*?\s*?;")
SKIN_BRA_SUB_3 = "BRA sk_end3 \\1;"
SKIN_BRA_TGT_REGEX = recompile(r"skinning_(?:\S*)end:")
SKIN_BRA_TGT_SUB = "\nsk_end1:\nsk_end2:\nsk_end3:\n"
MATCH_FP_FNAME = recompile(r".*fp")
MATCH_VP_FNAME = recompile(r".*vp")
MATCH_ALL_FNAME = recompile(r".*")
MATCH_FONT_FNAME = recompile(r"font.*fp")
MATCH_TIGHTS_FNAME = recompile(r"tights.*vp")
# fourth param: apply continuously until no more matches are found
fix_repls = [
(MATCH_VP_FNAME, VP_SKINNING_REGEX, VP_SKINNING_SUB, False),
(MATCH_VP_FNAME, VP_SKINNING_REGEX_2, VP_SKINNING_SUB_2, False),
(MATCH_VP_FNAME, VP_SKINNING_REGEX_3, VP_SKINNING_SUB_3, False),
(MATCH_VP_FNAME, VP_SKINNING_REGEX_3b, VP_SKINNING_SUB_3, False),
(MATCH_VP_FNAME, VP_SKINNING_REGEX_3c, VP_SKINNING_SUB_3, False),
(MATCH_VP_FNAME, VP_SKINNING_REGEX_3d, VP_SKINNING_SUB_3, False),
(MATCH_TIGHTS_FNAME, TIGHTS_SKINNING_EFFECT_REGEX, TIGHTS_SKINNING_EFFECT_SUB, False),
(MATCH_TIGHTS_FNAME, TIGHTS_SKINNING_EFFECT_REGEX_2, TIGHTS_SKINNING_EFFECT_SUB_2, False),
(MATCH_TIGHTS_FNAME, TIGHTS_SKINNING_EFFECT_REGEX_3, TIGHTS_SKINNING_EFFECT_SUB_3, False),
(MATCH_TIGHTS_FNAME, TIGHTS_SKINNING_EFFECT_REGEX_4, TIGHTS_SKINNING_EFFECT_SUB_4, False),
(MATCH_VP_FNAME, VP_SKINNING_REGEX_4, VP_SKINNING_SUB_4, True),
(MATCH_VP_FNAME, SKIN_BRA_REGEX, SKIN_BRA_SUB, False),
(MATCH_VP_FNAME, SKIN_BRA_REGEX_2, SKIN_BRA_SUB_2, False),
(MATCH_VP_FNAME, SKIN_BRA_REGEX_3, SKIN_BRA_SUB_3, False),
(MATCH_VP_FNAME, SKIN_BRA_TGT_REGEX, SKIN_BRA_TGT_SUB, False),
]
# use to open a differnt file instead of the original shader (eg. to replace with an easier-to-patch variant
def filename_filter(fname):
if fname == 'sss_filter.130.fp':
return 'sss_filter.120.fp'
elif fname == 'sss_filter.131.fp':
return 'sss_filter.121.fp'
else:
return fname
# use to tweak reasults after patching
def post_filter(fname, f_full):
# ideally this could do with a generic version in the main script...
f_full = f_full.replace('MOV.F.CC1 ', 'MOV ')
f_full = f_full.replace('\nBUFFER4', '\n#BUFFER4')
# fix tex sampler offsets
# diva has something we can use for pixel size (program.local[0]), but it isn't in fonts so they get an approximation instead
if MATCH_FONT_FNAME.match(fname):
f_full = f_full.replace('{ 0.00078, 0.0014 }; ADD tex_offset_coord', '0.00035; ADD tex_offset_coord')
else:
f_full = f_full.replace('{ 0.00078, 0.0014 }; ADD tex_offset_coord', 'program.local[0]; ADD tex_offset_coord')
return f_full
@@ -0,0 +1,187 @@
from rules import *
def patch_shader(fname, f_lines, game_settings_module=None, enable_print_warnings=True):
f_lines[0] += "#" + fname +"\n"
# do attrib, output, param simplification and replace header tags
attribs = {}
outputs = {}
params = {}
for i in range(0, len(f_lines)):
no_repl = False
m = ATTRIB_REGEX.match(f_lines[i])
while m:
attribs[m.group(3)] = m.group(4)
f_lines[i] = ATTRIB_REGEX.sub('#\\1 \n', f_lines[i], count=1)
no_repl = True
m = ATTRIB_REGEX.match(f_lines[i])
m = OUTPUT_REGEX.match(f_lines[i])
while m:
outputs[m.group(3)] = m.group(4)
f_lines[i] = OUTPUT_REGEX.sub('#\\1 \n', f_lines[i], count=1)
no_repl = True
m = OUTPUT_REGEX.match(f_lines[i])
m = PARAM_REGEX.match(f_lines[i])
while m:
params[m.group(3)] = m.group(4)
f_lines[i] = PARAM_REGEX.sub('#\\1 \n', f_lines[i], count=1)
no_repl = True
m = PARAM_REGEX.match(f_lines[i])
if no_repl:
continue
replace_tokens = attribs.copy()
replace_tokens.update(outputs)
replace_tokens.update(params)
# sort from longest to shortest to minimize risk of names that are substrings of others causing issues
for k in sorted(replace_tokens.keys(), reverse=True, key=lambda s: len(s)):
# look for specific preceding characters to not replace in the middle of a name
f_lines[i] = f_lines[i].replace(' ' + k, ' ' + replace_tokens[k])
f_lines[i] = f_lines[i].replace('-' + k, '-' + replace_tokens[k])
f_lines[i] = f_lines[i].replace('{' + k, '{' + replace_tokens[k])
# options are required for conditional stuff...
# tysm nezarn for teaching me this because otherwise the patch would never be anywhere near as complete
f_lines[i] = NVFP_HEADER_REGEX.sub("!!ARBfp1.0\nOPTION NV_fragment_program2;", f_lines[i])
f_lines[i] = NVVP_HEADER_REGEX.sub("!!ARBvp1.0\nOPTION NV_vertex_program3;", f_lines[i])
f_full = ''.join(f_lines)
f_full = UNUSED_PARAM_REGEX.sub("#\\1 \n", f_full)
# add a temp vars for NRM and SSG replacements before their first use
f_full = f_full.replace('NRM ', 'TEMP nrm_tmp; NRM ', 1)
f_full = f_full.replace('SSG ', 'TEMP ssg_tmp1, ssg_tmp2; SSG ', 1)
# this needs to use the regex to match because it isn't just a single instruction
f_full = SAMPLER_OFFSET_REGEX.sub('TEMP tex_offset_coord; \\g<0>', f_full, count=1)
for r in fix_repls:
if r[0].match(fname):
if r[3]: # r[3]: apply until no matches
while r[1].search(f_full):
f_full = r[1].sub(r[2], f_full)
else:
f_full = r[1].sub(r[2], f_full)
if game_settings_module and game_settings_module.fix_repls:
for r in game_settings_module.fix_repls:
if r[0].match(fname):
if r[3]: # r[3]: apply until no matches
while r[1].search(f_full):
f_full = r[1].sub(r[2], f_full)
else:
f_full = r[1].sub(r[2], f_full)
f_full = f_full.replace('INT TEMP ', 'TEMP ')
if game_settings_module and game_settings_module.post_filter:
f_full = game_settings_module.post_filter(fname, f_full)
bad_instr = BAD_INSTR_REGEX.search(f_full)
if bad_instr:
if enable_print_warnings:
print ('Warning -- unpatched known bad instruction in {}: {}'.format(fname, bad_instr.group(1)))
f_full = BAD_INSTR_REGEX.sub("#\\1 \n", f_full)
f_full = f_full.replace('\r', '')
return f_full
if __name__ == '__main__':
from os import listdir, makedirs
from os.path import join as joinpath, splitext, isfile, exists
from re import compile as recompile
import sys
import importlib
def get_args():
# make a list of game settings modules
game_modules_list = [f[:-3] for f in listdir('gamesettings') if isfile(joinpath('gamesettings', f)) and splitext(f)[1] == '.py']
import argparse
parser = argparse.ArgumentParser(description='An attempt at converting Nvidia-only ARB shaders to work on AMD.')
parser.add_argument('-i', '--in_dir', default='shader in', help='input directory containing .vp and .fp files')
parser.add_argument('-o', '--out_dir', default='shader patched', help='output directory')
parser.add_argument('-f', '--file_filter', default=None, help='only process files that match this filter (regex)')
parser.add_argument('-g', '--game_settings', default=None, choices=game_modules_list, help='game-specific settings to load')
parser.add_argument('--list_games', action='store_true', help='list game names for game-specific settings options instead of patching shaders')
return parser.parse_args()
def print_game_modules():
# make a list of game settings modules
game_modules_list = [f[:-3] for f in listdir('gamesettings') if isfile(joinpath('gamesettings', f)) and splitext(f)[1] == '.py']
for game in game_modules_list:
game_settings_module = importlib.import_module('gamesettings.{}'.format(game))
if game_settings_module.GAME_NAME:
full_name = game_settings_module.GAME_NAME
else:
full_name = 'Unknown'
print ('{}: {}'.format(game, full_name))
args = get_args()
if args.list_games:
print_game_modules()
sys.exit()
if args.game_settings:
game_settings_module = importlib.import_module('gamesettings.{}'.format(args.game_settings))
else:
game_settings_module = None
IN_DIR = args.in_dir
OUT_DIR = args.out_dir
if not exists(IN_DIR):
print ("Directory '{}' does not exist. Aborting".format(IN_DIR))
sys.exit()
elif isfile(IN_DIR):
print ("'{}' is not a directory. Aborting".format(IN_DIR))
sys.exit()
if exists(OUT_DIR):
if isfile(OUT_DIR):
print ("'{}' already exists but is not a directory. Aborting".format(OUT_DIR))
sys.exit()
else:
makedirs(OUT_DIR)
if args.file_filter:
file_filter = recompile(args.file_filter)
else:
file_filter = None
for fname in listdir(IN_DIR):
if file_filter and not file_filter.match(fname):
continue
if game_settings_module and game_settings_module.filename_filter:
openname = game_settings_module.filename_filter(fname)
else:
openname = fname
with open(joinpath(IN_DIR, openname), 'r') as f:
f_lines = f.readlines()
f_full = patch_shader(fname, f_lines, game_settings_module, True)
print ('Patched {}'.format(fname))
with open(joinpath(OUT_DIR, fname), 'wb') as f:
f.write(f_full.encode('utf-8'))
@@ -0,0 +1,62 @@
# regexes and config info is stored here to reduce clutter in main script
from re import compile as recompile
# inline these because... idk exactly
ATTRIB_REGEX = recompile(r"(?<!#)(?<!#\s)((SHORT\s*?|LONG\s*?)?ATTRIB\s*?([^\[\]\s]*?)\s*?=\s*?([^\s].*?);)\s*")
OUTPUT_REGEX = recompile(r"(?<!#)(?<!#\s)((SHORT\s*?|LONG\s*?)?OUTPUT\s*?([^\[\]\s]*?)\s*?=\s*?([^\s].*?);)\s*")
PARAM_REGEX = recompile(r"(?<!#)(?<!#\s)((SHORT\s*?|LONG\s*?)?PARAM\s*?([^\[\]\s]*?)\s*?=\s*?([^\s].*?);)\s*") # (params because of limit)
# help reduce param count more by commenting out unused ones that weren't inlined (array params can't be inlined)
UNUSED_PARAM_REGEX = recompile(r"(?<!#)(?<!#\s)((SHORT\s*?|LONG\s*?)?PARAM\s*?([^\[\]\s]*?)(\s*?\[.*?\])?\s*?=\s*?([^\s].*?);)\s*(?![\s\S]*[\s\-{]\3[\s,;\[])")
# I think some of these instructions might still be left after other patches.. they will be commented out
BAD_INSTR_REGEX = recompile(r"((?<!#)(?<!#\s)(CVT.S32.F32|BUFFER4|POW|NRM|REP)\s+?.*?;|ENDREP;|[^;\n]*\s*?(p_coef2d\[idx.x\]).*?;)\s*")
# find the header tag to replace with !!ARBxx1.0 and an NV OPTION
NVFP_HEADER_REGEX = recompile(r"!!NVfp\d\.\d")
NVVP_HEADER_REGEX = recompile(r"!!NVvp\d\.\d")
# POW, NRM, and SSG sometimes(?) don't work (depending on inputs?), so just reimplement with other instructions
POW_REGEX = recompile(r"POW(\s+?|_SAT\s+?)(.*?),\s+?(.*?),\s+?(.*?);")
POW_SUB = "LG2 \\2, \\3; MUL \\2, \\2, \\4; EX2\\1\\2, \\2;"
NRM_REGEX = recompile(r"NRM\s+?(.*?),\s+?(.*?);")
NRM_SUB = "DP3 nrm_tmp.w, \\2, \\2; RSQ nrm_tmp.w, nrm_tmp.w; MUL \\1, nrm_tmp.w, \\2;"
SSG_REGEX = recompile(r"SSG\s+?(\S.*?\.\S*?),\s+?(.*?);")
SSG_SUB = "SGE ssg_tmp1, \\2, 0; SUB ssg_tmp1, ssg_tmp1, 1; SGT ssg_tmp2, \\2, 0; ADD \\1, ssg_tmp1, ssg_tmp2;"
# seems like using a TEX with pixel offsets is pretty common, but I can't make that work
# afaik there's no general way to get texture resolution so I'll approximate one here that can be refined later per-shader
# ({ 0.00078, 0.0014 } is approximately one pixel of a 720p texture/framebuffer)
SAMPLER_OFFSET_REGEX = recompile(r"TEX\s+?([^;,]*?),\s+?([^;,]*?),\s+?([^;,]*?),\s+?2D,\s+?\((.*?)\);")
SAMPLER_OFFSET_SUB = "MUL tex_offset_coord, {\\4}, { 0.00078, 0.0014 }; ADD tex_offset_coord, \\2, tex_offset_coord; TEX \\1, tex_offset_coord, \\3, 2D;"
# ret at the end of an IF statement is moved to outside of it (or won't compile)
# not a perfect solution, but works well enough
RET_IF_REGEX = recompile(r"(IF\s+?(.*?);[\s\S]*?)RET;\s*?ENDIF;")
RET_IF_SUB = "\\1ENDIF;\nRET (\\2);"
# RET behaves differently on AMD, so convert conditional reta into if/else
# (may not work properly with CAL)
CONDITIONAL_RET_REGEX = recompile(r"RET\s+?\((.*?)\);([\s\S]*?)(END\s*?$|RET;)")
CONDITIONAL_RET_SUB = "IF \\1; ELSE; \\2 ENDIF; \\3"
MATCH_FP_FNAME = recompile(r".*fp")
MATCH_VP_FNAME = recompile(r".*vp")
MATCH_ALL_FNAME = recompile(r".*")
# fourth param: apply continuously until no more matches are found
fix_repls = [
(MATCH_ALL_FNAME, POW_REGEX, POW_SUB, False),
(MATCH_ALL_FNAME, NRM_REGEX, NRM_SUB, False),
(MATCH_ALL_FNAME, SSG_REGEX, SSG_SUB, False),
(MATCH_ALL_FNAME, SAMPLER_OFFSET_REGEX, SAMPLER_OFFSET_SUB, False),
(MATCH_ALL_FNAME, RET_IF_REGEX, RET_IF_SUB, False),
(MATCH_ALL_FNAME, CONDITIONAL_RET_REGEX, CONDITIONAL_RET_SUB, True),
]