Add iohook for GetFileType (#7)

Unity (or mono in that case) is a bit special when it comes to handling file I/O, as before any operation (read, write) it will do a call to `GetFileType` to determine how to set the parameters for the following winapi calls. Handles that we replace with `iohook_open_nul_fd` however don't return the expected `FILE_TYPE_DISK`, as a null handle is a `FILE_TYPE_CHAR` which works by chance for serial ports, but not for files:

Related: https://gitea.tendokyu.moe/TeamTofuShop/segatools/issues/116
Reviewed-on: https://gitea.tendokyu.moe/TeamTofuShop/capnhook/pulls/7
Co-authored-by: kyoubate-haruka <46010460+kyoubate-haruka@users.noreply.github.com>
Co-committed-by: kyoubate-haruka <46010460+kyoubate-haruka@users.noreply.github.com>

hook/iohook.c

@@ -35,6 +35,7 @@ static HRESULT iohook_invoke_real_write(struct irp *irp);
 static HRESULT iohook_invoke_real_seek(struct irp *irp);
 static HRESULT iohook_invoke_real_fsync(struct irp *irp);
 static HRESULT iohook_invoke_real_ioctl(struct irp *irp);
+static HRESULT iohook_invoke_real_ftype(struct irp *irp);
 
 /* API hooks. We take some liberties with function signatures here (e.g.
    stdint.h types instead of DWORD and LARGE_INTEGER et al). */
@@ -97,6 +98,8 @@ static BOOL WINAPI iohook_DeviceIoControl(
         uint32_t *lpBytesReturned,
         OVERLAPPED *lpOverlapped);
 
+static DWORD WINAPI iohook_GetFileType(HANDLE hFile);
+
 /* Links */
 
 static BOOL (WINAPI *next_CloseHandle)(HANDLE fd);
@@ -157,6 +160,8 @@ static BOOL (WINAPI *next_SetFilePointerEx)(
 
 static BOOL (WINAPI *next_FlushFileBuffers)(HANDLE fd);
 
+static DWORD (WINAPI *next_GetFileType)(HANDLE hFile);
+
 /* Hook symbol table */
 
 static const struct hook_symbol iohook_kernel32_syms[] = {
@@ -196,6 +201,10 @@ static const struct hook_symbol iohook_kernel32_syms[] = {
         .name   = "FlushFileBuffers",
         .patch  = iohook_FlushFileBuffers,
         .link   = (void *) &next_FlushFileBuffers,
+    }, {
+        .name   = "GetFileType",
+        .patch  = iohook_GetFileType,
+        .link   = (void *) &next_GetFileType,
     },
 };
 
@@ -207,6 +216,7 @@ static const iohook_fn_t iohook_real_handlers[] = {
     [IRP_OP_SEEK]   = iohook_invoke_real_seek,
     [IRP_OP_FSYNC]  = iohook_invoke_real_fsync,
     [IRP_OP_IOCTL]  = iohook_invoke_real_ioctl,
+    [IRP_OP_FTYPE]  = iohook_invoke_real_ftype,
 };
 
 static bool iohook_initted;
@@ -587,6 +597,18 @@ static HRESULT iohook_invoke_real_ioctl(struct irp *irp)
     return S_OK;
 }
 
+static HRESULT iohook_invoke_real_ftype(struct irp *irp)
+{
+    irp->file_type = next_GetFileType(irp->fd);
+    uint32_t err = GetLastError();
+
+    if (irp->file_type == FILE_TYPE_UNKNOWN && err != ERROR_SUCCESS) {
+        return HRESULT_FROM_WIN32(err);
+    }
+
+    return S_OK;
+}
+
 static HANDLE WINAPI iohook_CreateFileA(
         const char *lpFileName,
         uint32_t dwDesiredAccess,
@@ -980,3 +1002,28 @@ static BOOL WINAPI iohook_DeviceIoControl(
             lpOverlapped,
             irp.read.pos);
 }
+
+static DWORD WINAPI iohook_GetFileType(HANDLE hFile){
+    struct irp irp;
+    HRESULT hr;
+
+    if (hFile == NULL || hFile == INVALID_HANDLE_VALUE) {
+        SetLastError(ERROR_INVALID_PARAMETER);
+        return FILE_TYPE_UNKNOWN;
+    }
+
+    memset(&irp, 0, sizeof(irp));
+    irp.op = IRP_OP_FTYPE;
+    irp.fd = hFile;
+    irp.file_type = FILE_TYPE_UNKNOWN;
+
+    hr = iohook_invoke_next(&irp);
+
+    if (FAILED(hr)) {
+        return hr_propagate_win32(hr, FALSE);
+    }
+
+    SetLastError(ERROR_SUCCESS);
+
+    return irp.file_type;
+}

hook/iohook.h

@@ -15,6 +15,7 @@ enum irp_op {
     IRP_OP_IOCTL,
     IRP_OP_FSYNC,
     IRP_OP_SEEK,
+    IRP_OP_FTYPE
 };
 
 struct irp {
@@ -35,6 +36,7 @@ struct irp {
     uint32_t seek_origin;
     int64_t seek_offset;
     uint64_t seek_pos;
+    uint32_t file_type;
 };
 
 typedef HRESULT (*iohook_fn_t)(struct irp *irp);