.gitlab-ci.yml

@@ -1,136 +0,0 @@
-#
-# This pipeline requires packages to be switched on under the repository settings. Otherwise, you will 403s when
-# uploading to the package repo is triggered.
-#
-# Variables to setup in GitLab CI/CD settings of the project
-# 
-# The variables with BASE64 postfixes need to contain the base64 encoded data. Otherwise, masking
-# in GitLab won't work due to not matching their pre-defined regex 
-#
-# CI_PIGSTALL_DATA_PREFIX_BASE64
-# CI_PIGSTALL_LINK_BASE64
-# CI_PIGSTALL_PHP_SESSION_ID
-# CI_PIGSTALL_SESSION_BASE64
-# 
-# CI_TOOLS_UPLOAD_KEY
-# CI_TOOLS_UPLOAD_URL
-# CI_TOOLS_URL
-
-image: docker:stable
-
-variables:
-    DOCKER_TLS_CERTDIR: "/certs"
-    DIST_PACKAGE_RELATIVE_PATH: "build/docker/bemanitools.zip"
-    PACKAGE_REGISTRY_URL: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/bemanitools"
-
-services:
-  - docker:19.03.8-dind
-
-stages:
-  - build
-  - upload
-  - release
-
-build:
-  stage: build
-  only:
-    refs:
-      - master
-      - tags
-  before_script:
-    - apk update && apk add make > /dev/null
-  script:
-    - make build-docker
-  artifacts:
-    paths:
-      - build
-    expire_in: 1 week
-
-upload-package-registry:
-  stage: upload
-  image: curlimages/curl:latest
-  only:
-    refs:
-      - master
-      - tags
-  dependencies:
-    - build
-  script:
-    - |
-      if [ "${CI_COMMIT_TAG}" ]; then
-        version="${CI_COMMIT_TAG}"
-      else
-        version="${CI_COMMIT_SHORT_SHA}"
-      fi
-    - |
-      curl \
-        --silent \
-        --fail \
-        --show-error \
-        --header "JOB-TOKEN: ${CI_JOB_TOKEN}" \
-        --upload-file "${DIST_PACKAGE_RELATIVE_PATH}" \
-        $PACKAGE_REGISTRY_URL/${version}/bemanitools.zip
-
-upload-tools-page:
-  stage: upload
-  image: curlimages/curl:latest
-  only:
-    refs:
-      - tags
-  dependencies:
-    - build
-  script:
-    - |
-      curl \
-        --silent \
-        --fail \
-        --show-error \
-        --connect-timeout 5 \
-        --max-time 10 \
-        --retry 5 \
-        -F "key=${CI_TOOLS_UPLOAD_KEY}" \
-        -F "filename=${CI_PROJECT_NAME}-v${CI_COMMIT_TAG}.zip" \
-        -F "file=@${DIST_PACKAGE_RELATIVE_PATH}" \
-        ${CI_TOOLS_UPLOAD_URL}
-
-release-gitlab:
-  stage: release
-  image: registry.gitlab.com/gitlab-org/release-cli:v0.8.0
-  only:
-    refs:
-      - tags
-  script:
-    - version="$CI_COMMIT_TAG"
-    - release_message="$(scripts/ci/create-release-message.sh "${version}" < CHANGELOG.md)"
-    - |
-      release-cli create \
-        --name "bemanitools ${version}" \
-        --description="${release_message}" \
-        --tag-name ${version} \
-        --assets-link "{\"name\":\"Distribution binaries\",\"url\":\"${PACKAGE_REGISTRY_URL}/${version}/bemanitools.zip\"}"
-
-release-pigstall:
-  stage: release
-  image: curlimages/curl:latest
-  only:
-    refs:
-      - tags
-  script:
-    - version="${CI_COMMIT_TAG}"
-    - changelog_excerpt="$(scripts/ci/create-release-message.sh "${version}" < CHANGELOG.md)"
-    - release_message="$(printf "bemanitools ${version} released\n${CI_TOOLS_URL}/bemanitools-v${version}.zip\n${changelog_excerpt}")"
-    - session="$(echo "$CI_PIGSTALL_SESSION_BASE64" | base64 -d)"
-    - data_prefix="$(echo "$CI_PIGSTALL_DATA_PREFIX_BASE64" | base64 -d)"
-    - link="$(echo "$CI_PIGSTALL_LINK_BASE64" | base64 -d)"
-    - |
-      curl \
-        --silent \
-        --fail \
-        --connect-timeout 5 \
-        --max-time 10 \
-        --retry 5 \
-        --show-error \
-        -H 'Content-Type: application/x-www-form-urlencoded' \
-        -H "Cookie: PHPSESSID=$CI_PIGSTALL_PHP_SESSION_ID; session=${session}" \
-        --data-raw "${data_prefix}&body=${release_message}" \
-        "${link}"

CHANGELOG.md

@@ -4,8 +4,14 @@ given in tags. The pipeline will pick this up and cuts out the relevant section
 
 ## 5.46
 ### Features
+* feat(ddrio): Wrapper/shim library to drive another ddrio in a dedicated IO thread. Improves performance for highly IO
+  bound ddrio implementations, e.g. ddrio-p3io
+* feat(iidxiotest): bi2a-iidx support
 
 ### Fixes
+* fix(jb03/04): Missing XML in gamestart script
+* fix(jbhook1): Rotate error message box when screen is rotated
+* fix(jbhook1): Improve/fix automatic config/nvram fs initialization
 
 ## 5.45
 ### Features

Module.mk

@@ -104,6 +104,7 @@ include src/main/d3d9exhook/Module.mk
 include src/main/ddrhook-util/Module.mk
 include src/main/ddrhook1/Module.mk
 include src/main/ddrhook2/Module.mk
+include src/main/ddrio-async/Module.mk
 include src/main/ddrio-p3io/Module.mk
 include src/main/ddrio-mm/Module.mk
 include src/main/ddrio-smx/Module.mk
@@ -730,6 +731,7 @@ $(zipdir)/ddr-16-x64.zip: \
 	$(V)zip -j $@ $^
 
 $(zipdir)/ddr-hwio-x86.zip: \
+		build/bin/indep-32/ddrio-async.dll \
 		build/bin/indep-32/ddrio-p3io.dll \
 		build/bin/indep-32/ddrio-mm.dll \
 		build/bin/indep-32/ddrio-smx.dll \
@@ -741,6 +743,7 @@ $(zipdir)/ddr-hwio-x86.zip: \
 	$(V)zip -j $@ $^
 
 $(zipdir)/ddr-hwio-x64.zip: \
+		build/bin/indep-64/ddrio-async.dll \
 		build/bin/indep-64/ddrio-p3io.dll \
 		build/bin/indep-64/ddrio-mm.dll \
 		build/bin/indep-64/ddrio-smx.dll \

README.md

@@ -1,9 +1,8 @@
 # Bemanitools 5
-[![pipeline status](https://dev.s-ul.net/djhackers/bemanitools/badges/master/pipeline.svg)](https://dev.s-ul.net/djhackers/bemanitools/commits/master)
 
-Version: 5.46
+Version: `5.46`
 
-[Release history](CHANGELOG.md)
+[Changelog](CHANGELOG.md)
 
 A collection of tools to run [various Bemani arcade games](#supported-games).
 

doc/README.md

@@ -18,7 +18,8 @@ Table of contents:
   * [popnhook](popnhook/README.md): Documentation relevant to `popnhook` implementations
   * [sdvxhook](sdvxhook/README.md): Documentation relevant to `sdvxhook` implementations
 * Development
-  * [API](api.md): Available APIs for BT5 and instructions how to use them
+  * [API](api.md): Available APIs and IO (hardware) implementations for BT5 and instructions how
+    to use them
   * [Architecture](architecture.md): Outline of BT5's architecture, how things are designed and why
   * [Development](development.md): Development environment, building, releasing, etc.
   * [Developer documentation](dev/README.md): Various lose documentation/notes by developers

doc/api.md

@@ -25,6 +25,8 @@ The following implementations are already shipped with BT5.
     * [ddrio-p3io.dll](ddrhook/ddrio-p3io.md): DDR P3IO (Dragon PCB) + EXTIO hardware
     * ddrio-mm.dll: Minimaid hardware
     * [ddrio-smx.dll](ddrhook/ddrio-smx.md): StepManiaX platforms
+    * [ddrio-async](ddrhook/ddrio-async.md): Wrapper/shim library to drive another ddrio in
+      a dedicated IO thread
 * Beatmania IIDX
     * iidxio.dll (default): Keyboard, joystick and mouse input
     * [iidxio-bio2.dll](iidxhook/iidxio-bio2.md): BIO2 driver

doc/ddrhook/README.md

@@ -46,7 +46,10 @@ IO hardware you want to use:
 * `ddrio`: Default implementation supporting keyboard, mouse and USB
 game controllers
 * ddrio-mm: Support Minimaid custom interface
-* [ddrio-smx](ddrhook/ddrio-smx.md): Support for StepManiaX dance platforms
+* [ddrio-smx](ddrio-smx.md): Support for StepManiaX dance platforms
+* [ddrio-p3io](ddrio-p3io.md): P3IO + EXTIO driver implementation
+* [ddrio-async](ddrio-async.md): Wrapper/shim library to drive another ddrio in
+  a dedicated IO thread
 
 ## Unicorntail
 

doc/ddrhook/ddrio-async.md

@@ -0,0 +1,36 @@
+# Asynchronous proxy wrapper for other ddrio implementations
+
+This implementation of the Bemanitools API is not implementing support for any
+specific IO hardware. It is a proxy/shim library that loads another ddrio
+library, e.g. ddrio-p3io, and drives the entire backend in a dedicated IO
+thread. By implementing this behind the ddrio API, it is fully transparent to
+any existing application using it.
+
+The main benefit is the improved IO polling performance depending on how
+expensive the synchronous calls of the actual hardware are. For example,
+*ddrio-p3io* has very expensive write calls with ~12 ms duration while read
+calls take ~4 ms. Therefore, a full update cycle is already about as costly
+as rendering an entire frame (at 60 fps).
+
+## Setup
+
+For hook libraries, i.e. ddrhookX, but likely applicable to 3rd party
+applications (consolidate their manuals).
+
+* Have `ddrio-async.dll` in the same folder as your `ddrhookX.dll`
+* Rename `ddrio-async.dll` to `ddrio.dll`
+* Pick another ddrio library as the backend of your choice, e.g. `ddrio-p3io.dl`
+  and put it next to the async `ddrio.dll`
+* Rename it to `ddrio-async-child.dll`, ddrio-async is looking for that filename
+  in the same folder
+* Ensure that your `gamestart.bat` actually injects the appropriate `ddrhook.dll`, for example:
+
+```bat
+inject ddrhook1.dll ddr.exe ...*
+```
+
+or
+
+```bat
+launcher -K ddrhook2.dll arkmdxp3.dll ...*
+```

doc/release-process.md

@@ -1,18 +1,24 @@
 # Release process
 
-For bemanitools maintainers, steps for the release process:
+For maintainers, steps for the release process:
 
 1. Update the corresponding section in the [changelog](CHANGELOG.md) with bullet points reflecting
-major features merged into master since the previous release
+   major features merged into master since the previous release
 1. Make sure you have the latest `master` state pulled to your local copy
 1. Tag the release with the next release number which also be found at the
-[top of the readme](#bemanitools-5) as it always reflects the current version beinged worked on
-on the `master` branch , e.g. 5.35: `git tag 5.35`
+   [top of the readme](../README.md) as it always reflects the current version being worked on
+   on the `master` branch , e.g. 0.04: `git tag 5.35`
 1. Push the tag to upstream: `git push origin 5.35`
-1. The [build pipeline](https://dev.s-ul.net/djhackers/bemanitools/-/pipelines) should start
-automatically once the tag is pushed including the steps `build` and `upload-release`
-1. Once completed successfully, the release is uploaded
-1. Take the changelog of the published version and notify the pigs in the stall about it:
+1. The [build pipeline](https://github.com/djhackersdev/bemanitools/actions) should start
+   automatically once the tag is pushed including the steps `build` and `publish-release`
+1. Once completed successfully, the release is published on the
+   [releases page](https://github.com/djhackersdev/bemanitools/releases/)
+1. Edit the latest release
+1. Copy-pate the change log section of the respective version into the description
+1. Un-check *Set as pre-release*
+1. Check *Set as the latest release*
+1. Click *Update release*
+1. Notify any other channels, e.g. the pigs in the stall, about the latest release:
     1. New post in thread
         ```
         <insert version here> released: <direct link to published version>
@@ -23,7 +29,7 @@ automatically once the tag is pushed including the steps `build` and `upload-rel
     1. Update the OP title by bumping the version number in it
     1. Update the OP post by extending it accordingly (see previous entries)
 
-1. Bump the version number at the [top of this readme](#bemanitools-5) and add a new empty section
-in the [changelog](CHANGELOG.md) with the new version number as title. Commit changes and push to
+1. Bump the version number at the [top of the readme](../README.md) and add a new empty section
+in the [changelog](../CHANGELOG.md) with the new version number as title. Commit changes and push to
 `master` branch
-1. Continue developing and merging MRs until you decide its time for another release
+1. Continue developing and merging PRs until you decide its time for another release

src/main/ddrio-async/Module.mk

@@ -0,0 +1,9 @@
+dlls		+= ddrio-async
+
+ldflags_ddrio-async:= \
+
+libs_ddrio-async	:= \
+	util \
+
+src_ddrio-async	:= \
+	ddrio.c \

src/main/ddrio-async/ddrio-async.def

@@ -0,0 +1,11 @@
+LIBRARY ddrio-async
+
+EXPORTS
+    ddr_io_set_loggers
+    ddr_io_fini
+    ddr_io_init
+    ddr_io_read_pad
+    ddr_io_set_lights_extio
+    ddr_io_set_lights_p3io
+    ddr_io_set_lights_hdxs_panel
+    ddr_io_set_lights_hdxs_rgb

src/main/ddrio-async/ddrio.c

@@ -0,0 +1,268 @@
+#define LOG_MODULE "ddrio-async"
+
+#include <windows.h>
+
+#include <stdatomic.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <stdio.h>
+
+#include "bemanitools/ddrio.h"
+
+#include "util/log.h"
+#include "util/thread.h"
+#include "util/time.h"
+
+typedef void (*ddr_io_set_loggers_t)(
+    log_formatter_t misc,
+    log_formatter_t info,
+    log_formatter_t warning,
+    log_formatter_t fatal);
+typedef bool (*ddr_io_init_t)(
+    thread_create_t thread_create,
+    thread_join_t thread_join,
+    thread_destroy_t thread_destroy);
+typedef uint32_t (*ddr_io_read_pad_t)(void);
+typedef void (*ddr_io_set_lights_extio_t)(uint32_t extio_lights);
+typedef void (*ddr_io_set_lights_p3io_t)(uint32_t p3io_lights);
+typedef void (*ddr_io_set_lights_hdxs_panel_t)(uint32_t hdxs_lights);
+typedef void (*ddr_io_set_lights_hdxs_rgb_t)(uint8_t idx, uint8_t r, uint8_t g, uint8_t b);
+typedef void (*ddr_io_fini_t)(void);
+
+static HMODULE _child_ddr_io_module;
+
+static ddr_io_set_loggers_t _child_ddr_io_set_loggers;
+static ddr_io_init_t _child_ddr_io_init;
+static ddr_io_read_pad_t _child_ddr_io_read_pad;
+static ddr_io_set_lights_extio_t _child_ddr_io_set_lights_extio;
+static ddr_io_set_lights_p3io_t _child_ddr_io_set_lights_p3io;
+static ddr_io_set_lights_hdxs_panel_t _child_ddr_io_set_lights_hdxs_panel;
+static ddr_io_set_lights_hdxs_rgb_t _child_ddr_io_set_lights_hdxs_rgb;
+static ddr_io_fini_t _child_ddr_io_fini;
+
+static log_formatter_t _log_formatter_misc;
+static log_formatter_t _log_formatter_info;
+static log_formatter_t _log_formatter_warning;
+static log_formatter_t _log_formatter_fatal;
+
+static _Atomic(bool) _io_thread_proc_loop;
+static _Atomic(bool) _io_thread_proc_running;
+
+static _Atomic(uint32_t) _child_ddr_io_data_pad;
+static _Atomic(uint32_t) _child_ddr_io_data_extio_lights;
+static _Atomic(uint32_t) _child_ddr_io_data_p3io_lights;
+
+static int _io_thread_proc(void *ctx)
+{
+    uint64_t time_start;
+    uint64_t time_end;
+    uint64_t loop_counter;
+    uint64_t total_time;
+
+    uint32_t prev_child_ddr_io_data_extio_lights;
+    uint32_t prev_child_ddr_io_data_p3io_lights;
+
+    uint32_t local_tmp;
+
+    atomic_store_explicit(
+            &_io_thread_proc_running,
+            true,
+            memory_order_seq_cst);
+
+    log_info("IO thread running");
+
+    prev_child_ddr_io_data_extio_lights = atomic_load_explicit(
+        &_child_ddr_io_data_extio_lights, memory_order_seq_cst);
+    prev_child_ddr_io_data_p3io_lights = atomic_load_explicit(
+        &_child_ddr_io_data_p3io_lights, memory_order_seq_cst);
+
+    time_start = time_get_counter();
+    loop_counter = 0;
+
+    while (atomic_load_explicit(&_io_thread_proc_loop, memory_order_seq_cst)) {
+        local_tmp = _child_ddr_io_read_pad();
+
+        atomic_store_explicit(
+            &_child_ddr_io_data_pad,
+            local_tmp,
+            memory_order_relaxed);
+
+        // Only update outputs when they change gives this loop a major performance boost
+        // The write calls on a p3io for the outputs are very expensive (~12 ms) as they
+        // are executed over the ACIO protocol compared to only the input read
+        // calls (~4 ms) which have a dedicated IOCTL call/endpoint
+
+        local_tmp = atomic_load_explicit(
+            &_child_ddr_io_data_extio_lights,
+            memory_order_relaxed);
+
+        if (local_tmp != prev_child_ddr_io_data_extio_lights) {
+            _child_ddr_io_set_lights_extio(local_tmp);
+            prev_child_ddr_io_data_extio_lights = local_tmp;
+        }
+
+        local_tmp = atomic_load_explicit(
+            &_child_ddr_io_data_p3io_lights,
+            memory_order_relaxed);
+        
+        if (local_tmp != prev_child_ddr_io_data_p3io_lights) {
+            _child_ddr_io_set_lights_p3io(local_tmp);
+            prev_child_ddr_io_data_p3io_lights = local_tmp;
+        }
+
+        // Don't hog the CPU
+        SwitchToThread();
+
+        loop_counter++;
+    }
+
+    time_end = time_get_counter();
+    total_time = time_get_elapsed_us(time_end - time_start);
+
+    log_info("IO thread performance: total iterations %lld, avg. loop cycle time %f us",
+        loop_counter, ((double) total_time) / loop_counter);
+
+    atomic_store_explicit(
+            &_io_thread_proc_running,
+            false,
+            memory_order_seq_cst);
+
+    log_info("IO thread shut down");
+
+    return 0;
+}
+
+static void* _load_function(HMODULE module, const char* name)
+{
+    void* ptr;
+
+    ptr = GetProcAddress(module, name);
+
+    if (ptr == NULL) {
+        log_fatal("Could not find function %s in ddr3io child library", name);
+    }
+
+    return ptr;
+}
+
+void ddr_io_set_loggers(
+    log_formatter_t misc,
+    log_formatter_t info,
+    log_formatter_t warning,
+    log_formatter_t fatal)
+{
+    _log_formatter_misc = misc;
+    _log_formatter_info = info;
+    _log_formatter_warning = warning;
+    _log_formatter_fatal = fatal;
+
+    log_to_external(misc, info, warning, fatal);
+}
+
+bool ddr_io_init(
+    thread_create_t thread_create,
+    thread_join_t thread_join,
+    thread_destroy_t thread_destroy)
+{
+    log_info("Loading ddrio-async-child.dll as child ddrio library...");
+
+    _child_ddr_io_module = LoadLibraryA("ddrio-async-child.dll");
+
+    if (_child_ddr_io_module == NULL) {
+        log_warning("Loading ddrio-async-child.dll failed");
+        return false;
+    }
+
+    _child_ddr_io_set_loggers = _load_function(_child_ddr_io_module, "ddr_io_set_loggers");
+    _child_ddr_io_init = _load_function(_child_ddr_io_module, "ddr_io_init");
+    _child_ddr_io_read_pad = _load_function(_child_ddr_io_module, "ddr_io_read_pad");
+    _child_ddr_io_set_lights_extio = _load_function(_child_ddr_io_module, "ddr_io_set_lights_extio");
+    _child_ddr_io_set_lights_p3io = _load_function(_child_ddr_io_module, "ddr_io_set_lights_p3io");
+    _child_ddr_io_set_lights_hdxs_panel = _load_function(_child_ddr_io_module, "ddr_io_set_lights_hdxs_panel");
+    _child_ddr_io_set_lights_hdxs_rgb = _load_function(_child_ddr_io_module, "ddr_io_set_lights_hdxs_rgb");
+    _child_ddr_io_fini = _load_function(_child_ddr_io_module, "ddr_io_fini");
+
+    _child_ddr_io_set_loggers(
+        _log_formatter_misc,
+        _log_formatter_info,
+        _log_formatter_warning,
+        _log_formatter_fatal);
+
+    log_info("Calling child ddr_io_init...");
+
+    if (!_child_ddr_io_init(thread_create, thread_join, thread_destroy)) {
+        log_warning("Child ddr_io_init failed");
+        FreeLibrary(_child_ddr_io_module);
+
+        return false;
+    }
+
+    atomic_store_explicit(
+            &_io_thread_proc_loop,
+            true,
+            memory_order_seq_cst);
+
+    if (!thread_create(_io_thread_proc, NULL, 16384, 0)) {
+        log_warning("Creating IO thread failed");
+
+        _child_ddr_io_fini();
+        FreeLibrary(_child_ddr_io_module);
+
+        return false;
+    }
+
+    return true;
+}
+
+uint32_t ddr_io_read_pad(void)
+{
+    return atomic_load_explicit(&_child_ddr_io_data_pad, memory_order_relaxed);
+}
+
+void ddr_io_set_lights_extio(uint32_t extio_lights)
+{
+    atomic_store_explicit(
+            &_child_ddr_io_data_extio_lights,
+            extio_lights,
+            memory_order_relaxed);
+}
+
+void ddr_io_set_lights_p3io(uint32_t p3io_lights)
+{
+    atomic_store_explicit(
+            &_child_ddr_io_data_p3io_lights,
+            p3io_lights,
+            memory_order_relaxed);
+}
+
+void ddr_io_set_lights_hdxs_panel(uint32_t lights)
+{
+    // Not implemented for now
+}
+
+void ddr_io_set_lights_hdxs_rgb(uint8_t idx, uint8_t r, uint8_t g, uint8_t b)
+{
+    // Not implemented for now
+}
+
+void ddr_io_fini(void)
+{
+    atomic_store_explicit(
+            &_io_thread_proc_loop,
+            false,
+            memory_order_seq_cst);
+
+    log_info("Shutting down IO thread and waiting for it to finish...");
+
+    while (atomic_load_explicit(&_io_thread_proc_running, memory_order_seq_cst)) {
+        Sleep(1);
+    }
+
+    log_info("IO thread finished");
+
+    _child_ddr_io_fini();
+
+    FreeLibrary(_child_ddr_io_module);
+}