idz: Implement multi-page Garage fetch

package-lock.json

@@ -192,6 +192,14 @@
         }
       }
     },
+    "@decafcode/sqlite": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/@decafcode/sqlite/-/sqlite-2.0.0.tgz",
+      "integrity": "sha512-CvDlc1G2ExETrRXVjjHRZmZiDCY56pw3xU3pxWnOzkQ4j4bi+DUAYlQbFKwpb4BzRdFo84e7H+T2kmoEBf90JA==",
+      "requires": {
+        "node-gyp-build": "^4.2.0"
+      }
+    },
     "@jest/console": {
       "version": "24.9.0",
       "resolved": "https://registry.npmjs.org/@jest/console/-/console-24.9.0.tgz",
@@ -401,15 +409,6 @@
         "@babel/types": "^7.3.0"
       }
     },
-    "@types/better-sqlite3": {
-      "version": "5.4.0",
-      "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-5.4.0.tgz",
-      "integrity": "sha512-nzm7lJ7l3jBmGUbtkL8cdOMhPkN6Pw2IM+b0V7iIKba+YKiLrjkIy7vVLsBIVnd7+lgzBzrHsXZxCaFTcmw5Ow==",
-      "dev": true,
-      "requires": {
-        "@types/integer": "*"
-      }
-    },
     "@types/body-parser": {
       "version": "1.17.1",
       "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.17.1.tgz",
@@ -474,12 +473,6 @@
         "@types/range-parser": "*"
       }
     },
-    "@types/integer": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/@types/integer/-/integer-1.0.0.tgz",
-      "integrity": "sha512-3viiRKLoSP2Qr78nMoQjkDc0fan4BgmpOyV1+1gKjE8wWXo3QQ78WItO6f9WuBf3qe3ymDYhM65oqHTOZ0rFxw==",
-      "dev": true
-    },
     "@types/istanbul-lib-coverage": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.1.tgz",
@@ -889,15 +882,6 @@
         "tweetnacl": "^0.14.3"
       }
     },
-    "better-sqlite3": {
-      "version": "5.4.3",
-      "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-5.4.3.tgz",
-      "integrity": "sha512-fPp+8f363qQIhuhLyjI4bu657J/FfMtgiiHKfaTsj3RWDkHlWC1yT7c6kHZDnBxzQVoAINuzg553qKmZ4F1rEw==",
-      "requires": {
-        "integer": "^2.1.0",
-        "tar": "^4.4.10"
-      }
-    },
     "body-parser": {
       "version": "1.19.0",
       "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz",
@@ -1104,11 +1088,6 @@
         }
       }
     },
-    "chownr": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.3.tgz",
-      "integrity": "sha512-i70fVHhmV3DtTl6nqvZOnIjbY0Pe4kAUjwHj8z0zAdgBtYrJyYwLKCCuRBQ5ppkyL0AkN7HKRnETdmdp1zqNXw=="
-    },
     "ci-info": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz",
@@ -1983,14 +1962,6 @@
       "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
       "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac="
     },
-    "fs-minipass": {
-      "version": "1.2.7",
-      "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz",
-      "integrity": "sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==",
-      "requires": {
-        "minipass": "^2.6.0"
-      }
-    },
     "fs.realpath": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@@ -2770,11 +2741,6 @@
       "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
       "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
     },
-    "integer": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/integer/-/integer-2.1.0.tgz",
-      "integrity": "sha512-vBtiSgrEiNocWvvZX1RVfeOKa2mCHLZQ2p9nkQkQZ/BvEiY+6CcUz0eyjvIiewjJoeNidzg2I+tpPJvpyspL1w=="
-    },
     "invariant": {
       "version": "2.2.4",
       "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz",
@@ -3818,24 +3784,8 @@
     "minimist": {
       "version": "0.0.8",
       "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
-      "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
+      "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=",
-    },
+      "dev": true
-    "minipass": {
-      "version": "2.9.0",
-      "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.9.0.tgz",
-      "integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==",
-      "requires": {
-        "safe-buffer": "^5.1.2",
-        "yallist": "^3.0.0"
-      }
-    },
-    "minizlib": {
-      "version": "1.3.3",
-      "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.3.3.tgz",
-      "integrity": "sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==",
-      "requires": {
-        "minipass": "^2.9.0"
-      }
     },
     "mixin-deep": {
       "version": "1.3.2",
@@ -3862,6 +3812,7 @@
       "version": "0.5.1",
       "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
       "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
+      "dev": true,
       "requires": {
         "minimist": "0.0.8"
       }
@@ -3968,6 +3919,11 @@
         "lower-case": "^1.1.1"
       }
     },
+    "node-gyp-build": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.2.0.tgz",
+      "integrity": "sha512-4oiumOLhCDU9Rronz8PZ5S4IvT39H5+JEv/hps9V8s7RSLhsac0TCP78ulnHXOo8X1wdpPiTayGlM1jr4IbnaQ=="
+    },
     "node-int64": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz",
@@ -4634,7 +4590,8 @@
     "safe-buffer": {
       "version": "5.2.0",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz",
-      "integrity": "sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg=="
+      "integrity": "sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg==",
+      "dev": true
     },
     "safe-regex": {
       "version": "1.1.0",
@@ -5191,20 +5148,6 @@
       "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==",
       "dev": true
     },
-    "tar": {
-      "version": "4.4.13",
-      "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.13.tgz",
-      "integrity": "sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA==",
-      "requires": {
-        "chownr": "^1.1.1",
-        "fs-minipass": "^1.2.5",
-        "minipass": "^2.8.6",
-        "minizlib": "^1.2.1",
-        "mkdirp": "^0.5.0",
-        "safe-buffer": "^5.1.2",
-        "yallist": "^3.0.3"
-      }
-    },
     "test-exclude": {
       "version": "5.2.3",
       "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-5.2.3.tgz",
@@ -5650,11 +5593,6 @@
       "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==",
       "dev": true
     },
-    "yallist": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
-      "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
-    },
     "yargs": {
       "version": "13.3.0",
       "resolved": "https://registry.npmjs.org/yargs/-/yargs-13.3.0.tgz",

package.json

@@ -5,7 +5,7 @@
   "main": "src/index.js",
   "private": true,
   "dependencies": {
-    "better-sqlite3": "^5.4.3",
+    "@decafcode/sqlite": "^2.0.0",
     "compression": "^1.7.3",
     "date-fns": "^1.30.1",
     "debug": "^4.1.1",
@@ -20,7 +20,6 @@
     "supports-color": "^7.1.0"
   },
   "devDependencies": {
-    "@types/better-sqlite3": "^5.4.0",
     "@types/compression": "^0.0.36",
     "@types/debug": "^4.1.5",
     "@types/dotenv": "^6.1.1",

schema/init/cm.sql

@@ -145,6 +145,23 @@ create table "cm_user_data_ex" (
     "ext_long5" integer not null
 );
 
+create table "cm_user_duel_list" (
+    "id" integer primary key not null,
+    "profile_id" integer not null
+            references "cm_user_data"("id")
+            on delete cascade,
+    "duel_id" integer not null,
+    "progress" integer not null,
+    "point" integer not null,
+    "is_clear" boolean not null,
+    "last_play_date" text not null,
+    "param1" integer not null,
+    "param2" integer not null,
+    "param3" integer not null,
+    "param4" integer not null,
+    constraint "cm_user_duel_list_uq" unique ("profile_id", "duel_id")
+);
+
 create table "cm_user_character" (
     "id" integer primary key not null,
     "profile_id" integer not null

schema/migrate/M0013-cm-user-duel-list-fix.sql

@@ -0,0 +1,17 @@
+-- This was also not present in the db init scripts...
+create table if not exists "cm_user_duel_list" (
+    "id" integer primary key not null,
+    "profile_id" integer not null
+            references "cm_user_data"("id")
+            on delete cascade,
+    "duel_id" integer not null,
+    "progress" integer not null,
+    "point" integer not null,
+    "is_clear" boolean not null,
+    "last_play_date" text not null,
+    "param1" integer not null,
+    "param2" integer not null,
+    "param3" integer not null,
+    "param4" integer not null,
+    constraint "cm_user_duel_list_uq" unique ("profile_id", "duel_id")
+);

src/idz/handler/loadGarage.ts

@@ -10,6 +10,6 @@ export async function loadGarage(
 
   return {
     type: "load_garage_res",
-    cars: await w.car().loadAllCars(profileId),
+    cars: await w.car().loadCars(profileId, 10, req.fetchOffset),
   };
 }

src/idz/repo.ts

@@ -11,7 +11,11 @@ export type TeamSpec = Subtract<
 export interface CarRepository {
   countCars(profileId: Id<Model.Profile>): Promise<number>;
 
-  loadAllCars(profileId: Id<Model.Profile>): Promise<Model.Car[]>;
+  loadCars(
+    profileId: Id<Model.Profile>,
+    limit: number,
+    offset: number
+  ): Promise<Model.Car[]>;
 
   loadSelectedCar(profileId: Id<Model.Profile>): Promise<Model.Car>;
 

src/idz/sql/car.ts

@@ -40,11 +40,18 @@ export class SqlCarRepository implements CarRepository {
     return parseInt(row!.result);
   }
 
-  async loadAllCars(profileId: Id<Profile>): Promise<Car[]> {
+  async loadCars(
+    profileId: Id<Profile>,
+    limit: number,
+    offset: number
+  ): Promise<Car[]> {
     const loadSql = sql
       .select("c.*")
       .from("idz_car c")
-      .where("c.profile_id", profileId);
+      .where("c.profile_id", profileId)
+      .orderBy("selector asc")
+      .limit(limit)
+      .offset(offset);
 
     const rows = await this._txn.fetchRows(loadSql);
 

src/sql/sqlite.ts

@@ -1,14 +1,10 @@
-import Database from "better-sqlite3";
+import Database, { ResultRow } from "@decafcode/sqlite";
 import { randomBytes } from "crypto";
 import * as sql from "sql-bricks-postgres";
 
 import { DataSource, Row, Transaction } from "./api";
 import { Id } from "../model";
 
-type MixedRow = {
-  [key: string]: any;
-};
-
 // bless me father for i have sinned
 const fuFixup = new RegExp(" FOR UPDATE$");
 
@@ -41,7 +37,7 @@ function _preprocess(stmt: sql.Statement) {
   };
 }
 
-function _postprocess(obj: MixedRow): Row {
+function _postprocess(obj: ResultRow): Row {
   const result = {};
 
   for (const [k, v] of Object.entries(obj)) {
@@ -58,7 +54,7 @@ function _postprocess(obj: MixedRow): Row {
 }
 
 class SqliteTransaction implements Transaction {
-  constructor(private readonly _db: Database.Database) {}
+  constructor(private readonly _db: Database) {}
 
   generateId<T>(): Id<T> {
     const buf = randomBytes(8);
@@ -73,26 +69,43 @@ class SqliteTransaction implements Transaction {
 
   modify(stmt: sql.Statement): Promise<void> {
     const params = _preprocess(stmt);
+    const prepared = this._db.prepare(params.text);
 
-    this._db.prepare(params.text).run(...params.values);
+    try {
+      prepared.run(params.values);
+    } finally {
+      prepared.close();
+    }
 
     return Promise.resolve();
   }
 
   fetchRow(stmt: sql.SelectStatement): Promise<Row | undefined> {
     const params = _preprocess(stmt);
-    const raw = this._db.prepare(params.text).get(...params.values);
+    const prepared = this._db.prepare(params.text);
-    const result = raw && _postprocess(raw);
 
-    return Promise.resolve(result);
+    try {
+      const raw = prepared.one(params.values);
+      const result = raw && _postprocess(raw);
+
+      return Promise.resolve(result);
+    } finally {
+      prepared.close();
+    }
   }
 
   fetchRows(stmt: sql.SelectStatement): Promise<Row[]> {
     const params = _preprocess(stmt);
-    const raw = this._db.prepare(params.text).all(...params.values);
+    const prepared = this._db.prepare(params.text);
-    const result = raw.map(_postprocess);
 
-    return Promise.resolve(result);
+    try {
+      const raw = prepared.all(params.values);
+      const result = raw.map(_postprocess);
+
+      return Promise.resolve(result);
+    } finally {
+      prepared.close();
+    }
   }
 
   raw(sql: string): Promise<void> {
@@ -110,28 +123,32 @@ class SqliteDataSource implements DataSource {
   ): Promise<T> {
     const db = new Database(this._path);
 
-    db.defaultSafeIntegers();
-    db.prepare("pragma foreign_keys = on").run();
-    db.prepare("begin").run();
-
     try {
+      db.exec("begin");
+
       const txn = new SqliteTransaction(db);
       const result = await callback(txn);
 
-      db.prepare("commit").run();
+      db.exec("commit");
 
       return result;
     } catch (e) {
-      db.prepare("rollback").run();
+      db.exec("rollback");
 
       return Promise.reject(e);
+    } finally {
+      db.close();
     }
   }
 
   vacuum(): Promise<void> {
     const db = new Database(this._path);
 
-    db.prepare("vacuum").run();
+    try {
+      db.exec("vacuum");
+    } finally {
+      db.close();
+    }
 
     return Promise.resolve();
   }

start.bat

@@ -22,9 +22,6 @@ if not exist node_modules (
     echo The server's dependencies will now be installed.
     echo This process requires an internet connection and may take some time.
     echo.
-    echo NOTE: If install fails try running the following command as admin:
-    echo npm install -g windows-build-tools
-    echo.
 
     pause
     call npm install

types/sql-bricks-postgres/index.d.ts

@@ -7,7 +7,7 @@ declare module "sql-bricks-postgres" {
   }
 
   interface LimitClause extends sql.SelectStatement {
-    offset(value: number): sql.Statement;
+    offset(value: number): sql.SelectStatement;
   }
 
   export interface PgInsertStatement extends sql.InsertStatement {