From dcc41adeec0ada454d59865ea568deb5e386648b Mon Sep 17 00:00:00 2001 From: zk Date: Sun, 24 May 2026 13:06:13 +0100 Subject: [PATCH] feat: import timestops (#1574) * feat: import timestops * feat: and a page to edit it --- .../20260523120000_import_timestop.sql | 6 + .../users/UserImportTimestopsTab.tsx | 227 ++++++++++++++++++ .../pages/dashboard/users/UserImportsPage.tsx | 24 +- typescript/db/src/generated/index.ts | 1 + .../db/src/generated/public/PublicSchema.ts | 3 + .../framework/common/timestop.test.ts | 194 +++++++++++++++ .../score-import/framework/common/timestop.ts | 113 +++++++++ .../score-importing/score-import-main.ts | 17 ++ .../import-types/api/eag-iidx/parser.ts | 8 +- .../import-types/api/eag-sdvx/parser.ts | 8 +- .../import-types/api/flo-iidx/parser.ts | 8 +- .../import-types/api/flo-sdvx/parser.ts | 8 +- .../import-types/api/min-sdvx/parser.ts | 8 +- .../import-types/api/myt-chunithm/parser.ts | 12 +- .../import-types/api/myt-maimaidx/parser.ts | 12 +- .../import-types/api/myt-ongeki/parser.ts | 12 +- .../import-types/api/myt-wacca/parser.ts | 19 +- .../import-types/common/api-cg/parsers.ts | 59 +++-- .../common/api-kai/iidx/parser.ts | 19 +- .../common/api-kai/kai-timestop.test.ts | 96 ++++++++ .../common/api-kai/kai-timestop.ts | 33 +++ .../common/api-kai/sdvx/parser.ts | 19 +- .../server/src/server/router/api/v1/router.ts | 1 + .../server/src/server/router/api/v1/spec.ts | 28 +++ .../_userID/import-timestops/router.test.ts | 121 ++++++++++ .../users/_userID/import-timestops/router.ts | 84 +++++++ 26 files changed, 1085 insertions(+), 55 deletions(-) create mode 100644 db/migrations/20260523120000_import_timestop.sql create mode 100644 typescript/client/src/app/pages/dashboard/users/UserImportTimestopsTab.tsx create mode 100644 typescript/server/src/lib/score-import/framework/common/timestop.test.ts create mode 100644 typescript/server/src/lib/score-import/framework/common/timestop.ts create mode 100644 typescript/server/src/lib/score-import/import-types/common/api-kai/kai-timestop.test.ts create mode 100644 typescript/server/src/lib/score-import/import-types/common/api-kai/kai-timestop.ts create mode 100644 typescript/server/src/server/router/api/v1/users/_userID/import-timestops/router.test.ts create mode 100644 typescript/server/src/server/router/api/v1/users/_userID/import-timestops/router.ts diff --git a/db/migrations/20260523120000_import_timestop.sql b/db/migrations/20260523120000_import_timestop.sql new file mode 100644 index 000000000..9f8edb4e4 --- /dev/null +++ b/db/migrations/20260523120000_import_timestop.sql @@ -0,0 +1,6 @@ +CREATE TABLE import_timestop ( + user_id BIGINT NOT NULL REFERENCES account(id), + import_type IMPORT_TYPE NOT NULL, + last_score_time TIMESTAMPTZ NOT NULL, + PRIMARY KEY (user_id, import_type) +); diff --git a/typescript/client/src/app/pages/dashboard/users/UserImportTimestopsTab.tsx b/typescript/client/src/app/pages/dashboard/users/UserImportTimestopsTab.tsx new file mode 100644 index 000000000..6620fcd71 --- /dev/null +++ b/typescript/client/src/app/pages/dashboard/users/UserImportTimestopsTab.tsx @@ -0,0 +1,227 @@ +import Card from "#components/layout/page/Card"; +import ApiError from "#components/util/ApiError"; +import Loading from "#components/util/Loading"; +import useApiQuery from "#components/util/query/useApiQuery"; +import { APIFetchV1 } from "#util/api"; +import { FormatTime } from "#util/time"; +import { DateTime } from "luxon"; +import React, { useMemo, useState } from "react"; +import { Alert, Button, Form, Table } from "react-bootstrap"; +import { useQueryClient } from "react-query"; +import { type UserDocument } from "tachi-common"; + +type ImportTimestopEntry = { + importType: string; + lastScoreTime: number | null; +}; + +function toDatetimeLocalValue(ms: number | null): string { + if (ms === null) { + return ""; + } + + return DateTime.fromMillis(ms).toFormat("yyyy-MM-dd'T'HH:mm"); +} + +function fromDatetimeLocalValue(value: string): number | null { + if (!value) { + return null; + } + + const parsed = DateTime.fromFormat(value, "yyyy-MM-dd'T'HH:mm"); + + if (!parsed.isValid) { + return null; + } + + return parsed.toMillis(); +} + +export default function UserImportTimestopsTab({ reqUser }: { reqUser: UserDocument }) { + const queryKey = `/users/${reqUser.id}/import-timestops`; + const queryClient = useQueryClient(); + const { data, error } = useApiQuery<{ timestops: Array }>(queryKey); + const [busyImportType, setBusyImportType] = useState(null); + const [draftTimes, setDraftTimes] = useState>({}); + const [message, setMessage] = useState(null); + + const timestops = useMemo(() => { + if (!data) { + return []; + } + + return [...data.timestops].sort((a, b) => a.importType.localeCompare(b.importType)); + }, [data]); + + if (error) { + return ; + } + + if (!data) { + return ; + } + + const invalidate = async () => { + await queryClient.invalidateQueries([queryKey]); + }; + + const onReset = async (importType: string) => { + if ( + !window.confirm( + `Reset the timestop for ${importType}? The next import will fetch all available scores again.`, + ) + ) { + return; + } + + setBusyImportType(importType); + setMessage(null); + + const res = await APIFetchV1( + `/users/${reqUser.id}/import-timestops`, + { + body: JSON.stringify({ importType }), + headers: { "Content-Type": "application/json" }, + method: "DELETE", + }, + true, + true, + ); + + setBusyImportType(null); + + if (!res.success) { + setMessage(res.description); + return; + } + + setDraftTimes((prev) => { + const next = { ...prev }; + delete next[importType]; + return next; + }); + setMessage(`Reset timestop for ${importType}.`); + await invalidate(); + }; + + const onSave = async (importType: string) => { + const draft = + draftTimes[importType] ?? + toDatetimeLocalValue( + timestops.find((entry) => entry.importType === importType)?.lastScoreTime ?? null, + ); + const lastScoreTime = fromDatetimeLocalValue(draft); + + if (lastScoreTime === null) { + setMessage("Enter a valid date and time."); + return; + } + + setBusyImportType(importType); + setMessage(null); + + const res = await APIFetchV1( + `/users/${reqUser.id}/import-timestops`, + { + body: JSON.stringify({ importType, lastScoreTime }), + headers: { "Content-Type": "application/json" }, + method: "PUT", + }, + true, + true, + ); + + setBusyImportType(null); + + if (!res.success) { + setMessage(res.description); + return; + } + + setMessage(`Updated timestop for ${importType}.`); + await invalidate(); + }; + + return ( +
+ +

+ API imports track the newest score they have already imported. On the next + import, anything at or before that timestamp is skipped. +

+

+ Reset a timestop to re-fetch all scores, or set a custom cursor if you need to + re-import from a specific point in time. +

+ + + {message && {message}} + + + + + + + + + + + {timestops.map((entry) => { + const draft = + draftTimes[entry.importType] ?? + toDatetimeLocalValue(entry.lastScoreTime); + const isBusy = busyImportType === entry.importType; + + return ( + + + + + + + ); + })} + +
Import typeCurrent cursorSet cursor +
+ {entry.importType} + + {entry.lastScoreTime === null ? ( + Not set + ) : ( + FormatTime(entry.lastScoreTime) + )} + + + setDraftTimes((prev) => ({ + ...prev, + [entry.importType]: event.target.value, + })) + } + type="datetime-local" + value={draft} + /> + + + +
+
+ ); +} diff --git a/typescript/client/src/app/pages/dashboard/users/UserImportsPage.tsx b/typescript/client/src/app/pages/dashboard/users/UserImportsPage.tsx index fd4707c73..68ea2662e 100644 --- a/typescript/client/src/app/pages/dashboard/users/UserImportsPage.tsx +++ b/typescript/client/src/app/pages/dashboard/users/UserImportsPage.tsx @@ -9,9 +9,11 @@ import useApiQuery from "#components/util/query/useApiQuery"; import SelectLinkButton from "#components/util/SelectLinkButton"; import { type FailedImportDataset, type ImportDataset } from "#types/tables"; import React, { useMemo, useState } from "react"; -import { Route, Switch } from "react-router-dom"; +import { Route, Switch, useLocation } from "react-router-dom"; import { type ImportDocument, type ImportTrackerFailed, type UserDocument } from "tachi-common"; +import UserImportTimestopsTab from "./UserImportTimestopsTab"; + export default function UserImportsPage({ reqUser }: { reqUser: UserDocument }) { useSetSubheader( ["Users", reqUser.username, "Imports"], @@ -37,6 +39,9 @@ export default function UserImportsPage({ reqUser }: { reqUser: UserDocument }) return p; }, [userIntent, importType]); + const location = useLocation(); + const onTimestopsTab = location.pathname.endsWith("/imports/timestops"); + return ( <>
@@ -46,10 +51,19 @@ export default function UserImportsPage({ reqUser }: { reqUser: UserDocument }) Recent Failed Imports + + Timestops +
- - + {!onTimestopsTab && ( + <> + + + + )} @@ -59,6 +73,10 @@ export default function UserImportsPage({ reqUser }: { reqUser: UserDocument }) + + + + ); diff --git a/typescript/db/src/generated/index.ts b/typescript/db/src/generated/index.ts index e6d6296f9..6cf221637 100644 --- a/typescript/db/src/generated/index.ts +++ b/typescript/db/src/generated/index.ts @@ -5,6 +5,7 @@ export { type priv_svc_cg_card_info_service, type default as PrivSvcCgCardInfoTa export { type score_rederive_chart_id, type default as ScoreRederiveTable, type ScoreRederive, type NewScoreRederive, type ScoreRederiveUpdate } from './public/ScoreRederive'; export { type class_achievement_row_id, type default as ClassAchievementTable, type ClassAchievement, type NewClassAchievement, type ClassAchievementUpdate } from './public/ClassAchievement'; export { type account_id, type default as AccountTable, type Account, type NewAccount, type AccountUpdate } from './public/Account'; +export { type import_timestop_import_type, type default as ImportTimestopTable, type ImportTimestop, type NewImportTimestop, type ImportTimestopUpdate } from './public/ImportTimestop'; export { type import_tracker_import_id, type default as ImportTrackerTable, type ImportTracker, type NewImportTracker, type ImportTrackerUpdate } from './public/ImportTracker'; export { type default as SvcFerSettingsTable, type SvcFerSettings, type NewSvcFerSettings, type SvcFerSettingsUpdate } from './public/SvcFerSettings'; export { type default as InviteLockTable, type InviteLock, type NewInviteLock, type InviteLockUpdate } from './public/InviteLock'; diff --git a/typescript/db/src/generated/public/PublicSchema.ts b/typescript/db/src/generated/public/PublicSchema.ts index 6e40bfd69..c8cbc25d7 100644 --- a/typescript/db/src/generated/public/PublicSchema.ts +++ b/typescript/db/src/generated/public/PublicSchema.ts @@ -5,6 +5,7 @@ import type { default as PrivSvcCgCardInfoTable } from './PrivSvcCgCardInfo'; import type { default as ScoreRederiveTable } from './ScoreRederive'; import type { default as ClassAchievementTable } from './ClassAchievement'; import type { default as AccountTable } from './Account'; +import type { default as ImportTimestopTable } from './ImportTimestop'; import type { default as ImportTrackerTable } from './ImportTracker'; import type { default as SvcFerSettingsTable } from './SvcFerSettings'; import type { default as InviteLockTable } from './InviteLock'; @@ -79,6 +80,8 @@ export default interface PublicSchema { account: AccountTable; + import_timestop: ImportTimestopTable; + import_tracker: ImportTrackerTable; svc_fer_settings: SvcFerSettingsTable; diff --git a/typescript/server/src/lib/score-import/framework/common/timestop.test.ts b/typescript/server/src/lib/score-import/framework/common/timestop.test.ts new file mode 100644 index 000000000..f99004b57 --- /dev/null +++ b/typescript/server/src/lib/score-import/framework/common/timestop.test.ts @@ -0,0 +1,194 @@ +import DB from "#services/pg/db"; +import { seedUser } from "#test-utils/pg-fixtures"; +import { beforeEach, describe, expect, it } from "vitest"; + +import { + DeleteImportTimestop, + GetImportTimestop, + ListImportTimestops, + SetImportTimestop, + SetImportTimestopManual, +} from "./timestop"; + +describe("GetImportTimestop", () => { + let userID: number; + + beforeEach(async () => { + ({ id: userID } = await seedUser()); + }); + + it("returns null when no row exists", async () => { + const result = await GetImportTimestop(userID, "api/eag-iidx"); + + expect(result).toBeNull(); + }); + + it("returns the stored date after SetImportTimestop", async () => { + const time = new Date("2025-03-15T12:00:00.000Z"); + + await SetImportTimestop(userID, "api/eag-iidx", time); + + const result = await GetImportTimestop(userID, "api/eag-iidx"); + + expect(result).not.toBeNull(); + expect(result!.getTime()).toBe(time.getTime()); + }); + + it("is scoped per import type", async () => { + const time = new Date("2025-01-01T00:00:00.000Z"); + + await SetImportTimestop(userID, "api/eag-iidx", time); + + expect(await GetImportTimestop(userID, "api/eag-sdvx")).toBeNull(); + }); + + it("is scoped per user", async () => { + const other = await seedUser({ username: "other_user" }); + const time = new Date("2025-01-01T00:00:00.000Z"); + + await SetImportTimestop(other.id, "api/eag-iidx", time); + + expect(await GetImportTimestop(userID, "api/eag-iidx")).toBeNull(); + }); +}); + +describe("SetImportTimestop", () => { + let userID: number; + + beforeEach(async () => { + ({ id: userID } = await seedUser()); + }); + + it("inserts a new row when none exists", async () => { + const time = new Date("2025-06-01T00:00:00.000Z"); + + await SetImportTimestop(userID, "api/flo-sdvx", time); + + const row = await DB.selectFrom("import_timestop") + .select(["import_timestop.last_score_time"]) + .where("import_timestop.user_id", "=", userID) + .where("import_timestop.import_type", "=", "api/flo-sdvx") + .executeTakeFirst(); + + expect(row).toBeDefined(); + expect(new Date(row!.last_score_time).getTime()).toBe(time.getTime()); + }); + + it("advances the cursor to a later timestamp", async () => { + const t1 = new Date("2025-01-01T00:00:00.000Z"); + const t2 = new Date("2025-06-01T00:00:00.000Z"); + + await SetImportTimestop(userID, "api/eag-iidx", t1); + await SetImportTimestop(userID, "api/eag-iidx", t2); + + const result = await GetImportTimestop(userID, "api/eag-iidx"); + + expect(result!.getTime()).toBe(t2.getTime()); + }); + + it("does not go backwards when called with an earlier timestamp", async () => { + const t1 = new Date("2025-06-01T00:00:00.000Z"); + const t2 = new Date("2025-01-01T00:00:00.000Z"); + + await SetImportTimestop(userID, "api/eag-iidx", t1); + await SetImportTimestop(userID, "api/eag-iidx", t2); + + const result = await GetImportTimestop(userID, "api/eag-iidx"); + + // GREATEST keeps t1 + expect(result!.getTime()).toBe(t1.getTime()); + }); + + it("only maintains one row per (user, import_type)", async () => { + const t1 = new Date("2025-01-01T00:00:00.000Z"); + const t2 = new Date("2025-06-01T00:00:00.000Z"); + + await SetImportTimestop(userID, "api/eag-iidx", t1); + await SetImportTimestop(userID, "api/eag-iidx", t2); + + const count = await DB.selectFrom("import_timestop") + .select(DB.fn.countAll().as("count")) + .where("import_timestop.user_id", "=", userID) + .where("import_timestop.import_type", "=", "api/eag-iidx") + .executeTakeFirstOrThrow(); + + expect(Number(count.count)).toBe(1); + }); + + it("does not affect other users", async () => { + const other = await seedUser({ username: "other_user" }); + const myTime = new Date("2025-01-01T00:00:00.000Z"); + const theirTime = new Date("2026-01-01T00:00:00.000Z"); + + await SetImportTimestop(userID, "api/eag-iidx", myTime); + await SetImportTimestop(other.id, "api/eag-iidx", theirTime); + + const myResult = await GetImportTimestop(userID, "api/eag-iidx"); + const theirResult = await GetImportTimestop(other.id, "api/eag-iidx"); + + expect(myResult!.getTime()).toBe(myTime.getTime()); + expect(theirResult!.getTime()).toBe(theirTime.getTime()); + }); +}); + +describe("ListImportTimestops", () => { + let userID: number; + + beforeEach(async () => { + ({ id: userID } = await seedUser()); + }); + + it("returns all API import types with null when unset", async () => { + const result = await ListImportTimestops(userID); + + expect(result.length).toBeGreaterThan(0); + expect(result.every((entry) => entry.lastScoreTime === null)).toBe(true); + }); + + it("includes stored cursors", async () => { + const time = new Date("2025-03-15T12:00:00.000Z"); + + await SetImportTimestop(userID, "api/eag-iidx", time); + + const result = await ListImportTimestops(userID); + const entry = result.find((e) => e.importType === "api/eag-iidx"); + + expect(entry?.lastScoreTime).toBe(time.getTime()); + }); +}); + +describe("DeleteImportTimestop", () => { + let userID: number; + + beforeEach(async () => { + ({ id: userID } = await seedUser()); + }); + + it("removes an existing cursor", async () => { + await SetImportTimestop(userID, "api/eag-iidx", new Date("2025-01-01T00:00:00.000Z")); + + await DeleteImportTimestop(userID, "api/eag-iidx"); + + expect(await GetImportTimestop(userID, "api/eag-iidx")).toBeNull(); + }); +}); + +describe("SetImportTimestopManual", () => { + let userID: number; + + beforeEach(async () => { + ({ id: userID } = await seedUser()); + }); + + it("can move the cursor backwards", async () => { + const later = new Date("2025-06-01T00:00:00.000Z"); + const earlier = new Date("2025-01-01T00:00:00.000Z"); + + await SetImportTimestop(userID, "api/eag-iidx", later); + await SetImportTimestopManual(userID, "api/eag-iidx", earlier); + + const result = await GetImportTimestop(userID, "api/eag-iidx"); + + expect(result!.getTime()).toBe(earlier.getTime()); + }); +}); diff --git a/typescript/server/src/lib/score-import/framework/common/timestop.ts b/typescript/server/src/lib/score-import/framework/common/timestop.ts new file mode 100644 index 000000000..05c21e7bf --- /dev/null +++ b/typescript/server/src/lib/score-import/framework/common/timestop.ts @@ -0,0 +1,113 @@ +import type { APIImportTypes, ImportTypes, integer } from "tachi-common"; + +import DB from "#services/pg/db"; +import { sql } from "kysely"; +import { apiImportTypes } from "tachi-common/constants/import-types"; + +export type ImportTimestopEntry = { + importType: APIImportTypes; + lastScoreTime: number | null; +}; + +/** + * Returns the stored last-score-time cursor for a given user + import type, + * or null if no import has been completed yet. + */ +export async function GetImportTimestop( + userID: integer, + importType: ImportTypes, +): Promise { + const row = await DB.selectFrom("import_timestop") + .select(["import_timestop.last_score_time"]) + .where("import_timestop.user_id", "=", userID) + .where("import_timestop.import_type", "=", importType) + .executeTakeFirst(); + + if (!row) { + return null; + } + + return new Date(row.last_score_time); +} + +/** + * Upserts the last-score-time cursor for a given user + import type, always + * advancing to the greater of the existing value and the new one. + */ +export async function SetImportTimestop( + userID: integer, + importType: ImportTypes, + time: Date, +): Promise { + const isoTime = time.toISOString(); + + await DB.insertInto("import_timestop") + .values({ + user_id: userID, + import_type: importType, + last_score_time: isoTime, + }) + .onConflict((oc) => + oc.columns(["user_id", "import_type"]).doUpdateSet({ + last_score_time: sql`GREATEST(EXCLUDED.last_score_time, import_timestop.last_score_time)`, + }), + ) + .execute(); +} + +/** + * Returns every API import type and its stored cursor (null when unset). + */ +export async function ListImportTimestops(userID: integer): Promise> { + const rows = await DB.selectFrom("import_timestop") + .select(["import_timestop.import_type", "import_timestop.last_score_time"]) + .where("import_timestop.user_id", "=", userID) + .execute(); + + const byType = new Map( + rows.map((row) => [row.import_type, new Date(row.last_score_time).getTime()]), + ); + + return apiImportTypes.map((importType) => ({ + importType, + lastScoreTime: byType.get(importType) ?? null, + })); +} + +/** + * Removes the stored cursor so the next import starts from scratch. + */ +export async function DeleteImportTimestop( + userID: integer, + importType: ImportTypes, +): Promise { + await DB.deleteFrom("import_timestop") + .where("import_timestop.user_id", "=", userID) + .where("import_timestop.import_type", "=", importType) + .execute(); +} + +/** + * Sets the cursor to an exact timestamp, replacing any existing value. + * Unlike {@link SetImportTimestop}, this can move the cursor backwards. + */ +export async function SetImportTimestopManual( + userID: integer, + importType: ImportTypes, + time: Date, +): Promise { + const isoTime = time.toISOString(); + + await DB.insertInto("import_timestop") + .values({ + user_id: userID, + import_type: importType, + last_score_time: isoTime, + }) + .onConflict((oc) => + oc.columns(["user_id", "import_type"]).doUpdateSet({ + last_score_time: isoTime, + }), + ) + .execute(); +} diff --git a/typescript/server/src/lib/score-import/framework/score-importing/score-import-main.ts b/typescript/server/src/lib/score-import/framework/score-importing/score-import-main.ts index f61775060..b3d3d8880 100644 --- a/typescript/server/src/lib/score-import/framework/score-importing/score-import-main.ts +++ b/typescript/server/src/lib/score-import/framework/score-importing/score-import-main.ts @@ -7,6 +7,7 @@ import type { ScoreImportJob } from "#lib/score-import/worker/types"; import { LoadImportDocumentById } from "#lib/db-formats/import-document"; import { clearPbDirtyForUser } from "#lib/jobs/drain-dirty-queues"; +import { SetImportTimestop } from "#lib/score-import/framework/common/timestop"; import { runWithImportContext } from "#lib/score-import/framework/import-run-context"; import { deleteImportRun, @@ -19,6 +20,7 @@ import DB from "#services/pg/db"; import { GetMillisecondsSince } from "#utils/misc"; import { GetUserWithID } from "#utils/user"; import { + type APIImportTypes, type GameGroup, GetGameGroupConfig, type GoalImportInfo, @@ -29,6 +31,7 @@ import { type UserDocument, type V3Game, } from "tachi-common"; +import { apiImportTypes } from "tachi-common/constants/import-types"; import type { ClassProvider } from "../calculated-data/types"; import type { ChartIDGameMap, ScoreGameMap } from "../common/types"; @@ -238,6 +241,20 @@ export default async function ScoreImportMain( observeScoreImportDuration(importType, Date.now() - timeStarted); + // For API imports, advance the per-user timestop cursor so subsequent + // imports can stop once they reach already-imported scores. + if (apiImportTypes.includes(importType as APIImportTypes)) { + const maxTimeAchieved = importInfo + .filter((i): i is { success: true } & typeof i => i.success) + .map((i) => i.content.score.timeAchieved) + .filter((t): t is number => t !== null) + .reduce((max, t) => Math.max(max, t), -Infinity); + + if (Number.isFinite(maxTimeAchieved)) { + await SetImportTimestop(userID, importType, new Date(maxTimeAchieved)); + } + } + const loaded = await LoadImportDocumentById(importID); if (!loaded) { diff --git a/typescript/server/src/lib/score-import/import-types/api/eag-iidx/parser.ts b/typescript/server/src/lib/score-import/import-types/api/eag-iidx/parser.ts index f09e7d4a7..e58ed58cf 100644 --- a/typescript/server/src/lib/score-import/import-types/api/eag-iidx/parser.ts +++ b/typescript/server/src/lib/score-import/import-types/api/eag-iidx/parser.ts @@ -1,11 +1,15 @@ import type { KtLogger } from "#lib/log/log"; import type { integer } from "tachi-common"; +import { GetImportTimestop } from "#lib/score-import/framework/common/timestop"; import { ParseKaiIIDX } from "#lib/score-import/import-types/common/api-kai/iidx/parser"; import { GetKaiAuthGuaranteed } from "#utils/queries/auth"; export async function ParseEagIIDX(userID: integer, log: KtLogger) { - const authDoc = await GetKaiAuthGuaranteed(userID, "EAG", log); + const [authDoc, lastScoreTime] = await Promise.all([ + GetKaiAuthGuaranteed(userID, "EAG", log), + GetImportTimestop(userID, "api/eag-iidx"), + ]); - return ParseKaiIIDX("EAG", authDoc, log); + return ParseKaiIIDX("EAG", authDoc, log, undefined, null, lastScoreTime); } diff --git a/typescript/server/src/lib/score-import/import-types/api/eag-sdvx/parser.ts b/typescript/server/src/lib/score-import/import-types/api/eag-sdvx/parser.ts index 41b8f6d0b..8bc90f3db 100644 --- a/typescript/server/src/lib/score-import/import-types/api/eag-sdvx/parser.ts +++ b/typescript/server/src/lib/score-import/import-types/api/eag-sdvx/parser.ts @@ -1,11 +1,15 @@ import type { KtLogger } from "#lib/log/log"; import type { integer } from "tachi-common"; +import { GetImportTimestop } from "#lib/score-import/framework/common/timestop"; import { ParseKaiSDVX } from "#lib/score-import/import-types/common/api-kai/sdvx/parser"; import { GetKaiAuthGuaranteed } from "#utils/queries/auth"; export async function ParseEagSDVX(userID: integer, log: KtLogger) { - const authDoc = await GetKaiAuthGuaranteed(userID, "EAG", log); + const [authDoc, lastScoreTime] = await Promise.all([ + GetKaiAuthGuaranteed(userID, "EAG", log), + GetImportTimestop(userID, "api/eag-sdvx"), + ]); - return ParseKaiSDVX("EAG", authDoc, log); + return ParseKaiSDVX("EAG", authDoc, log, undefined, null, lastScoreTime); } diff --git a/typescript/server/src/lib/score-import/import-types/api/flo-iidx/parser.ts b/typescript/server/src/lib/score-import/import-types/api/flo-iidx/parser.ts index a72bc7513..bf2e2e6a0 100644 --- a/typescript/server/src/lib/score-import/import-types/api/flo-iidx/parser.ts +++ b/typescript/server/src/lib/score-import/import-types/api/flo-iidx/parser.ts @@ -1,11 +1,15 @@ import type { KtLogger } from "#lib/log/log"; import type { integer } from "tachi-common"; +import { GetImportTimestop } from "#lib/score-import/framework/common/timestop"; import { ParseKaiIIDX } from "#lib/score-import/import-types/common/api-kai/iidx/parser"; import { GetKaiAuthGuaranteed } from "#utils/queries/auth"; export async function ParseFloIIDX(userID: integer, log: KtLogger) { - const authDoc = await GetKaiAuthGuaranteed(userID, "FLO", log); + const [authDoc, lastScoreTime] = await Promise.all([ + GetKaiAuthGuaranteed(userID, "FLO", log), + GetImportTimestop(userID, "api/flo-iidx"), + ]); - return ParseKaiIIDX("FLO", authDoc, log); + return ParseKaiIIDX("FLO", authDoc, log, undefined, null, lastScoreTime); } diff --git a/typescript/server/src/lib/score-import/import-types/api/flo-sdvx/parser.ts b/typescript/server/src/lib/score-import/import-types/api/flo-sdvx/parser.ts index bb0e3d856..dda939d2b 100644 --- a/typescript/server/src/lib/score-import/import-types/api/flo-sdvx/parser.ts +++ b/typescript/server/src/lib/score-import/import-types/api/flo-sdvx/parser.ts @@ -1,11 +1,15 @@ import type { KtLogger } from "#lib/log/log"; import type { integer } from "tachi-common"; +import { GetImportTimestop } from "#lib/score-import/framework/common/timestop"; import { ParseKaiSDVX } from "#lib/score-import/import-types/common/api-kai/sdvx/parser"; import { GetKaiAuthGuaranteed } from "#utils/queries/auth"; export async function ParseFloSDVX(userID: integer, log: KtLogger) { - const authDoc = await GetKaiAuthGuaranteed(userID, "FLO", log); + const [authDoc, lastScoreTime] = await Promise.all([ + GetKaiAuthGuaranteed(userID, "FLO", log), + GetImportTimestop(userID, "api/flo-sdvx"), + ]); - return ParseKaiSDVX("FLO", authDoc, log); + return ParseKaiSDVX("FLO", authDoc, log, undefined, null, lastScoreTime); } diff --git a/typescript/server/src/lib/score-import/import-types/api/min-sdvx/parser.ts b/typescript/server/src/lib/score-import/import-types/api/min-sdvx/parser.ts index a9c168495..01af7cbf5 100644 --- a/typescript/server/src/lib/score-import/import-types/api/min-sdvx/parser.ts +++ b/typescript/server/src/lib/score-import/import-types/api/min-sdvx/parser.ts @@ -1,11 +1,15 @@ import type { KtLogger } from "#lib/log/log"; import type { integer } from "tachi-common"; +import { GetImportTimestop } from "#lib/score-import/framework/common/timestop"; import { ParseKaiSDVX } from "#lib/score-import/import-types/common/api-kai/sdvx/parser"; import { GetKaiAuthGuaranteed } from "#utils/queries/auth"; export async function ParseMinSDVX(userID: integer, log: KtLogger) { - const authDoc = await GetKaiAuthGuaranteed(userID, "MIN", log); + const [authDoc, lastScoreTime] = await Promise.all([ + GetKaiAuthGuaranteed(userID, "MIN", log), + GetImportTimestop(userID, "api/min-sdvx"), + ]); - return ParseKaiSDVX("MIN", authDoc, log); + return ParseKaiSDVX("MIN", authDoc, log, undefined, null, lastScoreTime); } diff --git a/typescript/server/src/lib/score-import/import-types/api/myt-chunithm/parser.ts b/typescript/server/src/lib/score-import/import-types/api/myt-chunithm/parser.ts index 1cb2be529..950900721 100644 --- a/typescript/server/src/lib/score-import/import-types/api/myt-chunithm/parser.ts +++ b/typescript/server/src/lib/score-import/import-types/api/myt-chunithm/parser.ts @@ -3,6 +3,7 @@ import type { ParserFunctionReturns } from "#lib/score-import/import-types/commo import type { EmptyObject } from "#utils/types"; import type { GamesForGroup, integer } from "tachi-common"; +import { GetImportTimestop } from "#lib/score-import/framework/common/timestop"; import { drainMytPlaylogStream } from "#lib/score-import/import-types/common/api-myt/buffer-playlog-stream"; import { CreateMytTransport, @@ -15,9 +16,16 @@ import { createClient } from "@connectrpc/connect"; import type { MytChunithmScore } from "./types"; async function* streamPlaylog(userID: integer, log: KtLogger): AsyncIterable { - const profileApiId = await FetchMytTitleAPIID(userID, "chunithm", log); + const [profileApiId, lastScoreTime] = await Promise.all([ + FetchMytTitleAPIID(userID, "chunithm", log), + GetImportTimestop(userID, "api/myt-chunithm"), + ]); + const client = createClient(ChunithmUser, CreateMytTransport()); - const request = create(GetPlaylogRequestSchema, { profileApiId }); + const request = create(GetPlaylogRequestSchema, { + profileApiId, + lastUserPlayDate: lastScoreTime?.toISOString() ?? undefined, + }); yield* await drainMytPlaylogStream(client.getPlaylog(request), log, { gameLabel: "Chunithm", diff --git a/typescript/server/src/lib/score-import/import-types/api/myt-maimaidx/parser.ts b/typescript/server/src/lib/score-import/import-types/api/myt-maimaidx/parser.ts index 018865c1a..9e2feace7 100644 --- a/typescript/server/src/lib/score-import/import-types/api/myt-maimaidx/parser.ts +++ b/typescript/server/src/lib/score-import/import-types/api/myt-maimaidx/parser.ts @@ -3,6 +3,7 @@ import type { ParserFunctionReturns } from "#lib/score-import/import-types/commo import type { EmptyObject } from "#utils/types"; import type { GamesForGroup, integer } from "tachi-common"; +import { GetImportTimestop } from "#lib/score-import/framework/common/timestop"; import { drainMytPlaylogStream } from "#lib/score-import/import-types/common/api-myt/buffer-playlog-stream"; import { CreateMytTransport, @@ -15,9 +16,16 @@ import { createClient } from "@connectrpc/connect"; import type { MytMaimaiDxScore } from "./types"; async function* streamPlaylog(userID: integer, log: KtLogger): AsyncIterable { - const profileApiId = await FetchMytTitleAPIID(userID, "maimaidx", log); + const [profileApiId, lastScoreTime] = await Promise.all([ + FetchMytTitleAPIID(userID, "maimaidx", log), + GetImportTimestop(userID, "api/myt-maimaidx"), + ]); + const client = createClient(MaimaiUser, CreateMytTransport()); - const request = create(GetPlaylogRequestSchema, { profileApiId }); + const request = create(GetPlaylogRequestSchema, { + profileApiId, + lastUserPlayDate: lastScoreTime?.toISOString() ?? undefined, + }); yield* await drainMytPlaylogStream(client.getPlaylog(request), log, { gameLabel: "maimai DX", diff --git a/typescript/server/src/lib/score-import/import-types/api/myt-ongeki/parser.ts b/typescript/server/src/lib/score-import/import-types/api/myt-ongeki/parser.ts index c1c224b3b..f90fae568 100644 --- a/typescript/server/src/lib/score-import/import-types/api/myt-ongeki/parser.ts +++ b/typescript/server/src/lib/score-import/import-types/api/myt-ongeki/parser.ts @@ -3,6 +3,7 @@ import type { ParserFunctionReturns } from "#lib/score-import/import-types/commo import type { EmptyObject } from "#utils/types"; import type { GamesForGroup, integer } from "tachi-common"; +import { GetImportTimestop } from "#lib/score-import/framework/common/timestop"; import { drainMytPlaylogStream } from "#lib/score-import/import-types/common/api-myt/buffer-playlog-stream"; import { CreateMytTransport, @@ -15,9 +16,16 @@ import { createClient } from "@connectrpc/connect"; import type { MytOngekiScore } from "./types"; async function* streamPlaylog(userID: integer, log: KtLogger): AsyncIterable { - const profileApiId = await FetchMytTitleAPIID(userID, "ongeki", log); + const [profileApiId, lastScoreTime] = await Promise.all([ + FetchMytTitleAPIID(userID, "ongeki", log), + GetImportTimestop(userID, "api/myt-ongeki"), + ]); + const client = createClient(OngekiUser, CreateMytTransport()); - const request = create(GetPlaylogRequestSchema, { profileApiId }); + const request = create(GetPlaylogRequestSchema, { + profileApiId, + lastUserPlayDate: lastScoreTime?.toISOString() ?? undefined, + }); yield* await drainMytPlaylogStream(client.getPlaylog(request), log, { gameLabel: "Ongeki", diff --git a/typescript/server/src/lib/score-import/import-types/api/myt-wacca/parser.ts b/typescript/server/src/lib/score-import/import-types/api/myt-wacca/parser.ts index 996a27c30..4374f5ad4 100644 --- a/typescript/server/src/lib/score-import/import-types/api/myt-wacca/parser.ts +++ b/typescript/server/src/lib/score-import/import-types/api/myt-wacca/parser.ts @@ -3,6 +3,7 @@ import type { ParserFunctionReturns } from "#lib/score-import/import-types/commo import type { EmptyObject } from "#utils/types"; import type { GamesForGroup, integer } from "tachi-common"; +import { GetImportTimestop } from "#lib/score-import/framework/common/timestop"; import ScoreImportFatalError from "#lib/score-import/framework/score-importing/score-import-error"; import { drainMytPlaylogStream } from "#lib/score-import/import-types/common/api-myt/buffer-playlog-stream"; import { @@ -21,6 +22,7 @@ async function* streamPlaylog( apiId: string, log: KtLogger, userID: integer, + lastScoreTime: Date | null, ): AsyncIterable { const client = createClient(WaccaUser, CreateMytTransport()); const request = create(PlaylogRequestSchema, { apiId }); @@ -30,12 +32,22 @@ async function* streamPlaylog( userID, }); + const cutoff = lastScoreTime?.getTime() ?? null; + for (const item of items) { if (!item.info) { log.warn(`Received WACCA playlog stream item with no info - skipping.`); continue; } + if (cutoff !== null) { + const parsed = Date.parse(item.info.userPlayDate); + + if (!Number.isNaN(parsed) && parsed <= cutoff) { + continue; + } + } + yield item.info; } } @@ -44,7 +56,10 @@ export default async function ParseMytWACCA( userID: integer, log: KtLogger, ): Promise> { - const titleApiId = await FetchMytTitleAPIID(userID, "wacca", log); + const [titleApiId, lastScoreTime] = await Promise.all([ + FetchMytTitleAPIID(userID, "wacca", log), + GetImportTimestop(userID, "api/myt-wacca"), + ]); let classProvider; @@ -57,7 +72,7 @@ export default async function ParseMytWACCA( return { service: "MYT", - iterable: streamPlaylog(titleApiId, log, userID), + iterable: streamPlaylog(titleApiId, log, userID, lastScoreTime), context: {}, classProvider, gameGroup: "wacca", diff --git a/typescript/server/src/lib/score-import/import-types/common/api-cg/parsers.ts b/typescript/server/src/lib/score-import/import-types/common/api-cg/parsers.ts index 3163a6c0c..fed67bbb0 100644 --- a/typescript/server/src/lib/score-import/import-types/common/api-cg/parsers.ts +++ b/typescript/server/src/lib/score-import/import-types/common/api-cg/parsers.ts @@ -1,11 +1,12 @@ import type { KtLogger } from "#lib/log/log"; import { SELECT_CG_CARD_INFO, ToCGCardInfo } from "#lib/db-formats/cg-card-info"; +import { GetImportTimestop } from "#lib/score-import/framework/common/timestop"; import ScoreImportFatalError from "#lib/score-import/framework/score-importing/score-import-error"; import DB from "#services/pg/db"; import fetch from "node-fetch"; import { p, type PrudenceSchema } from "prudence"; -import { FormatPrError, type integer, type V3Game } from "tachi-common"; +import { type APIImportTypes, FormatPrError, type integer, type V3Game } from "tachi-common"; import type { ParserFunctionReturns } from "../types"; import type { @@ -106,16 +107,23 @@ const CG_SCHEMAS: Record = { * Create a CG parser for this supported game. Since all CG parsing code is effectively * identical, this basically just placeholders cgGame and service. */ -export function CreateCGParser(cgGame: CGSupportedGames, service: CGServices) { +export function CreateCGParser( + cgGame: CGSupportedGames, + service: CGServices, + importType: APIImportTypes, +) { return async ( userID: integer, log: KtLogger, ): Promise> => { - const row = await DB.selectFrom("priv_svc_cg_card_info") - .select(SELECT_CG_CARD_INFO) - .where("user_id", "=", userID) - .where("service", "=", service) - .executeTakeFirst(); + const [row, lastScoreTime] = await Promise.all([ + DB.selectFrom("priv_svc_cg_card_info") + .select(SELECT_CG_CARD_INFO) + .where("priv_svc_cg_card_info.user_id", "=", userID) + .where("priv_svc_cg_card_info.service", "=", service) + .executeTakeFirst(), + GetImportTimestop(userID, importType), + ]); const cardInfo = row ? ToCGCardInfo(row) : undefined; @@ -137,6 +145,17 @@ export function CreateCGParser(cgGame: CGSupportedGames, service: CGServices) throw new ScoreImportFatalError(400, FormatPrError(err, `Invalid CG ${cgGame} Score.`)); } + const cutoff = lastScoreTime?.getTime() ?? null; + + const filtered = + cutoff === null + ? (scores as Array) + : (scores as Array).filter((s) => { + const parsed = Date.parse(s.dateTime); + + return Number.isNaN(parsed) || parsed > cutoff; + }); + return { service: FormatCGService(service), context: { @@ -144,23 +163,23 @@ export function CreateCGParser(cgGame: CGSupportedGames, service: CGServices) userID: cardInfo.userID, }, gameGroup: CGGameToTachiGame(cgGame), - iterable: scores as Array, + iterable: filtered, classProvider: null, }; }; } -export const ParseCGDevMuseca = CreateCGParser("msc", "dev"); -export const ParseCGDevSDVX = CreateCGParser("sdvx", "dev"); -export const ParseCGDevJubeat = CreateCGParser("jb", "dev"); -export const ParseCGDevPopn = CreateCGParser("popn", "dev"); +export const ParseCGDevMuseca = CreateCGParser("msc", "dev", "api/cg-dev-museca"); +export const ParseCGDevSDVX = CreateCGParser("sdvx", "dev", "api/cg-dev-sdvx"); +export const ParseCGDevJubeat = CreateCGParser("jb", "dev", "api/cg-dev-jubeat"); +export const ParseCGDevPopn = CreateCGParser("popn", "dev", "api/cg-dev-popn"); -export const ParseCGGanMuseca = CreateCGParser("msc", "gan"); -export const ParseCGGanSDVX = CreateCGParser("sdvx", "gan"); -export const ParseCGGanJubeat = CreateCGParser("jb", "gan"); -export const ParseCGGanPopn = CreateCGParser("popn", "gan"); +export const ParseCGGanMuseca = CreateCGParser("msc", "gan", "api/cg-gan-museca"); +export const ParseCGGanSDVX = CreateCGParser("sdvx", "gan", "api/cg-gan-sdvx"); +export const ParseCGGanJubeat = CreateCGParser("jb", "gan", "api/cg-gan-jubeat"); +export const ParseCGGanPopn = CreateCGParser("popn", "gan", "api/cg-gan-popn"); -export const ParseCGNagMuseca = CreateCGParser("msc", "nag"); -export const ParseCGNagSDVX = CreateCGParser("sdvx", "nag"); -export const ParseCGNagJubeat = CreateCGParser("jb", "nag"); -export const ParseCGNagPopn = CreateCGParser("popn", "nag"); +export const ParseCGNagMuseca = CreateCGParser("msc", "nag", "api/cg-nag-museca"); +export const ParseCGNagSDVX = CreateCGParser("sdvx", "nag", "api/cg-nag-sdvx"); +export const ParseCGNagJubeat = CreateCGParser("jb", "nag", "api/cg-nag-jubeat"); +export const ParseCGNagPopn = CreateCGParser("popn", "nag", "api/cg-nag-popn"); diff --git a/typescript/server/src/lib/score-import/import-types/common/api-kai/iidx/parser.ts b/typescript/server/src/lib/score-import/import-types/common/api-kai/iidx/parser.ts index 4b3e54047..5bffd7d7a 100644 --- a/typescript/server/src/lib/score-import/import-types/common/api-kai/iidx/parser.ts +++ b/typescript/server/src/lib/score-import/import-types/common/api-kai/iidx/parser.ts @@ -6,6 +6,7 @@ import nodeFetch from "#utils/fetch"; import type { KaiContext } from "../types"; +import { applyKaiTimestop } from "../kai-timestop"; import { CreateKaiReauthFunction } from "../reauth"; import { TraverseKaiAPI } from "../traverse-api"; import { KaiTypeToBaseURL } from "../utils"; @@ -17,6 +18,7 @@ export async function ParseKaiIIDX( log: KtLogger, fetch = nodeFetch, reauthFn: (() => Promise) | null = null, + lastScoreTime: Date | null = null, ): Promise> { const baseUrl = KaiTypeToBaseURL(service); @@ -27,13 +29,16 @@ export async function ParseKaiIIDX( return { service, - iterable: TraverseKaiAPI( - baseUrl, - "/api/iidx/v2/play_history", - authDoc.token, - log, - resolvedReauthFn, - fetch, + iterable: applyKaiTimestop( + TraverseKaiAPI( + baseUrl, + "/api/iidx/v2/play_history", + authDoc.token, + log, + resolvedReauthFn, + fetch, + ), + lastScoreTime, ), context: { service, diff --git a/typescript/server/src/lib/score-import/import-types/common/api-kai/kai-timestop.test.ts b/typescript/server/src/lib/score-import/import-types/common/api-kai/kai-timestop.test.ts new file mode 100644 index 000000000..261bc8845 --- /dev/null +++ b/typescript/server/src/lib/score-import/import-types/common/api-kai/kai-timestop.test.ts @@ -0,0 +1,96 @@ +import { agta } from "#test-utils/misc"; +import { describe, expect, it } from "vitest"; + +import { applyKaiTimestop } from "./kai-timestop"; + +function source(items: unknown[]) { + return (async function* () { + for (const item of items) { + yield item; + } + })(); +} + +const T_OLD = "2024-01-01T00:00:00.000Z"; // 1704067200000 +const T_NEW = "2025-01-01T00:00:00.000Z"; // 1735689600000 +const CUTOFF = new Date("2024-06-01T00:00:00.000Z"); // between old and new + +describe("applyKaiTimestop", () => { + it("yields all items when lastScoreTime is null", async () => { + const items = [{ timestamp: T_OLD }, { timestamp: T_NEW }, { other: "no timestamp" }]; + + const result = await agta(applyKaiTimestop(source(items), null)); + + expect(result).toStrictEqual(items); + }); + + it("yields items newer than the cutoff", async () => { + const items = [{ timestamp: T_NEW }, { timestamp: T_NEW }]; + + const result = await agta(applyKaiTimestop(source(items), CUTOFF)); + + expect(result).toStrictEqual(items); + }); + + it("stops at the first item at or before the cutoff", async () => { + const items = [ + { timestamp: T_NEW }, + { timestamp: T_OLD }, // <= cutoff → stop here + { timestamp: T_NEW }, // would have been yielded but we already stopped + ]; + + const result = await agta(applyKaiTimestop(source(items), CUTOFF)); + + // Only the first item (newer) should come through + expect(result).toStrictEqual([{ timestamp: T_NEW }]); + }); + + it("stops immediately when the very first item is at or before the cutoff", async () => { + const items = [{ timestamp: T_OLD }, { timestamp: T_NEW }]; + + const result = await agta(applyKaiTimestop(source(items), CUTOFF)); + + expect(result).toStrictEqual([]); + }); + + it("yields items with no timestamp field through without stopping", async () => { + const items = [{ other: "field" }, { timestamp: T_NEW }]; + + const result = await agta(applyKaiTimestop(source(items), CUTOFF)); + + expect(result).toStrictEqual(items); + }); + + it("yields items with a non-string timestamp field through without stopping", async () => { + const items = [{ timestamp: 12345 }, { timestamp: T_NEW }]; + + const result = await agta(applyKaiTimestop(source(items), CUTOFF)); + + expect(result).toStrictEqual(items); + }); + + it("yields items with an unparseable timestamp through without stopping", async () => { + const items = [{ timestamp: "not-a-date" }, { timestamp: T_NEW }]; + + const result = await agta(applyKaiTimestop(source(items), CUTOFF)); + + expect(result).toStrictEqual(items); + }); + + it("stops on an item whose timestamp exactly equals the cutoff", async () => { + const items = [ + { timestamp: T_NEW }, + { timestamp: CUTOFF.toISOString() }, // exactly at cutoff + ]; + + const result = await agta(applyKaiTimestop(source(items), CUTOFF)); + + expect(result).toStrictEqual([{ timestamp: T_NEW }]); + }); + + it("yields an empty source unchanged", async () => { + const result = await agta(applyKaiTimestop(source([]), CUTOFF)); + + expect(result).toStrictEqual([]); + }); +}); diff --git a/typescript/server/src/lib/score-import/import-types/common/api-kai/kai-timestop.ts b/typescript/server/src/lib/score-import/import-types/common/api-kai/kai-timestop.ts new file mode 100644 index 000000000..f77fd2feb --- /dev/null +++ b/typescript/server/src/lib/score-import/import-types/common/api-kai/kai-timestop.ts @@ -0,0 +1,33 @@ +/** + * Wraps a Kai async iterable and stops yielding once a score's timestamp is + * at or before the given lastScoreTime cursor. Kai play_history APIs return + * scores newest-first, so the first item whose timestamp is <= lastScoreTime + * marks the boundary of already-imported data. + * + * If lastScoreTime is null the source iterable is yielded through unchanged. + */ +export async function* applyKaiTimestop( + source: AsyncIterable, + lastScoreTime: Date | null, +): AsyncIterable { + if (!lastScoreTime) { + yield* source; + return; + } + + const cutoff = lastScoreTime.getTime(); + + for await (const item of source) { + const ts = (item as Record).timestamp; + + if (typeof ts === "string") { + const parsed = Date.parse(ts); + + if (!Number.isNaN(parsed) && parsed <= cutoff) { + return; + } + } + + yield item; + } +} diff --git a/typescript/server/src/lib/score-import/import-types/common/api-kai/sdvx/parser.ts b/typescript/server/src/lib/score-import/import-types/common/api-kai/sdvx/parser.ts index 1922b8673..9b33704f4 100644 --- a/typescript/server/src/lib/score-import/import-types/common/api-kai/sdvx/parser.ts +++ b/typescript/server/src/lib/score-import/import-types/common/api-kai/sdvx/parser.ts @@ -6,6 +6,7 @@ import nodeFetch from "#utils/fetch"; import type { KaiContext } from "../types"; +import { applyKaiTimestop } from "../kai-timestop"; import { CreateKaiReauthFunction } from "../reauth"; import { type KaiAPIReauthFunction, TraverseKaiAPI } from "../traverse-api"; import { KaiTypeToBaseURL } from "../utils"; @@ -17,6 +18,7 @@ export async function ParseKaiSDVX( log: KtLogger, fetch = nodeFetch, reauthFn: KaiAPIReauthFunction | null = null, + lastScoreTime: Date | null = null, ): Promise> { const baseUrl = KaiTypeToBaseURL(service); @@ -27,13 +29,16 @@ export async function ParseKaiSDVX( return { service, - iterable: TraverseKaiAPI( - baseUrl, - "/api/sdvx/v1/play_history", - authDoc.token, - log, - resolvedReauthFn, - fetch, + iterable: applyKaiTimestop( + TraverseKaiAPI( + baseUrl, + "/api/sdvx/v1/play_history", + authDoc.token, + log, + resolvedReauthFn, + fetch, + ), + lastScoreTime, ), context: { service, diff --git a/typescript/server/src/server/router/api/v1/router.ts b/typescript/server/src/server/router/api/v1/router.ts index 5d1cfd3dd..82b7e3826 100644 --- a/typescript/server/src/server/router/api/v1/router.ts +++ b/typescript/server/src/server/router/api/v1/router.ts @@ -41,6 +41,7 @@ import "./users/_userID/following/router"; import "./users/_userID/notifications/router"; import "./users/_userID/sessions/router"; import "./users/_userID/imports/router"; +import "./users/_userID/import-timestops/router"; import "./users/_userID/settings/router"; import "./users/_userID/integrations/router"; import "./users/_userID/integrations/cg/_cgType/router"; diff --git a/typescript/server/src/server/router/api/v1/spec.ts b/typescript/server/src/server/router/api/v1/spec.ts index 5280ebf01..a7101a418 100644 --- a/typescript/server/src/server/router/api/v1/spec.ts +++ b/typescript/server/src/server/router/api/v1/spec.ts @@ -543,6 +543,34 @@ export const API_V1_SPEC = { output: docArray(), }, + "GET /users/:userID/import-timestops": { + description: "List API import timestop cursors for the authenticated user.", + input: z.object({}), + output: z.strictObject({ + timestops: z.array( + z.strictObject({ + importType: z.string(), + lastScoreTime: z.number().nullable(), + }), + ), + }), + }, + + "DELETE /users/:userID/import-timestops": { + description: "Reset an API import timestop cursor.", + input: z.object({ importType: z.string() }), + output: empty, + }, + + "PUT /users/:userID/import-timestops": { + description: "Set an API import timestop cursor to a specific timestamp.", + input: z.object({ + importType: z.string(), + lastScoreTime: z.number(), + }), + output: empty, + }, + // ──────────────────────────────────────────────── // Users /:userID/integrations/myt (Kamaitachi only) // ──────────────────────────────────────────────── diff --git a/typescript/server/src/server/router/api/v1/users/_userID/import-timestops/router.test.ts b/typescript/server/src/server/router/api/v1/users/_userID/import-timestops/router.test.ts new file mode 100644 index 000000000..8de92329e --- /dev/null +++ b/typescript/server/src/server/router/api/v1/users/_userID/import-timestops/router.test.ts @@ -0,0 +1,121 @@ +import { SetImportTimestop } from "#lib/score-import/framework/common/timestop"; +import mockApi, { CloseServerConnection } from "#test-utils/mock-api"; +import { seedUser } from "#test-utils/pg-fixtures"; +import { afterAll, beforeEach, describe, expect, it } from "vitest"; + +afterAll(() => CloseServerConnection()); + +async function loginAs(username: string, password = "password123") { + const res = await mockApi.post("/api/v1/auth/login").send({ + username, + "!password": password, + captcha: "test", + }); + + return res.headers["set-cookie"] as unknown as string[]; +} + +describe("GET /api/v1/users/:userID/import-timestops", () => { + let cookie: string[]; + + beforeEach(async () => { + await seedUser({ username: "timestop_user", withCredential: true, withSettings: true }); + await seedUser({ + username: "other_user", + email: "other@example.com", + withCredential: true, + withSettings: true, + }); + + cookie = await loginAs("timestop_user"); + }); + + it("returns 401 when not authenticated", async () => { + const res = await mockApi.get("/api/v1/users/1/import-timestops"); + + expect(res.status).toBe(401); + }); + + it("returns 403 when viewing another user's timestops", async () => { + const res = await mockApi.get("/api/v1/users/2/import-timestops").set("Cookie", cookie); + + expect(res.status).toBe(403); + }); + + it("returns all API import types for the authenticated user", async () => { + await SetImportTimestop(1, "api/eag-iidx", new Date("2025-03-15T12:00:00.000Z")); + + const res = await mockApi.get("/api/v1/users/1/import-timestops").set("Cookie", cookie); + + expect(res.status).toBe(200); + expect(res.body.body.timestops.length).toBeGreaterThan(0); + + const entry = res.body.body.timestops.find( + (e: { importType: string }) => e.importType === "api/eag-iidx", + ); + + expect(entry.lastScoreTime).toBe(new Date("2025-03-15T12:00:00.000Z").getTime()); + }); +}); + +describe("DELETE /api/v1/users/:userID/import-timestops", () => { + let cookie: string[]; + + beforeEach(async () => { + await seedUser({ username: "timestop_user", withCredential: true, withSettings: true }); + await SetImportTimestop(1, "api/eag-iidx", new Date("2025-03-15T12:00:00.000Z")); + cookie = await loginAs("timestop_user"); + }); + + it("resets a timestop", async () => { + const res = await mockApi + .delete("/api/v1/users/1/import-timestops") + .set("Cookie", cookie) + .send({ importType: "api/eag-iidx" }); + + expect(res.status).toBe(200); + + const list = await mockApi.get("/api/v1/users/1/import-timestops").set("Cookie", cookie); + const entry = list.body.body.timestops.find( + (e: { importType: string }) => e.importType === "api/eag-iidx", + ); + + expect(entry.lastScoreTime).toBeNull(); + }); + + it("rejects invalid import types", async () => { + const res = await mockApi + .delete("/api/v1/users/1/import-timestops") + .set("Cookie", cookie) + .send({ importType: "file/batch-manual" }); + + expect(res.status).toBe(400); + }); +}); + +describe("PUT /api/v1/users/:userID/import-timestops", () => { + let cookie: string[]; + + beforeEach(async () => { + await seedUser({ username: "timestop_user", withCredential: true, withSettings: true }); + cookie = await loginAs("timestop_user"); + }); + + it("sets a timestop to a specific timestamp", async () => { + const lastScoreTime = new Date("2024-06-01T08:30:00.000Z").getTime(); + + const res = await mockApi + .put("/api/v1/users/1/import-timestops") + .set("Cookie", cookie) + .send({ importType: "api/myt-ongeki", lastScoreTime }); + + expect(res.status).toBe(200); + + const list = await mockApi.get("/api/v1/users/1/import-timestops").set("Cookie", cookie); + const entry = list.body.body.timestops.find( + (e: { importType: string }) => e.importType === "api/myt-ongeki", + ); + + expect(entry.lastScoreTime).toBe(lastScoreTime); + }); +}); diff --git a/typescript/server/src/server/router/api/v1/users/_userID/import-timestops/router.ts b/typescript/server/src/server/router/api/v1/users/_userID/import-timestops/router.ts new file mode 100644 index 000000000..8c719775f --- /dev/null +++ b/typescript/server/src/server/router/api/v1/users/_userID/import-timestops/router.ts @@ -0,0 +1,84 @@ +import { withRequestedUser, withSelf } from "#lib/router/middleware"; +import { success } from "#lib/router/typed-router"; +import { + DeleteImportTimestop, + ListImportTimestops, + SetImportTimestopManual, +} from "#lib/score-import/framework/common/timestop"; +import { API_V1_ROUTER } from "#server/router/api/v1/_singleton"; +import { ExpectedErr } from "bliss"; +import { type APIImportTypes } from "tachi-common"; +import { apiImportTypes } from "tachi-common/constants/import-types"; + +function assertApiImportType(importType: string): APIImportTypes { + if (!apiImportTypes.includes(importType as APIImportTypes)) { + throw new ExpectedErr(400, `Invalid import type: ${importType}`); + } + + return importType as APIImportTypes; +} + +/** + * List API import timestop cursors for this user. + * + * @name GET /api/v1/users/:userID/import-timestops + */ +API_V1_ROUTER.add( + "GET /users/:userID/import-timestops", + withRequestedUser, + withSelf, + async ({ ctx }) => { + const { requestedUser: user } = ctx; + + const timestops = await ListImportTimestops(user.id); + + return success(`Returned ${timestops.length} import timestops.`, { timestops }); + }, +); + +/** + * Reset an API import timestop cursor so the next import starts from scratch. + * + * @param importType - The API import type to reset. + * + * @name DELETE /api/v1/users/:userID/import-timestops + */ +API_V1_ROUTER.add( + "DELETE /users/:userID/import-timestops", + withRequestedUser, + withSelf, + async ({ input, ctx }) => { + const { requestedUser: user } = ctx; + const importType = assertApiImportType(input.importType); + + await DeleteImportTimestop(user.id, importType); + + return success(`Reset import timestop for ${importType}.`, {}); + }, +); + +/** + * Set an API import timestop cursor to a specific timestamp. + * + * @param importType - The API import type to update. + * @param lastScoreTime - Epoch milliseconds for the new cursor. + * + * @name PUT /api/v1/users/:userID/import-timestops + */ +API_V1_ROUTER.add( + "PUT /users/:userID/import-timestops", + withRequestedUser, + withSelf, + async ({ input, ctx }) => { + const { requestedUser: user } = ctx; + const importType = assertApiImportType(input.importType); + + if (!Number.isFinite(input.lastScoreTime)) { + throw new ExpectedErr(400, "lastScoreTime must be a finite number."); + } + + await SetImportTimestopManual(user.id, importType, new Date(input.lastScoreTime)); + + return success(`Updated import timestop for ${importType}.`, {}); + }, +);