From 32a4e7392e14f5d189266f65da24c07bcc35e643 Mon Sep 17 00:00:00 2001 From: zk Date: Wed, 22 Apr 2026 18:45:00 +0100 Subject: [PATCH] feat: orphan score stuff (#56) --- docs/docs/api/routes/import.md | 48 +++++ .../app/pages/dashboard/import/ImportPage.tsx | 3 + .../pages/dashboard/users/UserOrphansPage.tsx | 202 ++++++++++++++++++ .../client/src/app/routes/ImportRoutes.tsx | 3 + .../client/src/app/routes/UserRoutes.tsx | 6 + .../src/components/imports/ImportInfo.tsx | 32 ++- .../client/src/components/user/UserHeader.tsx | 5 + typescript/common/src/types/documents.ts | 5 +- .../score-import/framework/orphans/orphans.ts | 122 ++++++++++- .../score-importing/score-import-main.ts | 10 +- .../router/api/v1/import/router.test.ts | 93 ++++++++ .../src/server/router/api/v1/import/router.ts | 41 +++- .../server/src/server/router/api/v1/spec.ts | 30 +++ 13 files changed, 595 insertions(+), 5 deletions(-) create mode 100644 typescript/client/src/app/pages/dashboard/users/UserOrphansPage.tsx diff --git a/docs/docs/api/routes/import.md b/docs/docs/api/routes/import.md index 10b88871d..ef41b54e6 100644 --- a/docs/docs/api/routes/import.md +++ b/docs/docs/api/routes/import.md @@ -213,3 +213,51 @@ N/A "removed": 3 } ``` + +***** + +## List your orphaned scores + +`GET /api/v1/import/orphans` + +Returns rows from `orphan_score` for the authenticated user (scores that failed with SongOrChartNotFound and were persisted for later matching). Newest rows are returned first. + +### Permissions + +- submit_score + +### Query parameters + +| Parameter | Type | Description | +| :: | :: | :: | +| `limit` | Integer (optional) | Page size, 1–100. Default `50`. | +| `after` | String (optional) | Keyset cursor: the `rowID` from the last orphan in the previous page. Omit on the first request. | + +### Response + +| Property | Type | Description | +| :: | :: | :: | +| `orphans` | Array | Each object includes `orphanID`, `rowID`, `importType`, `gameGroup`, `timeInserted` (ms), `message` (nullable), `summary` (nullable, best-effort hint from stored data). | +| `hasMore` | Boolean | Whether another page exists after this one. | + +***** + +## Delete one orphaned score + +`DELETE /api/v1/import/orphans/:orphanID` + +Removes a single `orphan_score` row **only if** it belongs to the authenticated user. Use this when you no longer want Tachi to keep or retry a given orphan (for example after a bad import). + +### Permissions + +- submit_score + +### Parameters + +| Parameter | Type | Description | +| :: | :: | :: | +| `orphanID` | Path | The orphan’s `orphanID` (for example from an import error or from `GET /import/orphans`). | + +### Response + +Empty body on success (`success: true`). `404` if no such orphan exists for this user. diff --git a/typescript/client/src/app/pages/dashboard/import/ImportPage.tsx b/typescript/client/src/app/pages/dashboard/import/ImportPage.tsx index c024848bb..aacc6fd9e 100644 --- a/typescript/client/src/app/pages/dashboard/import/ImportPage.tsx +++ b/typescript/client/src/app/pages/dashboard/import/ImportPage.tsx @@ -57,6 +57,9 @@ export default function ImportPage({ user }: { user: UserDocument }) {
Want to manage or revert an import? Go to{" "} Import Management. +
+ Scores that could not be matched to a chart are kept as{" "} + orphans (view or delete them there).
([]); + const [hasMore, setHasMore] = useState(false); + const [loading, setLoading] = useState(true); + const [loadingMore, setLoadingMore] = useState(false); + const [reprocessBusy, setReprocessBusy] = useState(false); + const [lastMessage, setLastMessage] = useState(null); + + const fetchPage = useCallback(async (afterRowID: string | undefined, append: boolean) => { + const params = new URLSearchParams({ limit: "50" }); + if (afterRowID) { + params.set("after", afterRowID); + } + const res = await APIFetchV1(`/import/orphans?${params.toString()}`); + if (!res.success) { + setLastMessage(res.description); + if (!append) { + setOrphans([]); + setHasMore(false); + } + return; + } + setLastMessage(null); + setHasMore(res.body.hasMore); + if (append) { + setOrphans((prev) => [...prev, ...res.body.orphans]); + } else { + setOrphans(res.body.orphans); + } + }, []); + + useEffect(() => { + let cancelled = false; + void (async () => { + setLoading(true); + await fetchPage(undefined, false); + if (!cancelled) { + setLoading(false); + } + })(); + return () => { + cancelled = true; + }; + }, [fetchPage]); + + const onLoadMore = async () => { + const last = orphans[orphans.length - 1]; + if (!last) { + return; + } + setLoadingMore(true); + await fetchPage(last.rowID, true); + setLoadingMore(false); + }; + + const onReprocess = async () => { + setReprocessBusy(true); + setLastMessage(null); + const res = await APIFetchV1( + "/import/orphans", + { method: "POST", headers: { "Content-Type": "application/json" }, body: "{}" }, + true, + true, + ); + setReprocessBusy(false); + if (res.success) { + const { processed, failed, removed, success } = res.body; + setLastMessage( + `Reprocessed ${processed}: ${success} converted, ${failed} still unmatched, ${removed} removed as invalid.`, + ); + setLoading(true); + await fetchPage(undefined, false); + setLoading(false); + } + }; + + const onDelete = async (orphanID: string) => { + if (!window.confirm(`Delete orphan ${orphanID}? This cannot be undone.`)) { + return; + } + const res = await APIFetchV1( + `/import/orphans/${encodeURIComponent(orphanID)}`, + { method: "DELETE" }, + true, + true, + ); + if (!res.success) { + return; + } + setOrphans((prev) => prev.filter((o) => o.orphanID !== orphanID)); + }; + + return ( +
+
+

Orphan scores

+

+ When an import cannot match a song or chart (SongOrChartNotFound), Tachi still stores + that datapoint as an orphan. Orphans are retried automatically around{" "} + 00:01 UTC each day, or you can run a full reprocess below. +

+

+ Deleting an orphan only removes that queued datapoint; it does not revert an entire + import. +

+
+ +
+ +
+ + {lastMessage && {lastMessage}} + + {loading ? ( + + ) : orphans.length === 0 ? ( + You have no orphan scores right now. + ) : ( + <> + ( + + {o.summary ?? "—"} + + {o.importType} + + + {GetGameGroupConfig(o.gameGroup as GameGroup)?.name ?? o.gameGroup} + + {new Date(o.timeInserted).toLocaleString()} + {o.message ?? "—"} + + + + + )} + /> + {hasMore && ( + + )} + + )} +
+ ); +} diff --git a/typescript/client/src/app/routes/ImportRoutes.tsx b/typescript/client/src/app/routes/ImportRoutes.tsx index 270d12531..0b83e02ec 100644 --- a/typescript/client/src/app/routes/ImportRoutes.tsx +++ b/typescript/client/src/app/routes/ImportRoutes.tsx @@ -47,6 +47,9 @@ export default function ImportRoutes() { + + +
diff --git a/typescript/client/src/app/routes/UserRoutes.tsx b/typescript/client/src/app/routes/UserRoutes.tsx index e489dbb59..bcf4ac1ba 100644 --- a/typescript/client/src/app/routes/UserRoutes.tsx +++ b/typescript/client/src/app/routes/UserRoutes.tsx @@ -9,6 +9,7 @@ import UGPTSettingsPage from "#app/pages/dashboard/users/games/_game/_playtype/U import UGPTUtilsPage from "#app/pages/dashboard/users/games/_game/_playtype/utils/UGPTUtilsPage"; import UserGamesPage from "#app/pages/dashboard/users/UserGamesPage"; import UserImportsPage from "#app/pages/dashboard/users/UserImportsPage"; +import UserOrphansPage from "#app/pages/dashboard/users/UserOrphansPage"; import UserIntegrationsPage from "#app/pages/dashboard/users/UserIntegrationsPage"; import UserInvitesPage from "#app/pages/dashboard/users/UserInvitesPage"; import UserSettingsPage from "#app/pages/dashboard/users/UserSettingsPage"; @@ -133,6 +134,11 @@ function UserProfileRoutes({ reqUser }: { reqUser: UserDocument }) { + + + + + diff --git a/typescript/client/src/components/imports/ImportInfo.tsx b/typescript/client/src/components/imports/ImportInfo.tsx index 4aff011bd..472c90bbd 100644 --- a/typescript/client/src/components/imports/ImportInfo.tsx +++ b/typescript/client/src/components/imports/ImportInfo.tsx @@ -128,6 +128,16 @@ export default function ImportInfo({ Some of these errors might not be very useful. Depending on how scores are matched with data, all we have to display might be a hash. +
+ SongOrChartNotFound means the score was still{" "} + saved as an orphan for nightly matching (around 00:01 UTC) or + manual reprocess — see{" "} + {user ? ( + Orphan scores + ) : ( + "Orphan scores" + )} + . ( {r.type} - {r.message} + +
{r.message}
+ {(r.type === "SongOrChartNotFound" || + r.type === "OrphanExists" || + r.orphanID !== undefined) && ( +
+ This may be stored as an orphan.{" "} + {user ? ( + Open orphan queue + ) : ( + "Open orphan queue" + )} + {r.orphanID ? ( + <> + {" "} + (ID: {r.orphanID}) + + ) : null} +
+ )} + )} /> diff --git a/typescript/client/src/components/user/UserHeader.tsx b/typescript/client/src/components/user/UserHeader.tsx index 31a90893d..87b2a1db7 100644 --- a/typescript/client/src/components/user/UserHeader.tsx +++ b/typescript/client/src/components/user/UserHeader.tsx @@ -149,6 +149,11 @@ export function UserBottomNav({ baseUrl, reqUser }: { baseUrl: string; reqUser: Imports , ); + navItems.push( + + Orphans + , + ); // If mandates login, assume that we also use invite codes. // I'm sure we could set up an elaborate way of doing this by diff --git a/typescript/common/src/types/documents.ts b/typescript/common/src/types/documents.ts index 6aed229ef..cac0479d8 100644 --- a/typescript/common/src/types/documents.ts +++ b/typescript/common/src/types/documents.ts @@ -157,9 +157,12 @@ export interface SessionDocument { highlight: boolean; } -interface ImportErrContent { +/** One entry in {@link ImportDocument.errors} for a failed datapoint during import. */ +export interface ImportErrContent { type: string; message: string; + /** Set when this row was persisted as `orphan_score` (SongOrChartNotFound / OrphanExists). */ + orphanID?: string; } export interface ImportDocument { diff --git a/typescript/server/src/lib/score-import/framework/orphans/orphans.ts b/typescript/server/src/lib/score-import/framework/orphans/orphans.ts index b98203a76..366332c3b 100644 --- a/typescript/server/src/lib/score-import/framework/orphans/orphans.ts +++ b/typescript/server/src/lib/score-import/framework/orphans/orphans.ts @@ -14,6 +14,7 @@ import { Converters } from "#lib/score-import/import-types/converters"; import DB from "#services/pg/db"; import { GetBlacklist } from "#utils/queries/blacklist"; import { GetUserWithID } from "#utils/user"; +import { ExpectedErr } from "bliss"; import fjsh from "fast-json-stable-hash"; import { sql } from "kysely"; @@ -42,7 +43,126 @@ function pgOrphanRowToDocument(row: PgOrphanScoreRow): OrphanScoreDocument { } async function deleteOrphanByOrphanId(orphanID: string): Promise { - await DB.deleteFrom("orphan_score").where("orphan_id", "=", orphanID).execute(); + await DB.deleteFrom("orphan_score").where("orphan_score.orphan_id", "=", orphanID).execute(); +} + +/** API-facing row for listing a user’s orphan_score entries. */ +export type OrphanScoreListItem = { + orphanID: string; + rowID: string; + importType: string; + gameGroup: string; + timeInserted: number; + message: string | null; + summary: string | null; +}; + +function summarizeOrphanRow(row: PgOrphanScoreRow): string | null { + const data = row.data; + if (data && typeof data === "object") { + const d = data as Record; + for (const k of ["identifier", "title", "songTitle", "hashSHA256", "sha256"] as const) { + const v = d[k]; + if (typeof v === "string" && v.length > 0) { + return v.length > 120 ? `${v.slice(0, 117)}...` : v; + } + } + } + const ctx = row.context; + if (ctx && typeof ctx === "object") { + const c = ctx as Record; + for (const k of ["title", "identifier"] as const) { + const v = c[k]; + if (typeof v === "string" && v.length > 0) { + return v.length > 120 ? `${v.slice(0, 117)}...` : v; + } + } + const chart = c.chart; + if (chart && typeof chart === "object") { + const chartObj = chart as Record; + const sha = chartObj.sha256; + if (typeof sha === "string" && sha.length > 0) { + return `sha256:${sha.slice(0, 16)}...`; + } + } + } + return null; +} + +function orphanRowToListItem(row: PgOrphanScoreRow): OrphanScoreListItem { + const msg = row.error_message.trim(); + return { + orphanID: row.orphan_id, + rowID: row.row_id, + importType: row.import_type, + gameGroup: row.game_group, + timeInserted: new Date(row.time_inserted).getTime(), + message: msg.length > 0 ? msg : null, + summary: summarizeOrphanRow(row), + }; +} + +/** Deletes one orphan_score row if it belongs to the given user. Returns whether a row was removed. */ +export async function deleteOrphanScoreForUser(orphanID: string, userID: integer): Promise { + const result = await DB.deleteFrom("orphan_score") + .where("orphan_score.orphan_id", "=", orphanID) + .where("orphan_score.user_id", "=", userID) + .executeTakeFirst(); + + return Number(result.numDeletedRows ?? 0n) > 0; +} + +/** + * Lists orphan_score rows for a user, newest first, with keyset pagination on `row_id` + `time_inserted`. + * @param afterRowID — `row_id` of the last item from the previous page (omit on first page). + */ +export async function listOrphanScoresForUser(opts: { + userID: integer; + limit: number; + afterRowID?: string; +}): Promise<{ orphans: OrphanScoreListItem[]; hasMore: boolean }> { + const cap = Math.min(Math.max(opts.limit, 1), 100); + let anchor: { time_inserted: string; row_id: string } | undefined; + + if (opts.afterRowID !== undefined && opts.afterRowID.length > 0) { + anchor = await DB.selectFrom("orphan_score") + .select(["orphan_score.time_inserted", "orphan_score.row_id"]) + .where("orphan_score.user_id", "=", opts.userID) + .where("orphan_score.row_id", "=", opts.afterRowID) + .executeTakeFirst(); + + if (anchor === undefined) { + throw new ExpectedErr(400, "Invalid pagination cursor."); + } + } + + let q = DB.selectFrom("orphan_score") + .select(SELECT_ORPHAN_SCORE) + .where("orphan_score.user_id", "=", opts.userID) + .orderBy("orphan_score.time_inserted", "desc") + .orderBy("orphan_score.row_id", "desc") + .limit(cap + 1); + + if (anchor !== undefined) { + const anchorRow = anchor; + q = q.where((eb) => + eb.or([ + eb("orphan_score.time_inserted", "<", anchorRow.time_inserted), + eb.and([ + eb("orphan_score.time_inserted", "=", anchorRow.time_inserted), + eb("orphan_score.row_id", "<", anchorRow.row_id), + ]), + ]), + ); + } + + const rows = await q.execute(); + const hasMore = rows.length > cap; + const slice = hasMore ? rows.slice(0, cap) : rows; + return { + orphans: slice.map(orphanRowToListItem), + hasMore, + }; } /** diff --git a/typescript/server/src/lib/score-import/framework/score-importing/score-import-main.ts b/typescript/server/src/lib/score-import/framework/score-importing/score-import-main.ts index 314caae8f..66928a73e 100644 --- a/typescript/server/src/lib/score-import/framework/score-importing/score-import-main.ts +++ b/typescript/server/src/lib/score-import/framework/score-importing/score-import-main.ts @@ -433,7 +433,15 @@ function ParseImportInfo(importInfo: Array) { scoreGameMap[v3Game] = [info.content.score]; } } else { - errors.push({ type: info.type, message: info.message }); + if (info.type === "SongOrChartNotFound" || info.type === "OrphanExists") { + errors.push({ + type: info.type, + message: info.message, + orphanID: info.content.orphanID, + }); + } else { + errors.push({ type: info.type, message: info.message }); + } } } diff --git a/typescript/server/src/server/router/api/v1/import/router.test.ts b/typescript/server/src/server/router/api/v1/import/router.test.ts index c607af8af..31d54ab8f 100644 --- a/typescript/server/src/server/router/api/v1/import/router.test.ts +++ b/typescript/server/src/server/router/api/v1/import/router.test.ts @@ -121,3 +121,96 @@ describe("POST /api/v1/import/orphans", () => { expect(Number(remaining.c)).toBe(1); }); }); + +describe("GET /api/v1/import/orphans", () => { + it("returns the authenticated user’s orphan rows with pagination", async () => { + const { id: userId } = await seedUser({ + username: "import_orphan_list_user", + withCredential: true, + withSettings: true, + }); + const cookie = await loginAs("import_orphan_list_user"); + + await DB.insertInto("orphan_score") + .values({ + orphan_id: "O_LIST_A", + user_id: userId, + import_id: null, + import_type: "ir/direct-manual", + game_group: "iidx", + context: { game: "iidx-sp", version: "27", service: "foo" }, + data: { identifier: "Song A", score: 100 }, + time_inserted: new Date(2_000).toISOString(), + error_message: "err-a", + }) + .execute(); + + await DB.insertInto("orphan_score") + .values({ + orphan_id: "O_LIST_B", + user_id: userId, + import_id: null, + import_type: "ir/direct-manual", + game_group: "iidx", + context: { game: "iidx-sp", version: "27", service: "foo" }, + data: { identifier: "Song B", score: 200 }, + time_inserted: new Date(3_000).toISOString(), + error_message: "", + }) + .execute(); + + const first = await mockApi.get("/api/v1/import/orphans").query({ limit: 1 }).set("Cookie", cookie); + + expect(first.status).toBe(200); + expect(first.body.success).toBe(true); + expect(first.body.body.orphans).toHaveLength(1); + expect(first.body.body.hasMore).toBe(true); + expect(first.body.body.orphans[0].orphanID).toBe("O_LIST_B"); + expect(first.body.body.orphans[0].summary).toBe("Song B"); + + const rowID = first.body.body.orphans[0].rowID as string; + + const second = await mockApi + .get("/api/v1/import/orphans") + .query({ limit: 10, after: rowID }) + .set("Cookie", cookie); + + expect(second.status).toBe(200); + expect(second.body.body.orphans.some((o: { orphanID: string }) => o.orphanID === "O_LIST_A")).toBe( + true, + ); + }); +}); + +describe("DELETE /api/v1/import/orphans/:orphanID", () => { + it("deletes only the caller’s orphan row", async () => { + const { id: userId } = await seedUser({ + username: "import_orphan_del_user", + withCredential: true, + withSettings: true, + }); + const cookie = await loginAs("import_orphan_del_user"); + + await DB.insertInto("orphan_score") + .values({ + orphan_id: "O_DEL_ME", + user_id: userId, + import_id: null, + import_type: "ir/direct-manual", + game_group: "iidx", + context: {}, + data: {}, + time_inserted: new Date().toISOString(), + error_message: "", + }) + .execute(); + + const res = await mockApi.delete("/api/v1/import/orphans/O_DEL_ME").set("Cookie", cookie); + + expect(res.status).toBe(200); + expect(res.body.success).toBe(true); + + const again = await mockApi.delete("/api/v1/import/orphans/O_DEL_ME").set("Cookie", cookie); + expect(again.status).toBe(404); + }); +}); diff --git a/typescript/server/src/server/router/api/v1/import/router.ts b/typescript/server/src/server/router/api/v1/import/router.ts index 32ddac245..2d2511f03 100644 --- a/typescript/server/src/server/router/api/v1/import/router.ts +++ b/typescript/server/src/server/router/api/v1/import/router.ts @@ -6,7 +6,11 @@ import { SYMBOL_TACHI_API_AUTH } from "#lib/constants/tachi"; import { log } from "#lib/log/log"; import { success } from "#lib/router/typed-router"; import { ExpressWrappedScoreImportMain } from "#lib/score-import/framework/express-wrapper"; -import { DeorphanScores } from "#lib/score-import/framework/orphans/orphans"; +import { + DeorphanScores, + deleteOrphanScoreForUser, + listOrphanScoresForUser, +} from "#lib/score-import/framework/orphans/orphans"; import { MakeScoreImport } from "#lib/score-import/framework/score-import"; import { ServerConfig, TachiConfig } from "#lib/setup/config"; import { RequirePermissions } from "#server/middleware/auth"; @@ -15,6 +19,7 @@ import prValidate from "#server/middleware/prudence-validate"; import { ScoreImportRateLimiter } from "#server/middleware/rate-limiter"; import { Random20Hex } from "#utils/misc"; import { FormatUserDoc, GetUserWithIDGuaranteed } from "#utils/user"; +import { ExpectedErr } from "bliss"; import { p } from "prudence"; import { API_V1_ROUTER } from "../router"; @@ -169,3 +174,37 @@ API_V1_ROUTER.add("POST /import/orphans", async ({ req }) => { success: orphanSuccess, }); }); + +/** + * List orphaned scores for the current user (scores that could not be matched to a chart). + * + * @name GET /api/v1/import/orphans + */ +API_V1_ROUTER.add("GET /import/orphans", async ({ input, req }) => { + const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TACHI_API_AUTH].userID!); + + const body = await listOrphanScoresForUser({ + userID: userDoc.id, + limit: input.limit, + afterRowID: input.after, + }); + + return success(`Returned ${body.orphans.length} orphan scores.`, body); +}); + +/** + * Delete a single orphaned score row for the current user. + * + * @name DELETE /api/v1/import/orphans/:orphanID + */ +API_V1_ROUTER.add("DELETE /import/orphans/:orphanID", async ({ params, req }) => { + const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TACHI_API_AUTH].userID!); + + const deleted = await deleteOrphanScoreForUser(params.orphanID, userDoc.id); + + if (!deleted) { + throw new ExpectedErr(404, "No such orphan score for this user."); + } + + return success("Deleted orphan score.", {}); +}); diff --git a/typescript/server/src/server/router/api/v1/spec.ts b/typescript/server/src/server/router/api/v1/spec.ts index cfbf768a5..30fa4ecaf 100644 --- a/typescript/server/src/server/router/api/v1/spec.ts +++ b/typescript/server/src/server/router/api/v1/spec.ts @@ -1704,6 +1704,36 @@ export const API_V1_SPEC = { }), }, + "GET /import/orphans": { + description: + "List orphaned scores for the authenticated user (SongOrChartNotFound rows), newest first.", + input: z.object({ + limit: z.coerce.number().int().min(1).max(100).default(50), + /** Keyset cursor: `rowID` from the last item of the previous page. */ + after: z.string().optional(), + }), + output: z.strictObject({ + orphans: z.array( + z.strictObject({ + orphanID: z.string(), + rowID: z.string(), + importType: z.string(), + gameGroup: z.string(), + timeInserted: z.number(), + message: z.string().nullable(), + summary: z.string().nullable(), + }), + ), + hasMore: z.boolean(), + }), + }, + + "DELETE /import/orphans/:orphanID": { + description: "Delete one orphaned score row belonging to the authenticated user.", + input: z.object({}), + output: empty, + }, + // ──────────────────────────────────────────────── // Clients (OAuth2 clients) // ────────────────────────────────────────────────