feat: orphan score stuff (#56)

This commit is contained in:
zk
2026-04-22 18:45:00 +01:00
committed by GitHub
parent 20bfd4b06e
commit 32a4e7392e
13 changed files with 595 additions and 5 deletions
+48
View File
@@ -213,3 +213,51 @@ N/A
"removed": 3
}
```
*****
## List your orphaned scores
`GET /api/v1/import/orphans`
Returns rows from `orphan_score` for the authenticated user (scores that failed with SongOrChartNotFound and were persisted for later matching). Newest rows are returned first.
### Permissions
- submit_score
### Query parameters
| Parameter | Type | Description |
| :: | :: | :: |
| `limit` | Integer (optional) | Page size, 1–100. Default `50`. |
| `after` | String (optional) | Keyset cursor: the `rowID` from the last orphan in the previous page. Omit on the first request. |
### Response
| Property | Type | Description |
| :: | :: | :: |
| `orphans` | Array | Each object includes `orphanID`, `rowID`, `importType`, `gameGroup`, `timeInserted` (ms), `message` (nullable), `summary` (nullable, best-effort hint from stored data). |
| `hasMore` | Boolean | Whether another page exists after this one. |
*****
## Delete one orphaned score
`DELETE /api/v1/import/orphans/:orphanID`
Removes a single `orphan_score` row **only if** it belongs to the authenticated user. Use this when you no longer want Tachi to keep or retry a given orphan (for example after a bad import).
### Permissions
- submit_score
### Parameters
| Parameter | Type | Description |
| :: | :: | :: |
| `orphanID` | Path | The orphan’s `orphanID` (for example from an import error or from `GET /import/orphans`). |
### Response
Empty body on success (`success: true`). `404` if no such orphan exists for this user.
@@ -57,6 +57,9 @@ export default function ImportPage({ user }: { user: UserDocument }) {
<br />
Want to manage or revert an import? Go to{" "}
<Link to={`/u/${user.username}/imports`}>Import Management</Link>.
<br />
Scores that could not be matched to a chart are kept as{" "}
<Link to={`/u/${user.username}/orphans`}>orphans</Link> (view or delete them there).
</div>
<hr />
<Form.Select
@@ -0,0 +1,202 @@
import useSetSubheader from "#components/layout/header/useSetSubheader";
import TachiTable from "#components/tables/components/TachiTable";
import Loading from "#components/util/Loading";
import { APIFetchV1 } from "#util/api";
import React, { useCallback, useEffect, useState } from "react";
import { Alert, Button } from "react-bootstrap";
import { GetGameGroupConfig, type GameGroup, type UserDocument } from "tachi-common";
type OrphanListItem = {
orphanID: string;
rowID: string;
importType: string;
gameGroup: string;
timeInserted: number;
message: string | null;
summary: string | null;
};
type ListBody = { orphans: OrphanListItem[]; hasMore: boolean };
type ReprocessBody = {
processed: number;
removed: number;
failed: number;
success: number;
};
export default function UserOrphansPage({ reqUser }: { reqUser: UserDocument }) {
useSetSubheader(
["Users", reqUser.username, "Orphan scores"],
[reqUser],
`${reqUser.username}'s Orphan scores`,
);
const [orphans, setOrphans] = useState<OrphanListItem[]>([]);
const [hasMore, setHasMore] = useState(false);
const [loading, setLoading] = useState(true);
const [loadingMore, setLoadingMore] = useState(false);
const [reprocessBusy, setReprocessBusy] = useState(false);
const [lastMessage, setLastMessage] = useState<string | null>(null);
const fetchPage = useCallback(async (afterRowID: string | undefined, append: boolean) => {
const params = new URLSearchParams({ limit: "50" });
if (afterRowID) {
params.set("after", afterRowID);
}
const res = await APIFetchV1<ListBody>(`/import/orphans?${params.toString()}`);
if (!res.success) {
setLastMessage(res.description);
if (!append) {
setOrphans([]);
setHasMore(false);
}
return;
}
setLastMessage(null);
setHasMore(res.body.hasMore);
if (append) {
setOrphans((prev) => [...prev, ...res.body.orphans]);
} else {
setOrphans(res.body.orphans);
}
}, []);
useEffect(() => {
let cancelled = false;
void (async () => {
setLoading(true);
await fetchPage(undefined, false);
if (!cancelled) {
setLoading(false);
}
})();
return () => {
cancelled = true;
};
}, [fetchPage]);
const onLoadMore = async () => {
const last = orphans[orphans.length - 1];
if (!last) {
return;
}
setLoadingMore(true);
await fetchPage(last.rowID, true);
setLoadingMore(false);
};
const onReprocess = async () => {
setReprocessBusy(true);
setLastMessage(null);
const res = await APIFetchV1<ReprocessBody>(
"/import/orphans",
{ method: "POST", headers: { "Content-Type": "application/json" }, body: "{}" },
true,
true,
);
setReprocessBusy(false);
if (res.success) {
const { processed, failed, removed, success } = res.body;
setLastMessage(
`Reprocessed ${processed}: ${success} converted, ${failed} still unmatched, ${removed} removed as invalid.`,
);
setLoading(true);
await fetchPage(undefined, false);
setLoading(false);
}
};
const onDelete = async (orphanID: string) => {
if (!window.confirm(`Delete orphan ${orphanID}? This cannot be undone.`)) {
return;
}
const res = await APIFetchV1(
`/import/orphans/${encodeURIComponent(orphanID)}`,
{ method: "DELETE" },
true,
true,
);
if (!res.success) {
return;
}
setOrphans((prev) => prev.filter((o) => o.orphanID !== orphanID));
};
return (
<div className="vstack gap-4">
<div>
<h4>Orphan scores</h4>
<p className="mb-2">
When an import cannot match a song or chart (SongOrChartNotFound), Tachi still stores
that datapoint as an <strong>orphan</strong>. Orphans are retried automatically around{" "}
<strong>00:01 UTC</strong> each day, or you can run a full reprocess below.
</p>
<p className="mb-0">
Deleting an orphan only removes that queued datapoint; it does not revert an entire
import.
</p>
</div>
<div className="d-flex flex-wrap gap-2 align-items-center">
<Button disabled={reprocessBusy} onClick={() => void onReprocess()} variant="primary">
{reprocessBusy ? "Reprocessing…" : "Reprocess all my orphans now"}
</Button>
</div>
{lastMessage && <Alert variant="info">{lastMessage}</Alert>}
{loading ? (
<Loading />
) : orphans.length === 0 ? (
<Alert variant="secondary">You have no orphan scores right now.</Alert>
) : (
<>
<TachiTable
dataset={orphans}
entryName="Orphan scores"
headers={[
["Summary", "Summary"],
["Import type", "Import type"],
["Game", "Game"],
["Time", "Time"],
["Message", "Message"],
["", "Actions"],
]}
rowFunction={(o) => (
<tr key={o.orphanID}>
<td>{o.summary ?? "—"}</td>
<td>
<code className="small">{o.importType}</code>
</td>
<td>
{GetGameGroupConfig(o.gameGroup as GameGroup)?.name ?? o.gameGroup}
</td>
<td>{new Date(o.timeInserted).toLocaleString()}</td>
<td className="small text-muted">{o.message ?? "—"}</td>
<td>
<Button
onClick={() => void onDelete(o.orphanID)}
size="sm"
variant="outline-danger"
>
Delete
</Button>
</td>
</tr>
)}
/>
{hasMore && (
<Button
disabled={loadingMore}
onClick={() => void onLoadMore()}
variant="outline-secondary"
>
{loadingMore ? "Loading…" : "Load more"}
</Button>
)}
</>
)}
</div>
);
}
@@ -47,6 +47,9 @@ export default function ImportRoutes() {
<Route exact path="/import">
<ImportPage user={user} />
</Route>
<Route exact path="/import/orphan-scores">
<Redirect to={`/u/${user.username}/orphans`} />
</Route>
<Switch>
<Route path="/import/*">
<div>
@@ -9,6 +9,7 @@ import UGPTSettingsPage from "#app/pages/dashboard/users/games/_game/_playtype/U
import UGPTUtilsPage from "#app/pages/dashboard/users/games/_game/_playtype/utils/UGPTUtilsPage";
import UserGamesPage from "#app/pages/dashboard/users/UserGamesPage";
import UserImportsPage from "#app/pages/dashboard/users/UserImportsPage";
import UserOrphansPage from "#app/pages/dashboard/users/UserOrphansPage";
import UserIntegrationsPage from "#app/pages/dashboard/users/UserIntegrationsPage";
import UserInvitesPage from "#app/pages/dashboard/users/UserInvitesPage";
import UserSettingsPage from "#app/pages/dashboard/users/UserSettingsPage";
@@ -133,6 +134,11 @@ function UserProfileRoutes({ reqUser }: { reqUser: UserDocument }) {
<UserImportsPage reqUser={reqUser} />
</RequireAuthAsUserParam>
</Route>
<Route exact path="/u/:userID/orphans">
<RequireAuthAsUserParam>
<UserOrphansPage reqUser={reqUser} />
</RequireAuthAsUserParam>
</Route>
<Route exact path="/u/:userID/invites">
<RequireAuthAsUserParam>
<UserInvitesPage reqUser={reqUser} />
@@ -128,6 +128,16 @@ export default function ImportInfo({
<Alert variant="warning">
Some of these errors might not be very useful. Depending on how scores
are matched with data, all we have to display might be a hash.
<br />
<strong>SongOrChartNotFound</strong> means the score was still{" "}
<strong>saved as an orphan</strong> for nightly matching (around 00:01 UTC) or
manual reprocess — see{" "}
{user ? (
<Link to={`/u/${user.username}/orphans`}>Orphan scores</Link>
) : (
"Orphan scores"
)}
.
</Alert>
<TachiTable
dataset={data.import.errors}
@@ -139,7 +149,27 @@ export default function ImportInfo({
rowFunction={(r) => (
<tr>
<td>{r.type}</td>
<td>{r.message}</td>
<td>
<div>{r.message}</div>
{(r.type === "SongOrChartNotFound" ||
r.type === "OrphanExists" ||
r.orphanID !== undefined) && (
<div className="mt-2 small text-muted">
This may be stored as an orphan.{" "}
{user ? (
<Link to={`/u/${user.username}/orphans`}>Open orphan queue</Link>
) : (
"Open orphan queue"
)}
{r.orphanID ? (
<>
{" "}
(ID: <code>{r.orphanID}</code>)
</>
) : null}
</div>
)}
</td>
</tr>
)}
/>
@@ -149,6 +149,11 @@ export function UserBottomNav({ baseUrl, reqUser }: { baseUrl: string; reqUser:
Imports
</Navbar.Item>,
);
navItems.push(
<Navbar.Item key="orphans" to={`${baseUrl}/orphans`}>
Orphans
</Navbar.Item>,
);
// If mandates login, assume that we also use invite codes.
// I'm sure we could set up an elaborate way of doing this by
+4 -1
View File
@@ -157,9 +157,12 @@ export interface SessionDocument<TGame extends V3Game = V3Game> {
highlight: boolean;
}
interface ImportErrContent {
/** One entry in {@link ImportDocument.errors} for a failed datapoint during import. */
export interface ImportErrContent {
type: string;
message: string;
/** Set when this row was persisted as `orphan_score` (SongOrChartNotFound / OrphanExists). */
orphanID?: string;
}
export interface ImportDocument {
@@ -14,6 +14,7 @@ import { Converters } from "#lib/score-import/import-types/converters";
import DB from "#services/pg/db";
import { GetBlacklist } from "#utils/queries/blacklist";
import { GetUserWithID } from "#utils/user";
import { ExpectedErr } from "bliss";
import fjsh from "fast-json-stable-hash";
import { sql } from "kysely";
@@ -42,7 +43,126 @@ function pgOrphanRowToDocument(row: PgOrphanScoreRow): OrphanScoreDocument {
}
async function deleteOrphanByOrphanId(orphanID: string): Promise<void> {
await DB.deleteFrom("orphan_score").where("orphan_id", "=", orphanID).execute();
await DB.deleteFrom("orphan_score").where("orphan_score.orphan_id", "=", orphanID).execute();
}
/** API-facing row for listing a user’s orphan_score entries. */
export type OrphanScoreListItem = {
orphanID: string;
rowID: string;
importType: string;
gameGroup: string;
timeInserted: number;
message: string | null;
summary: string | null;
};
function summarizeOrphanRow(row: PgOrphanScoreRow): string | null {
const data = row.data;
if (data && typeof data === "object") {
const d = data as Record<string, unknown>;
for (const k of ["identifier", "title", "songTitle", "hashSHA256", "sha256"] as const) {
const v = d[k];
if (typeof v === "string" && v.length > 0) {
return v.length > 120 ? `${v.slice(0, 117)}...` : v;
}
}
}
const ctx = row.context;
if (ctx && typeof ctx === "object") {
const c = ctx as Record<string, unknown>;
for (const k of ["title", "identifier"] as const) {
const v = c[k];
if (typeof v === "string" && v.length > 0) {
return v.length > 120 ? `${v.slice(0, 117)}...` : v;
}
}
const chart = c.chart;
if (chart && typeof chart === "object") {
const chartObj = chart as Record<string, unknown>;
const sha = chartObj.sha256;
if (typeof sha === "string" && sha.length > 0) {
return `sha256:${sha.slice(0, 16)}...`;
}
}
}
return null;
}
function orphanRowToListItem(row: PgOrphanScoreRow): OrphanScoreListItem {
const msg = row.error_message.trim();
return {
orphanID: row.orphan_id,
rowID: row.row_id,
importType: row.import_type,
gameGroup: row.game_group,
timeInserted: new Date(row.time_inserted).getTime(),
message: msg.length > 0 ? msg : null,
summary: summarizeOrphanRow(row),
};
}
/** Deletes one orphan_score row if it belongs to the given user. Returns whether a row was removed. */
export async function deleteOrphanScoreForUser(orphanID: string, userID: integer): Promise<boolean> {
const result = await DB.deleteFrom("orphan_score")
.where("orphan_score.orphan_id", "=", orphanID)
.where("orphan_score.user_id", "=", userID)
.executeTakeFirst();
return Number(result.numDeletedRows ?? 0n) > 0;
}
/**
* Lists orphan_score rows for a user, newest first, with keyset pagination on `row_id` + `time_inserted`.
* @param afterRowID — `row_id` of the last item from the previous page (omit on first page).
*/
export async function listOrphanScoresForUser(opts: {
userID: integer;
limit: number;
afterRowID?: string;
}): Promise<{ orphans: OrphanScoreListItem[]; hasMore: boolean }> {
const cap = Math.min(Math.max(opts.limit, 1), 100);
let anchor: { time_inserted: string; row_id: string } | undefined;
if (opts.afterRowID !== undefined && opts.afterRowID.length > 0) {
anchor = await DB.selectFrom("orphan_score")
.select(["orphan_score.time_inserted", "orphan_score.row_id"])
.where("orphan_score.user_id", "=", opts.userID)
.where("orphan_score.row_id", "=", opts.afterRowID)
.executeTakeFirst();
if (anchor === undefined) {
throw new ExpectedErr(400, "Invalid pagination cursor.");
}
}
let q = DB.selectFrom("orphan_score")
.select(SELECT_ORPHAN_SCORE)
.where("orphan_score.user_id", "=", opts.userID)
.orderBy("orphan_score.time_inserted", "desc")
.orderBy("orphan_score.row_id", "desc")
.limit(cap + 1);
if (anchor !== undefined) {
const anchorRow = anchor;
q = q.where((eb) =>
eb.or([
eb("orphan_score.time_inserted", "<", anchorRow.time_inserted),
eb.and([
eb("orphan_score.time_inserted", "=", anchorRow.time_inserted),
eb("orphan_score.row_id", "<", anchorRow.row_id),
]),
]),
);
}
const rows = await q.execute();
const hasMore = rows.length > cap;
const slice = hasMore ? rows.slice(0, cap) : rows;
return {
orphans: slice.map(orphanRowToListItem),
hasMore,
};
}
/**
@@ -433,7 +433,15 @@ function ParseImportInfo(importInfo: Array<ImportProcessingInfo>) {
scoreGameMap[v3Game] = [info.content.score];
}
} else {
errors.push({ type: info.type, message: info.message });
if (info.type === "SongOrChartNotFound" || info.type === "OrphanExists") {
errors.push({
type: info.type,
message: info.message,
orphanID: info.content.orphanID,
});
} else {
errors.push({ type: info.type, message: info.message });
}
}
}
@@ -121,3 +121,96 @@ describe("POST /api/v1/import/orphans", () => {
expect(Number(remaining.c)).toBe(1);
});
});
describe("GET /api/v1/import/orphans", () => {
it("returns the authenticated user’s orphan rows with pagination", async () => {
const { id: userId } = await seedUser({
username: "import_orphan_list_user",
withCredential: true,
withSettings: true,
});
const cookie = await loginAs("import_orphan_list_user");
await DB.insertInto("orphan_score")
.values({
orphan_id: "O_LIST_A",
user_id: userId,
import_id: null,
import_type: "ir/direct-manual",
game_group: "iidx",
context: { game: "iidx-sp", version: "27", service: "foo" },
data: { identifier: "Song A", score: 100 },
time_inserted: new Date(2_000).toISOString(),
error_message: "err-a",
})
.execute();
await DB.insertInto("orphan_score")
.values({
orphan_id: "O_LIST_B",
user_id: userId,
import_id: null,
import_type: "ir/direct-manual",
game_group: "iidx",
context: { game: "iidx-sp", version: "27", service: "foo" },
data: { identifier: "Song B", score: 200 },
time_inserted: new Date(3_000).toISOString(),
error_message: "",
})
.execute();
const first = await mockApi.get("/api/v1/import/orphans").query({ limit: 1 }).set("Cookie", cookie);
expect(first.status).toBe(200);
expect(first.body.success).toBe(true);
expect(first.body.body.orphans).toHaveLength(1);
expect(first.body.body.hasMore).toBe(true);
expect(first.body.body.orphans[0].orphanID).toBe("O_LIST_B");
expect(first.body.body.orphans[0].summary).toBe("Song B");
const rowID = first.body.body.orphans[0].rowID as string;
const second = await mockApi
.get("/api/v1/import/orphans")
.query({ limit: 10, after: rowID })
.set("Cookie", cookie);
expect(second.status).toBe(200);
expect(second.body.body.orphans.some((o: { orphanID: string }) => o.orphanID === "O_LIST_A")).toBe(
true,
);
});
});
describe("DELETE /api/v1/import/orphans/:orphanID", () => {
it("deletes only the caller’s orphan row", async () => {
const { id: userId } = await seedUser({
username: "import_orphan_del_user",
withCredential: true,
withSettings: true,
});
const cookie = await loginAs("import_orphan_del_user");
await DB.insertInto("orphan_score")
.values({
orphan_id: "O_DEL_ME",
user_id: userId,
import_id: null,
import_type: "ir/direct-manual",
game_group: "iidx",
context: {},
data: {},
time_inserted: new Date().toISOString(),
error_message: "",
})
.execute();
const res = await mockApi.delete("/api/v1/import/orphans/O_DEL_ME").set("Cookie", cookie);
expect(res.status).toBe(200);
expect(res.body.success).toBe(true);
const again = await mockApi.delete("/api/v1/import/orphans/O_DEL_ME").set("Cookie", cookie);
expect(again.status).toBe(404);
});
});
@@ -6,7 +6,11 @@ import { SYMBOL_TACHI_API_AUTH } from "#lib/constants/tachi";
import { log } from "#lib/log/log";
import { success } from "#lib/router/typed-router";
import { ExpressWrappedScoreImportMain } from "#lib/score-import/framework/express-wrapper";
import { DeorphanScores } from "#lib/score-import/framework/orphans/orphans";
import {
DeorphanScores,
deleteOrphanScoreForUser,
listOrphanScoresForUser,
} from "#lib/score-import/framework/orphans/orphans";
import { MakeScoreImport } from "#lib/score-import/framework/score-import";
import { ServerConfig, TachiConfig } from "#lib/setup/config";
import { RequirePermissions } from "#server/middleware/auth";
@@ -15,6 +19,7 @@ import prValidate from "#server/middleware/prudence-validate";
import { ScoreImportRateLimiter } from "#server/middleware/rate-limiter";
import { Random20Hex } from "#utils/misc";
import { FormatUserDoc, GetUserWithIDGuaranteed } from "#utils/user";
import { ExpectedErr } from "bliss";
import { p } from "prudence";
import { API_V1_ROUTER } from "../router";
@@ -169,3 +174,37 @@ API_V1_ROUTER.add("POST /import/orphans", async ({ req }) => {
success: orphanSuccess,
});
});
/**
* List orphaned scores for the current user (scores that could not be matched to a chart).
*
* @name GET /api/v1/import/orphans
*/
API_V1_ROUTER.add("GET /import/orphans", async ({ input, req }) => {
const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TACHI_API_AUTH].userID!);
const body = await listOrphanScoresForUser({
userID: userDoc.id,
limit: input.limit,
afterRowID: input.after,
});
return success(`Returned ${body.orphans.length} orphan scores.`, body);
});
/**
* Delete a single orphaned score row for the current user.
*
* @name DELETE /api/v1/import/orphans/:orphanID
*/
API_V1_ROUTER.add("DELETE /import/orphans/:orphanID", async ({ params, req }) => {
const userDoc = await GetUserWithIDGuaranteed(req[SYMBOL_TACHI_API_AUTH].userID!);
const deleted = await deleteOrphanScoreForUser(params.orphanID, userDoc.id);
if (!deleted) {
throw new ExpectedErr(404, "No such orphan score for this user.");
}
return success("Deleted orphan score.", {});
});
@@ -1704,6 +1704,36 @@ export const API_V1_SPEC = {
}),
},
"GET /import/orphans": {
description:
"List orphaned scores for the authenticated user (SongOrChartNotFound rows), newest first.",
input: z.object({
limit: z.coerce.number().int().min(1).max(100).default(50),
/** Keyset cursor: `rowID` from the last item of the previous page. */
after: z.string().optional(),
}),
output: z.strictObject({
orphans: z.array(
z.strictObject({
orphanID: z.string(),
rowID: z.string(),
importType: z.string(),
gameGroup: z.string(),
timeInserted: z.number(),
message: z.string().nullable(),
summary: z.string().nullable(),
}),
),
hasMore: z.boolean(),
}),
},
"DELETE /import/orphans/:orphanID": {
description: "Delete one orphaned score row belonging to the authenticated user.",
input: z.object({}),
output: empty,
},
// ────────────────────────────────────────────────
// Clients (OAuth2 clients)
// ────────────────────────────────────────────────