feat: make the duplicate-song-title-checker work (#1104)

* feat: make the duplicate-song-title-checker work

* fix: get rid of low hanging dupe alt-titles

* no low hanger

* feat: add checks for multi-song disambiguation

* feat: ambiguoustitlestuff

* fix: add warnings to match type

* fix: whoops

* skip(ci): changestwozk

We now can support Changes because of the song title changes

common/src/types/documents.ts

@@ -514,6 +514,15 @@ export interface ImportProcessInfoInvalidDatapoint {
 	content: Record<string, never>;
 }
 
+export interface ImportProcessInfoAmbiguousTitle {
+	success: false;
+	type: "AmbiguousTitle";
+	message: string;
+	content: {
+		title: string;
+	};
+}
+
 export interface ImportProcessInfoScoreImported<GPT extends GPTString = GPTString> {
 	success: true;
 	type: "ScoreImported";
@@ -543,6 +552,7 @@ export interface ImportProcessInfoSongOrChartNotFound {
 }
 
 export type ImportProcessingInfo<GPT extends GPTString = GPTString> =
+	| ImportProcessInfoAmbiguousTitle
 	| ImportProcessInfoInternalError
 	| ImportProcessInfoInvalidDatapoint
 	| ImportProcessInfoOrphanExists

database-seeds/collections/songs-iidx.json

@@ -1794,7 +1794,6 @@
 	},
 	{
 		"altTitles": [
-			"RIDE ON THE LIGHT (HI GREAT MIX) †",
 			"RIDE ON THE LIGHT (HI GREAT MIX) †",
 			"RIDE ON THE LIGHT (HI GREAT MIX) "
 		],
@@ -2096,7 +2095,6 @@
 	},
 	{
 		"altTitles": [
-			"BALLAD FOR YOU〜想いの雨〜",
 			"BALLAD FOR YOU〜想いの雨〜",
 			"Ballad For You"
 		],
@@ -2404,8 +2402,7 @@
 	},
 	{
 		"altTitles": [
-			"Theme Of Denjin J",
-			"電人イェーガーのテーマ(Theme of DENJIN J)"
+			"Theme Of Denjin J"
 		],
 		"artist": "L.E.D.",
 		"data": {
@@ -9815,7 +9812,6 @@
 	{
 		"altTitles": [
 			"Raison d'être〜交差する宿命〜",
-			"Raison d'être～交差する宿命～",
 			"Raison d'etre～交差する宿命～",
 			"Raison d'?tre～交差する宿命～",
 			"Raison d'être〜交差する宿命〜"
@@ -12520,7 +12516,6 @@
 	{
 		"altTitles": [
 			"キャトられ♥恋はモ〜モク",
-			"キャトられ♥恋はモ～モク",
 			"キャトられ恋はモ～モク",
 			"キャトられ❤恋はモ〜モク",
 			"キャトられ?恋はモ～モク"
@@ -12926,7 +12921,6 @@
 	},
 	{
 		"altTitles": [
-			"†渚の小悪魔ラヴリィ〜レイディオ†(IIDX EDIT)",
 			"†渚の小悪魔ラヴリィ〜レイディオ†(IIDX EDIT)"
 		],
 		"artist": "夏色ビキニのPrim",
@@ -13225,9 +13219,6 @@
 	{
 		"altTitles": [
 			"旋律のドグマ〜Misérables〜",
-			"旋律のドグマ〜Misérables〜",
-			"旋律のドグマ～Misérables～",
-			"旋律のドグマ～Misérables～",
 			"旋律のドグマ～Miserables～",
 			"旋律のドグマ ～Misérables～",
 			"旋律のドグマ～Mis?rables～"

database-seeds/scripts/rerunners/iidx/iidx-mdb-parse/blacklist.txt

@@ -14,9 +14,3 @@ S16072$
 S16080$
 S16081$
 S16082$
-
-# there are two songs called Changes in omnimix. We support neither.
-# this is because it means we can't use songTitle as a matching thing
-# even though we really want to.
-S10029$
-S6211$

database-seeds/scripts/test/match-type.test.ts

@@ -74,6 +74,7 @@ for (const { game, matchType, playtype } of uniquenessChecks) {
 
 	let success = 0;
 	let fails = 0;
+	let warns = 0;
 
 	const data =
 		handler.type === "CHARTS"
@@ -81,12 +82,16 @@ for (const { game, matchType, playtype } of uniquenessChecks) {
 			: ReadCollection(`songs-${game}.json`);
 
 	const uniqueIDs = new Set();
-	for (const el of data.filter((e) => e.playtype === playtype)) {
+	for (const el of data) {
 		// skip non-primaries as they can't really be matched anyway.
 		if (handler.type === "CHARTS" && !el.isPrimary) {
 			continue;
 		}
 
+		if (handler.type === "CHARTS" && el.playtype !== playtype) {
+			continue;
+		}
+
 		let newUniqueThingies = handler.fn(el);
 
 		// make single returns into arrays. convenient.
@@ -96,15 +101,27 @@ for (const { game, matchType, playtype } of uniquenessChecks) {
 
 		for (const maybeUnique of newUniqueThingies) {
 			if (uniqueIDs.has(maybeUnique)) {
-				console.log(
-					chalk.red(
-						`ID ${maybeUnique} wasn't unique in ${FormatGame(
-							game,
-							playtype
-						)} (matchType=${matchType}). It needs to be for this matchType to be legal.`
-					)
-				);
-				fails++;
+				if (matchType === "songTitle") {
+					console.log(
+						chalk.yellow(
+							`Song title ${maybeUnique} wasn't unique in ${FormatGame(
+								game,
+								playtype
+							)}. Imports using this song title *will* have their scores rejected.`
+						)
+					);
+					warns++;
+				} else {
+					console.log(
+						chalk.red(
+							`ID ${maybeUnique} wasn't unique in ${FormatGame(
+								game,
+								playtype
+							)} (matchType=${matchType}). It needs to be for this matchType to be legal.`
+						)
+					);
+					fails++;
+				}
 			} else {
 				success++;
 				uniqueIDs.add(maybeUnique);
@@ -112,18 +129,20 @@ for (const { game, matchType, playtype } of uniquenessChecks) {
 		}
 	}
 
-	const report = `GOOD: ${success}, BAD: ${fails}(${Math.min(
-		(success * 100) / fails,
+	const report = `GOOD: ${success}, WARNS: ${warns}, BAD: ${fails}(${Math.min(
+		(success * 100) / (success + fails),
 		100
 	).toFixed(2)}%)`;
 	if (fails > 0) {
 		console.error(chalk.red(`[FAILED] ${name}. ${report}.`));
 		exitCode++;
+	} else if (warns > 0) {
+		console.error(chalk.yellow(`[GOOD ISH] ${name}. ${report}.`));
 	} else {
 		console.log(chalk.green(`[GOOD] ${name}. ${report}.`));
 	}
 
-	suites.push({ name, report, good: fails === 0 });
+	suites.push({ name, report, good: success > 0 && fails === 0 });
 }
 
 console.log(`=== Suite Overview ===`);

server/src/lib/score-import/framework/common/converter-failures.ts

@@ -3,7 +3,12 @@
 import type { ImportTypeContextMap, ImportTypeDataMap } from "../../import-types/common/types";
 import type { ImportTypes } from "tachi-common";
 
-export type FailureTypes = "Internal" | "InvalidScore" | "SkipScore" | "SongOrChartNotFound";
+export type FailureTypes =
+	| "AmbiguousTitle"
+	| "Internal"
+	| "InvalidScore"
+	| "SkipScore"
+	| "SongOrChartNotFound";
 
 export class ConverterFailure extends Error {
 	message: string;
@@ -23,6 +28,20 @@ export class ConverterFailure extends Error {
 	}
 }
 
+/**
+ * AmbiguousTitleFailure - This score could not be processed because it uses the
+ * `songTitle` matching system, yet multiple songs share this title.
+ */
+export class AmbiguousTitleFailure extends ConverterFailure {
+	title: string;
+
+	constructor(songTitle: string, message: string) {
+		super(message, "AmbiguousTitle");
+
+		this.title = songTitle;
+	}
+}
+
 /**
  * SkipScoreFailure - This score could not be processed because we do not
  * support it. This is not an error, but is not a success either.

server/src/lib/score-import/framework/score-importing/score-importing.ts

@@ -7,9 +7,13 @@ import { OrphanScore } from "../orphans/orphans";
 import db from "external/mongo/db";
 import { AppendLogCtx } from "lib/logger/logger";
 import { GetGPTString } from "tachi-common";
-import { ClassToObject, DeleteUndefinedProps } from "utils/misc";
+import { ClassToObject } from "utils/misc";
 import type { ConverterFnSuccessReturn, ConverterFunction } from "../../import-types/common/types";
-import type { ConverterFailure, SongOrChartNotFoundFailure } from "../common/converter-failures";
+import type {
+	AmbiguousTitleFailure,
+	ConverterFailure,
+	SongOrChartNotFoundFailure,
+} from "../common/converter-failures";
 import type { DryScore } from "../common/types";
 import type { KtLogger } from "lib/logger/logger";
 import type { ScoreImportJob } from "lib/score-import/worker/types";
@@ -223,6 +227,21 @@ export async function ImportIterableDatapoint<D, C>(
 				};
 			}
 
+			case "AmbiguousTitle": {
+				const atErr = err as AmbiguousTitleFailure;
+
+				logger.info(`AmbiguousTitleFailure: ${err.message}`, { err: ClassToObject(err) });
+
+				return {
+					type: "AmbiguousTitle",
+					success: false,
+					message: err.message,
+					content: {
+						title: atErr.title,
+					},
+				};
+			}
+
 			case "SkipScore":
 				return null;
 

server/src/utils/queries/songs.ts

@@ -1,6 +1,9 @@
 import { EscapeStringRegexp } from "../misc";
 import db from "external/mongo/db";
-import { InternalFailure } from "lib/score-import/framework/common/converter-failures";
+import {
+	AmbiguousTitleFailure,
+	InternalFailure,
+} from "lib/score-import/framework/common/converter-failures";
 import type { KtLogger } from "lib/logger/logger";
 import type { FindOneResult } from "monk";
 import type { Game, integer, SongDocument } from "tachi-common";
@@ -13,42 +16,70 @@ import type { Game, integer, SongDocument } from "tachi-common";
  * @param title - The song title to match.
  * @returns SongDocument
  */
-export function FindSongOnTitle(game: Game, title: string): Promise<FindOneResult<SongDocument>> {
+export async function FindSongOnTitle(game: Game, title: string): Promise<SongDocument | null> {
 	// @optimisable: Performance should be tested here by having a utility field for all-titles.
-	return db.anySongs[game].findOne({
-		$or: [
-			{
-				title,
-			},
-			{
-				altTitles: title,
-			},
-		],
-	});
+	const res = await db.anySongs[game].find(
+		{
+			$or: [
+				{
+					title,
+				},
+				{
+					altTitles: title,
+				},
+			],
+		},
+		{
+			limit: 2,
+		}
+	);
+
+	if (res.length === 2) {
+		throw new AmbiguousTitleFailure(
+			title,
+			`Multiple songs exist with the title ${title}. We cannot resolve this. Please try and use a different song resolution method.`
+		);
+	}
+
+	return res[0] ?? null;
 }
 
 /**
  * Finds a song on a song title case-insensitively.
  * This is needed for services that provide horrifically mutated string titles.
  */
-export function FindSongOnTitleInsensitive(
+export async function FindSongOnTitleInsensitive(
 	game: Game,
 	title: string
-): Promise<FindOneResult<SongDocument>> {
+): Promise<SongDocument | null> {
 	// @optimisable: Performance should be tested here by having a utility field for all-titles.
 
 	const regex = new RegExp(`^${EscapeStringRegexp(title)}$`, "iu");
 
-	return db.anySongs[game].findOne({
-		$or: [
-			{
-				title: { $regex: regex },
-			},
-			{
-				altTitles: { $regex: regex },
-			},
-		],
-	});
+	const res = await db.anySongs[game].find(
+		{
+			$or: [
+				{
+					title: { $regex: regex },
+				},
+				{
+					altTitles: { $regex: regex },
+				},
+			],
+		},
+		{
+			limit: 2,
+		}
+	);
+
+	if (res.length === 2) {
+		throw new AmbiguousTitleFailure(
+			title,
+			`Multiple songs exist with the case-insensitive title ${title}. We cannot resolve this. Please try and use a different song resolution method.`
+		);
+	}
+
+	return res[0] ?? null;
 }
 
 /**