From ea82bdf3695a11411cf1c786bf17aa153c8af2cd Mon Sep 17 00:00:00 2001 From: zkldi Date: Tue, 25 May 2021 13:35:57 +0100 Subject: [PATCH] update charts-iidx indexes and commit arc-db-merge --- server/pnpm-lock.yaml | 6 +-- .../scripts/__JSON-TO-KTDB/iidx/diff-sync.ts | 1 + server/scripts/arc-db-sync/arc-db-fetch.ts | 18 ++++---- server/scripts/arc-db-sync/arc-db-merge.ts | 20 ++++----- server/scripts/set-indexes.ts | 42 +++++++++++++++---- 5 files changed, 56 insertions(+), 31 deletions(-) diff --git a/server/pnpm-lock.yaml b/server/pnpm-lock.yaml index 55ac1a539..6a47e8fd2 100644 --- a/server/pnpm-lock.yaml +++ b/server/pnpm-lock.yaml @@ -68,7 +68,7 @@ dependencies: express-session: 1.17.1 fast-json-stable-hash: 1.0.2 fast-xml-parser: 3.19.0 - kamaitachi-common: github.com/zkldi/Kamaitachi-common/cc29bbe30257c8fc009c20197b5bc37f17170a22 + kamaitachi-common: github.com/zkldi/Kamaitachi-common/73ea6c4d15a4febd7746fdf611954158afc7cc41 mongodb: 3.6.6 monk: 7.3.4 multer: 1.4.2 @@ -4856,8 +4856,8 @@ packages: '@types/yoga-layout': 1.9.2 dev: true - github.com/zkldi/Kamaitachi-common/cc29bbe30257c8fc009c20197b5bc37f17170a22: - resolution: {tarball: https://codeload.github.com/zkldi/Kamaitachi-common/tar.gz/cc29bbe30257c8fc009c20197b5bc37f17170a22} + github.com/zkldi/Kamaitachi-common/73ea6c4d15a4febd7746fdf611954158afc7cc41: + resolution: {tarball: https://codeload.github.com/zkldi/Kamaitachi-common/tar.gz/73ea6c4d15a4febd7746fdf611954158afc7cc41} name: kamaitachi-common version: 0.1.0 dependencies: diff --git a/server/scripts/__JSON-TO-KTDB/iidx/diff-sync.ts b/server/scripts/__JSON-TO-KTDB/iidx/diff-sync.ts index 1c8316c61..66230e6ba 100644 --- a/server/scripts/__JSON-TO-KTDB/iidx/diff-sync.ts +++ b/server/scripts/__JSON-TO-KTDB/iidx/diff-sync.ts @@ -38,6 +38,7 @@ const logger = CreateLogCtx("diff.ts"); "data.inGameID": data.songID, playtype: spl[0] as Playtypes["iidx"], difficulty: spl[1] as Difficulties["iidx:SP" | "iidx:DP"], + isPrimary: true, })) as ChartDocument<"iidx:SP" | "iidx:DP">; if (!chart) { diff --git a/server/scripts/arc-db-sync/arc-db-fetch.ts b/server/scripts/arc-db-sync/arc-db-fetch.ts index 6c1bcbcc5..a89d44f63 100644 --- a/server/scripts/arc-db-sync/arc-db-fetch.ts +++ b/server/scripts/arc-db-sync/arc-db-fetch.ts @@ -1,6 +1,6 @@ import { Command } from "commander"; -import CreateLogCtx from "../../src/logger"; -import fetch from "../../src/fetch"; +import CreateLogCtx from "../../src/common/logger"; +import fetch from "../../src/common/fetch"; import fs from "fs"; import path from "path"; @@ -18,14 +18,14 @@ program.parse(process.argv); const options = program.opts(); async function GetIIDXSongs() { - let allSongs = []; + const allSongs = []; let url = `https://${options.url}/api/v1/iidx/27/music/`; let moreData = true; while (moreData) { // eslint-disable-next-line no-await-in-loop - let songs = await fetch(url, { + const songs = await fetch(url, { headers: { Authorization: `Bearer ${options.bearer}`, }, @@ -44,12 +44,12 @@ async function GetIIDXSongs() { } async function GetCharts() { - let songs = JSON.parse(fs.readFileSync(path.join(__dirname, "./songs.json"), "utf-8")); + const songs = JSON.parse(fs.readFileSync(path.join(__dirname, "./songs.json"), "utf-8")); logger.info("Parsed songs.json"); - let allCharts: any[] = []; - let failed: any[] = []; + const allCharts: any[] = []; + const failed: any[] = []; async function DoStuff(song: any) { try { @@ -58,7 +58,7 @@ async function GetCharts() { ); // eslint-disable-next-line no-await-in-loop - let charts = await fetch( + const charts = await fetch( `https://${options.url}/api/v1/iidx/27/charts/?music_id=${song._id}&omnimix=true`, { headers: { @@ -77,7 +77,7 @@ async function GetCharts() { } } - let promises = songs.map((s: any) => DoStuff(s)); + const promises = songs.map((s: any) => DoStuff(s)); await Promise.all(promises); diff --git a/server/scripts/arc-db-sync/arc-db-merge.ts b/server/scripts/arc-db-sync/arc-db-merge.ts index ff574380a..a5176f77f 100644 --- a/server/scripts/arc-db-sync/arc-db-merge.ts +++ b/server/scripts/arc-db-sync/arc-db-merge.ts @@ -2,21 +2,21 @@ import fs from "fs"; import { ICollection } from "monk"; import path from "path"; import db from "../../src/db/db"; -import CreateLogCtx from "../../src/logger"; -import { FindChartWithPTDFVersion } from "../../src/score-import/database-lookup/chart-ptdf"; -import { FindSongOnTitle } from "../../src/score-import/database-lookup/song-title"; +import CreateLogCtx from "../../src/common/logger"; import { ChartDocument } from "kamaitachi-common"; +import { FindSongOnTitle } from "../../src/common/database-lookup/song"; +import { FindChartWithPTDFVersion } from "../../src/common/database-lookup/chart"; const logger = CreateLogCtx("arc-db-merge.ts"); async function MergeIDs() { - let charts = JSON.parse(fs.readFileSync(path.join(__dirname, "./charts.json"), "utf-8")); + const charts = JSON.parse(fs.readFileSync(path.join(__dirname, "./charts.json"), "utf-8")); for (const chartData of charts) { - let songTitle = chartData._related.music[0].title; + const songTitle = chartData._related.music[0].title; // eslint-disable-next-line no-await-in-loop - let song = await FindSongOnTitle("iidx", songTitle); + const song = await FindSongOnTitle("iidx", songTitle); if (!song) { logger.error(`Could not resolve ${songTitle}?`); @@ -24,9 +24,9 @@ async function MergeIDs() { } for (const chart of chartData._items) { - let playtype: "SP" | "DP" = chart.play_style === "DOUBLE" ? "DP" : "SP"; + const playtype: "SP" | "DP" = chart.play_style === "DOUBLE" ? "DP" : "SP"; let difficulty = chart.difficulty === "BLACK" ? "LEGGENDARIA" : chart.difficulty; - let version = "27"; + const version = "27-omni"; // yeah this fails for wacky playtype mismatches -- i don't care. if ( @@ -39,7 +39,7 @@ async function MergeIDs() { } // eslint-disable-next-line no-await-in-loop - let ktchiChart = await FindChartWithPTDFVersion( + const ktchiChart = await FindChartWithPTDFVersion( "iidx", song.id, playtype, @@ -70,7 +70,7 @@ async function MergeIDs() { } } - let r = await db.charts.iidx.update( + const r = await db.charts.iidx.update( { "data.arcChartID": { $exists: false }, }, diff --git a/server/scripts/set-indexes.ts b/server/scripts/set-indexes.ts index c3189b64c..50fe04555 100644 --- a/server/scripts/set-indexes.ts +++ b/server/scripts/set-indexes.ts @@ -1,8 +1,8 @@ import monk from "monk"; import { Command } from "commander"; -import CreateLogCtx from "../src/logger"; +import CreateLogCtx from "../src/common/logger"; import { IndexOptions } from "mongodb"; -import { Databases, ValidDatabases } from "kamaitachi-common"; +import { ValidDatabases } from "kamaitachi-common"; import { supportedGames } from "kamaitachi-common/js/config"; const logger = CreateLogCtx("set-indexes.ts"); @@ -31,7 +31,7 @@ function index(fields: Record, options?: IndexOptions) { const UNIQUE = { unique: true }; -const staticIndexes: Partial> = { +const staticIndexes: Partial> = { scores: [index({ scoreID: 1 }, UNIQUE)], "score-pbs": [ index({ chartID: 1, userID: 1 }, UNIQUE), @@ -64,7 +64,7 @@ const staticIndexes: Partial> = { ], // @todo #96 Add more indexes to the users collection. users: [index({ id: 1 }, UNIQUE)], - tierlist: [ + tierlists: [ index({ tierlistID: 1 }, UNIQUE), index({ game: 1, playtype: 1, isDefault: 1 }, UNIQUE), ], @@ -75,17 +75,41 @@ const staticIndexes: Partial> = { index({ game: 1, playtype: 1, table: 1, tableIndex: 1 }), ], "kai-auth-tokens": [index({ userID: 1, service: 1 }, UNIQUE)], + "charts-iidx": [ + index( + { "data.arcChartID": 1 }, + { unique: true, partialFilterExpression: { "data.arcChartID": { $type: "string" } } } + ), + index({ "data.hashSHA256": 1 }), + ], }; const indexes: Partial> = staticIndexes; for (const game of supportedGames) { - indexes[`charts-${game}` as "charts-iidx"] = [ - index({ chartID: 1 }, UNIQUE), - index({ songID: 1, difficulty: 1, playtype: 1, isPrimary: 1 }, UNIQUE), - ]; + if (indexes[`charts-${game}` as "charts-iidx"]) { + indexes[`charts-${game}` as "charts-iidx"]!.push( + index({ chartID: 1 }, UNIQUE), + index( + { songID: 1, difficulty: 1, playtype: 1, isPrimary: 1 }, + { unique: true, partialFilterExpression: { isPrimary: { $eq: true } } } + ) + ); + } else { + indexes[`charts-${game}` as "charts-iidx"] = [ + index({ chartID: 1 }, UNIQUE), + index({ songID: 1, difficulty: 1, playtype: 1, isPrimary: 1 }, UNIQUE), + ]; + } - indexes[`songs-${game}` as "songs-iidx"] = [index({ id: 1 }, UNIQUE), index({ title: 1 })]; + if (indexes[`songs-${game}` as "songs-iidx"]) { + indexes[`songs-${game}` as "songs-iidx"]!.push( + index({ id: 1 }, UNIQUE), + index({ title: 1 }) + ); + } else { + indexes[`songs-${game}` as "songs-iidx"] = [index({ id: 1 }, UNIQUE), index({ title: 1 })]; + } } (async () => {