Max/zkldi_Tachi
1
0
Fork 0
mirror of https://github.com/zkldi/Tachi.git synced 2026-09-29 02:18:09 +03:00
Code Issues Packages Projects Releases Wiki Activity

update charts-iidx indexes and commit arc-db-merge

Browse Source
This commit is contained in:
zkldi
2021-05-25 13:35:57 +01:00
parent 1224737ab7
commit ea82bdf369
5 changed files with 56 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/pnpm-lock.yaml
Generated
+3 -3
View File
@@ -68,7 +68,7 @@ dependencies:
express-session: 1.17.1
fast-json-stable-hash: 1.0.2
fast-xml-parser: 3.19.0
kamaitachi-common: github.com/zkldi/Kamaitachi-common/cc29bbe30257c8fc009c20197b5bc37f17170a22
kamaitachi-common: github.com/zkldi/Kamaitachi-common/73ea6c4d15a4febd7746fdf611954158afc7cc41
mongodb: 3.6.6
monk: 7.3.4
multer: 1.4.2
@@ -4856,8 +4856,8 @@ packages:
'@types/yoga-layout': 1.9.2
dev: true
github.com/zkldi/Kamaitachi-common/cc29bbe30257c8fc009c20197b5bc37f17170a22:
resolution: {tarball: https://codeload.github.com/zkldi/Kamaitachi-common/tar.gz/cc29bbe30257c8fc009c20197b5bc37f17170a22}
github.com/zkldi/Kamaitachi-common/73ea6c4d15a4febd7746fdf611954158afc7cc41:
resolution: {tarball: https://codeload.github.com/zkldi/Kamaitachi-common/tar.gz/73ea6c4d15a4febd7746fdf611954158afc7cc41}
name: kamaitachi-common
version: 0.1.0
dependencies:
server/scripts/__JSON-TO-KTDB/iidx/diff-sync.ts
+1
View File
@@ -38,6 +38,7 @@ const logger = CreateLogCtx("diff.ts");
"data.inGameID": data.songID,
playtype: spl[0] as Playtypes["iidx"],
difficulty: spl[1] as Difficulties["iidx:SP" | "iidx:DP"],
isPrimary: true,
})) as ChartDocument<"iidx:SP" | "iidx:DP">;
if (!chart) {
server/scripts/arc-db-sync/arc-db-fetch.ts
+9 -9
View File
@@ -1,6 +1,6 @@
import { Command } from "commander";
import CreateLogCtx from "../../src/logger";
import fetch from "../../src/fetch";
import CreateLogCtx from "../../src/common/logger";
import fetch from "../../src/common/fetch";
import fs from "fs";
import path from "path";
@@ -18,14 +18,14 @@ program.parse(process.argv);
const options = program.opts();
async function GetIIDXSongs() {
let allSongs = [];
const allSongs = [];
let url = `https://${options.url}/api/v1/iidx/27/music/`;
let moreData = true;
while (moreData) {
// eslint-disable-next-line no-await-in-loop
let songs = await fetch(url, {
const songs = await fetch(url, {
headers: {
Authorization: `Bearer ${options.bearer}`,
},
@@ -44,12 +44,12 @@ async function GetIIDXSongs() {
}
async function GetCharts() {
let songs = JSON.parse(fs.readFileSync(path.join(__dirname, "./songs.json"), "utf-8"));
const songs = JSON.parse(fs.readFileSync(path.join(__dirname, "./songs.json"), "utf-8"));
logger.info("Parsed songs.json");
let allCharts: any[] = [];
let failed: any[] = [];
const allCharts: any[] = [];
const failed: any[] = [];
async function DoStuff(song: any) {
try {
@@ -58,7 +58,7 @@ async function GetCharts() {
);
// eslint-disable-next-line no-await-in-loop
let charts = await fetch(
const charts = await fetch(
`https://${options.url}/api/v1/iidx/27/charts/?music_id=${song._id}&omnimix=true`,
{
headers: {
@@ -77,7 +77,7 @@ async function GetCharts() {
}
}
let promises = songs.map((s: any) => DoStuff(s));
const promises = songs.map((s: any) => DoStuff(s));
await Promise.all(promises);
server/scripts/arc-db-sync/arc-db-merge.ts
+10 -10
View File
@@ -2,21 +2,21 @@ import fs from "fs";
import { ICollection } from "monk";
import path from "path";
import db from "../../src/db/db";
import CreateLogCtx from "../../src/logger";
import { FindChartWithPTDFVersion } from "../../src/score-import/database-lookup/chart-ptdf";
import { FindSongOnTitle } from "../../src/score-import/database-lookup/song-title";
import CreateLogCtx from "../../src/common/logger";
import { ChartDocument } from "kamaitachi-common";
import { FindSongOnTitle } from "../../src/common/database-lookup/song";
import { FindChartWithPTDFVersion } from "../../src/common/database-lookup/chart";
const logger = CreateLogCtx("arc-db-merge.ts");
async function MergeIDs() {
let charts = JSON.parse(fs.readFileSync(path.join(__dirname, "./charts.json"), "utf-8"));
const charts = JSON.parse(fs.readFileSync(path.join(__dirname, "./charts.json"), "utf-8"));
for (const chartData of charts) {
let songTitle = chartData._related.music[0].title;
const songTitle = chartData._related.music[0].title;
// eslint-disable-next-line no-await-in-loop
let song = await FindSongOnTitle("iidx", songTitle);
const song = await FindSongOnTitle("iidx", songTitle);
if (!song) {
logger.error(`Could not resolve ${songTitle}?`);
@@ -24,9 +24,9 @@ async function MergeIDs() {
}
for (const chart of chartData._items) {
let playtype: "SP" | "DP" = chart.play_style === "DOUBLE" ? "DP" : "SP";
const playtype: "SP" | "DP" = chart.play_style === "DOUBLE" ? "DP" : "SP";
let difficulty = chart.difficulty === "BLACK" ? "LEGGENDARIA" : chart.difficulty;
let version = "27";
const version = "27-omni";
// yeah this fails for wacky playtype mismatches -- i don't care.
if (
@@ -39,7 +39,7 @@ async function MergeIDs() {
}
// eslint-disable-next-line no-await-in-loop
let ktchiChart = await FindChartWithPTDFVersion(
const ktchiChart = await FindChartWithPTDFVersion(
"iidx",
song.id,
playtype,
@@ -70,7 +70,7 @@ async function MergeIDs() {
}
}
let r = await db.charts.iidx.update(
const r = await db.charts.iidx.update(
{
"data.arcChartID": { $exists: false },
},
server/scripts/set-indexes.ts
+33 -9
View File
@@ -1,8 +1,8 @@
import monk from "monk";
import { Command } from "commander";
import CreateLogCtx from "../src/logger";
import CreateLogCtx from "../src/common/logger";
import { IndexOptions } from "mongodb";
import { Databases, ValidDatabases } from "kamaitachi-common";
import { ValidDatabases } from "kamaitachi-common";
import { supportedGames } from "kamaitachi-common/js/config";
const logger = CreateLogCtx("set-indexes.ts");
@@ -31,7 +31,7 @@ function index(fields: Record<string, unknown>, options?: IndexOptions) {
const UNIQUE = { unique: true };
const staticIndexes: Partial<Record<Databases, Index[]>> = {
const staticIndexes: Partial<Record<ValidDatabases, Index[]>> = {
scores: [index({ scoreID: 1 }, UNIQUE)],
"score-pbs": [
index({ chartID: 1, userID: 1 }, UNIQUE),
@@ -64,7 +64,7 @@ const staticIndexes: Partial<Record<Databases, Index[]>> = {
],
// @todo #96 Add more indexes to the users collection.
users: [index({ id: 1 }, UNIQUE)],
tierlist: [
tierlists: [
index({ tierlistID: 1 }, UNIQUE),
index({ game: 1, playtype: 1, isDefault: 1 }, UNIQUE),
],
@@ -75,17 +75,41 @@ const staticIndexes: Partial<Record<Databases, Index[]>> = {
index({ game: 1, playtype: 1, table: 1, tableIndex: 1 }),
],
"kai-auth-tokens": [index({ userID: 1, service: 1 }, UNIQUE)],
"charts-iidx": [
index(
{ "data.arcChartID": 1 },
{ unique: true, partialFilterExpression: { "data.arcChartID": { $type: "string" } } }
),
index({ "data.hashSHA256": 1 }),
],
};
const indexes: Partial<Record<ValidDatabases, Index[]>> = staticIndexes;
for (const game of supportedGames) {
indexes[`charts-${game}` as "charts-iidx"] = [
index({ chartID: 1 }, UNIQUE),
index({ songID: 1, difficulty: 1, playtype: 1, isPrimary: 1 }, UNIQUE),
];
if (indexes[`charts-${game}` as "charts-iidx"]) {
indexes[`charts-${game}` as "charts-iidx"]!.push(
index({ chartID: 1 }, UNIQUE),
index(
{ songID: 1, difficulty: 1, playtype: 1, isPrimary: 1 },
{ unique: true, partialFilterExpression: { isPrimary: { $eq: true } } }
)
);
} else {
indexes[`charts-${game}` as "charts-iidx"] = [
index({ chartID: 1 }, UNIQUE),
index({ songID: 1, difficulty: 1, playtype: 1, isPrimary: 1 }, UNIQUE),
];
}
indexes[`songs-${game}` as "songs-iidx"] = [index({ id: 1 }, UNIQUE), index({ title: 1 })];
if (indexes[`songs-${game}` as "songs-iidx"]) {
indexes[`songs-${game}` as "songs-iidx"]!.push(
index({ id: 1 }, UNIQUE),
index({ title: 1 })
);
} else {
indexes[`songs-${game}` as "songs-iidx"] = [index({ id: 1 }, UNIQUE), index({ title: 1 })];
}
}
(async () => {