fix: vendor fzf

This commit is contained in:
zkldi
2025-03-18 18:42:39 +00:00
parent 923703c0b0
commit f75c85fd52
17 changed files with 2452 additions and 10 deletions
+1
View File
@@ -1 +1,2 @@
src/proto/generated
src/lib/search/fzf/**/*.ts
-2
View File
@@ -83,8 +83,6 @@
"express-session": "1.17.2",
"fast-json-stable-hash": "1.0.3",
"fast-xml-parser": "4.1.2",
"fix-esm": "^1.0.1",
"fzf": "^0.5.1",
"helmet": "5.0.2",
"html-entities": "^2.3.2",
"json5": "2.2.0",
+29
View File
@@ -0,0 +1,29 @@
BSD 3-Clause License
Copyright (c) 2021, Ajit
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
File diff suppressed because it is too large Load Diff
+45
View File
@@ -0,0 +1,45 @@
/* eslint-disable */
// @ts-nocheck
import type { Rune } from "./runes";
// values for `\s` from https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Cheatsheet
const whitespaceRunes = new Set(
" \f\n\r\t\v\u00a0\u1680\u2028\u2029\u202f\u205f\u3000\ufeff"
.split("")
.map((v) => v.codePointAt(0)!)
);
for (let codePoint = "\u2000".codePointAt(0)!; codePoint <= "\u200a".codePointAt(0)!; codePoint++) {
whitespaceRunes.add(codePoint);
}
export const isWhitespace = (rune: Rune) => whitespaceRunes.has(rune);
export const whitespacesAtStart = (runes: Array<Rune>) => {
let whitespaces = 0;
for (const rune of runes) {
if (isWhitespace(rune)) {
whitespaces++;
} else {
break;
}
}
return whitespaces;
};
export const whitespacesAtEnd = (runes: Array<Rune>) => {
let whitespaces = 0;
for (let i = runes.length - 1; i >= 0; i--) {
if (isWhitespace(runes[i])) {
whitespaces++;
} else {
break;
}
}
return whitespaces;
};
+140
View File
@@ -0,0 +1,140 @@
/* eslint-disable */
// @ts-nocheck
import { TermType, termTypeMap } from "./pattern";
import { slab } from "./slab";
import type { AlgoFn } from "./algo";
import type { Int32 } from "./numerics";
import type { buildPatternForExtendedMatch } from "./pattern";
import type { Rune } from "./runes";
import type { Slab } from "./slab";
interface Token {
text: Array<Rune>;
prefixLength: Int32;
}
// this is [int32, int32] in golang code
type Offset = [number, number];
function iter(
algoFn: AlgoFn,
tokens: Array<Token>,
caseSensitive: boolean,
normalize: boolean,
forward: boolean,
pattern: Array<Rune>,
slab: Slab
): [Offset, number, Set<number> | null] {
for (const part of tokens) {
const [res, pos] = algoFn(
caseSensitive,
normalize,
forward,
part.text,
pattern,
true,
slab
);
if (res.start >= 0) {
// res.start and res.end were typecasted to int32 here
const sidx = res.start + part.prefixLength;
const eidx = res.end + part.prefixLength;
if (pos !== null) {
const newPos = new Set<number>();
// part.prefixLength is typecasted to int here
pos.forEach((v) => newPos.add(part.prefixLength + v));
return [[sidx, eidx], res.score, newPos];
}
return [[sidx, eidx], res.score, pos];
}
}
return [[-1, -1], 0, null];
}
export function computeExtendedMatch(
text: Array<Rune>,
pattern: ReturnType<typeof buildPatternForExtendedMatch>,
fuzzyAlgo: AlgoFn,
forward: boolean
) {
// https://github.com/junegunn/fzf/blob/764316a53d0eb60b315f0bbcd513de58ed57a876/src/pattern.go#L354
// ^ TODO maybe this helps in caching by not calculating already calculated stuff but whatever
const input: Array<{
text: Array<Rune>;
prefixLength: number;
}> = [
{
text,
prefixLength: 0,
},
];
const offsets: Array<Offset> = [];
let totalScore = 0;
const allPos = new Set<number>();
for (const termSet of pattern.termSets) {
let offset: Offset = [0, 0];
let currentScore = 0;
let matched = false;
for (const term of termSet) {
let algoFn = termTypeMap[term.typ];
if (term.typ === TermType.Fuzzy) {
algoFn = fuzzyAlgo;
}
const [off, score, pos] = iter(
algoFn,
input,
term.caseSensitive,
term.normalize,
forward,
term.text,
slab
);
const sidx = off[0];
if (sidx >= 0) {
if (term.inv) {
continue;
}
offset = off;
currentScore = score;
matched = true;
if (pos !== null) {
pos.forEach((v) => allPos.add(v));
} else {
for (let idx = off[0]; idx < off[1]; ++idx) {
// idx is typecasted to int
allPos.add(idx);
}
}
break;
} else if (term.inv) {
offset = [0, 0];
currentScore = 0;
matched = true;
continue;
}
}
if (matched) {
offsets.push(offset);
totalScore = totalScore + currentScore;
}
}
return { offsets, totalScore, allPos };
}
+175
View File
@@ -0,0 +1,175 @@
/* eslint-disable */
// @ts-nocheck
import { fuzzyMatchV2, fuzzyMatchV1, exactMatchNaive } from "./algo";
import { basicMatch, asyncBasicMatch } from "./matchers";
import { strToRunes } from "./runes";
import type { AlgoFn } from "./algo";
import type { Rune } from "./runes";
import type {
FzfResultItem,
BaseOptions,
SyncOptions,
AsyncOptions,
Tiebreaker,
Token,
Selector,
} from "./types";
export type ArrayElement<ArrayType extends ReadonlyArray<unknown>> =
ArrayType extends ReadonlyArray<infer ElementType> ? ElementType : never;
type SortAttrs<U> =
| {
sort?: true;
tiebreakers?: Array<Tiebreaker<U>>;
}
| { sort: false };
type BaseOptsToUse<U> = Omit<Partial<BaseOptions<U>>, "sort" | "tiebreakers"> & SortAttrs<U>;
// from https://stackoverflow.com/a/52318137/7683365
type BaseOptionsTuple<U> = U extends string
? [options?: BaseOptsToUse<U>]
: [options: BaseOptsToUse<U> & { selector: Selector<U> }];
const defaultOpts: BaseOptions<any> = {
limit: Infinity,
selector: (v) => v,
casing: "smart-case",
normalize: true,
fuzzy: "v2",
// example:
// tiebreakers: [byLengthAsc, byStartAsc],
tiebreakers: [],
sort: true,
forward: true,
};
export abstract class BaseFinder<L extends ReadonlyArray<any>> {
runesList: Array<Array<Rune>>;
items: L;
readonly opts: BaseOptions<ArrayElement<L>>;
algoFn: AlgoFn;
constructor(list: L, ...optionsTuple: BaseOptionsTuple<ArrayElement<L>>) {
this.opts = { ...defaultOpts, ...optionsTuple[0] };
this.items = list;
this.runesList = list.map((item) => strToRunes(this.opts.selector(item).normalize()));
this.algoFn = exactMatchNaive;
switch (this.opts.fuzzy) {
case "v2": {
this.algoFn = fuzzyMatchV2;
break;
}
case "v1": {
this.algoFn = fuzzyMatchV1;
break;
}
}
}
}
export type SyncOptsToUse<U> = BaseOptsToUse<U> & Partial<Pick<SyncOptions<U>, "match">>;
export type SyncOptionsTuple<U> = U extends string
? [options?: SyncOptsToUse<U>]
: [options: SyncOptsToUse<U> & { selector: Selector<U> }];
const syncDefaultOpts: SyncOptions<any> = {
...defaultOpts,
match: basicMatch,
};
export class SyncFinder<L extends ReadonlyArray<any>> extends BaseFinder<L> {
readonly opts: SyncOptions<ArrayElement<L>>;
constructor(list: L, ...optionsTuple: SyncOptionsTuple<ArrayElement<L>>) {
super(list, ...optionsTuple);
this.opts = { ...syncDefaultOpts, ...optionsTuple[0] };
}
find(query: string): Array<FzfResultItem<ArrayElement<L>>> {
if (query.length === 0 || this.items.length === 0) {
return this.items.slice(0, this.opts.limit).map(createResultItemWithEmptyPos);
}
query = query.normalize();
const result: Array<FzfResultItem<ArrayElement<L>>> = this.opts.match.bind(this)(query);
return postProcessResultItems(result, this.opts);
}
}
export type AsyncOptsToUse<U> = BaseOptsToUse<U> & Partial<Pick<AsyncOptions<U>, "match">>;
export type AsyncOptionsTuple<U> = U extends string
? [options?: AsyncOptsToUse<U>]
: [options: AsyncOptsToUse<U> & { selector: Selector<U> }];
const asyncDefaultOpts: AsyncOptions<any> = {
...defaultOpts,
match: asyncBasicMatch,
};
export class AsyncFinder<L extends ReadonlyArray<any>> extends BaseFinder<L> {
readonly opts: AsyncOptions<ArrayElement<L>>;
token: Token;
constructor(list: L, ...optionsTuple: AsyncOptionsTuple<ArrayElement<L>>) {
super(list, ...optionsTuple);
this.opts = { ...asyncDefaultOpts, ...optionsTuple[0] };
this.token = { cancelled: false };
}
async find(query: string): Promise<Array<FzfResultItem<ArrayElement<L>>>> {
this.token.cancelled = true;
this.token = { cancelled: false };
if (query.length === 0 || this.items.length === 0) {
return this.items.slice(0, this.opts.limit).map(createResultItemWithEmptyPos);
}
query = query.normalize();
const result = await this.opts.match.bind(this)(query, this.token);
return postProcessResultItems(result, this.opts);
}
}
const createResultItemWithEmptyPos = <U>(item: U): FzfResultItem<U> => ({
item,
start: -1,
end: -1,
score: 0,
positions: new Set(),
});
function postProcessResultItems<U>(result: Array<FzfResultItem<U>>, opts: BaseOptions<U>) {
if (opts.sort) {
const { selector } = opts;
result.sort((a, b) => {
if (a.score === b.score) {
for (const tiebreaker of opts.tiebreakers) {
const diff = tiebreaker(a, b, selector);
if (diff !== 0) {
return diff;
}
}
}
return 0;
});
}
if (Number.isFinite(opts.limit)) {
result.splice(opts.limit);
}
return result;
}
+44
View File
@@ -0,0 +1,44 @@
/* eslint-disable */
// @ts-nocheck
import { SyncFinder, AsyncFinder } from "./finders";
import type {
ArrayElement,
SyncOptionsTuple,
SyncOptsToUse,
AsyncOptionsTuple,
AsyncOptsToUse,
} from "./finders";
import type { SyncOptions, AsyncOptions } from "./types";
export type { FzfResultItem, Selector, Tiebreaker } from "./types";
export * from "./matchers";
export * from "./tiebreakers";
export type FzfOptions<U = string> = U extends string
? SyncOptsToUse<U>
: SyncOptsToUse<U> & { selector: SyncOptions<U>["selector"] };
export class Fzf<L extends ReadonlyArray<any>> {
private readonly finder: SyncFinder<L>;
find: SyncFinder<L>["find"];
constructor(list: L, ...optionsTuple: SyncOptionsTuple<ArrayElement<L>>) {
this.finder = new SyncFinder(list, ...optionsTuple);
this.find = this.finder.find.bind(this.finder);
}
}
export type AsyncFzfOptions<U = string> = U extends string
? AsyncOptsToUse<U>
: AsyncOptsToUse<U> & { selector: AsyncOptions<U>["selector"] };
export class AsyncFzf<L extends ReadonlyArray<any>> {
private readonly finder: AsyncFinder<L>;
find: AsyncFinder<L>["find"];
constructor(list: L, ...optionsTuple: AsyncOptionsTuple<ArrayElement<L>>) {
this.finder = new AsyncFinder(list, ...optionsTuple);
this.find = this.finder.find.bind(this.finder);
}
}
+250
View File
@@ -0,0 +1,250 @@
/* eslint-disable */
// @ts-nocheck
import { computeExtendedMatch } from "./extended";
import { buildPatternForBasicMatch, buildPatternForExtendedMatch } from "./pattern";
import { slab } from "./slab";
import type { BaseFinder, SyncFinder, AsyncFinder } from "./finders";
import type { FzfResultItem, Token } from "./types";
function getResultFromScoreMap<T>(
scoreMap: Record<number, Array<FzfResultItem<T>>>,
limit: number
): Array<FzfResultItem<T>> {
const scoresInDesc = Object.keys(scoreMap)
.map((v) => parseInt(v, 10))
.sort((a, b) => b - a);
let result: Array<FzfResultItem<T>> = [];
for (const score of scoresInDesc) {
result = result.concat(scoreMap[score]);
if (result.length >= limit) {
break;
}
}
return result;
}
function getBasicMatchIter<U>(
this: BaseFinder<ReadonlyArray<U>>,
scoreMap: Record<number, Array<FzfResultItem<U>>>,
queryRunes: Array<number>,
caseSensitive: boolean
) {
return (idx: number) => {
const itemRunes = this.runesList[idx];
if (queryRunes.length > itemRunes.length) {
return;
}
let [match, positions] = this.algoFn(
caseSensitive,
this.opts.normalize,
this.opts.forward,
itemRunes,
queryRunes,
true,
slab
);
if (match.start === -1) {
return;
}
// We don't get positions array back for exact match, so we'll fill it by ourselves.
if (this.opts.fuzzy === false) {
positions = new Set();
for (let position = match.start; position < match.end; ++position) {
positions.add(position);
}
}
// If we aren't sorting, we'll put all items in the same score bucket
// (we've chosen zero score for it below). This will result in us getting
// items in the same order in which we've send them in the list.
const scoreKey = this.opts.sort ? match.score : 0;
if (scoreMap[scoreKey] === undefined) {
scoreMap[scoreKey] = [];
}
scoreMap[scoreKey].push({
item: this.items[idx],
...match,
positions: positions ?? new Set(),
});
};
}
function getExtendedMatchIter<U>(
this: BaseFinder<ReadonlyArray<U>>,
scoreMap: Record<number, Array<FzfResultItem<U>>>,
pattern: ReturnType<typeof buildPatternForExtendedMatch>
) {
return (idx: number) => {
const runes = this.runesList[idx];
const match = computeExtendedMatch(runes, pattern, this.algoFn, this.opts.forward);
if (match.offsets.length !== pattern.termSets.length) {
return;
}
let sidx = -1;
let eidx = -1;
if (match.allPos.size > 0) {
sidx = Math.min(...match.allPos);
eidx = Math.max(...match.allPos) + 1;
}
const scoreKey = this.opts.sort ? match.totalScore : 0;
if (scoreMap[scoreKey] === undefined) {
scoreMap[scoreKey] = [];
}
scoreMap[scoreKey].push({
score: match.totalScore,
item: this.items[idx],
positions: match.allPos,
start: sidx,
end: eidx,
});
};
}
// Sync matchers:
export function basicMatch<U>(this: SyncFinder<ReadonlyArray<U>>, query: string) {
const { queryRunes, caseSensitive } = buildPatternForBasicMatch(
query,
this.opts.casing,
this.opts.normalize
);
const scoreMap: Record<number, Array<FzfResultItem<U>>> = {};
const iter = getBasicMatchIter.bind(this as BaseFinder<ReadonlyArray<U>>)(
scoreMap,
queryRunes,
caseSensitive
);
for (let i = 0, len = this.runesList.length; i < len; ++i) {
iter(i);
}
return getResultFromScoreMap(scoreMap, this.opts.limit);
}
export function extendedMatch<U>(this: SyncFinder<ReadonlyArray<U>>, query: string) {
const pattern = buildPatternForExtendedMatch(
Boolean(this.opts.fuzzy),
this.opts.casing,
this.opts.normalize,
query
);
const scoreMap: Record<number, Array<FzfResultItem<U>>> = {};
const iter = getExtendedMatchIter.bind(this as BaseFinder<ReadonlyArray<U>>)(scoreMap, pattern);
for (let i = 0, len = this.runesList.length; i < len; ++i) {
iter(i);
}
return getResultFromScoreMap(scoreMap, this.opts.limit);
}
// Async matchers:
const isNode =
// @ts-expect-error TS is configured for browsers so `require` is not present.
// This is also why we aren't using @ts-expect-error
typeof require !== "undefined" && typeof window === "undefined";
function asyncMatcher<F>(
token: Token,
len: number,
iter: (index: number) => unknown,
onFinish: () => F
): Promise<F> {
return new Promise((resolve, reject) => {
const INCREMENT = 1000;
let i = 0;
let end = Math.min(INCREMENT, len);
const step = () => {
if (token.cancelled) {
reject("search cancelled");
return;
}
for (; i < end; ++i) {
iter(i);
}
if (end < len) {
end = Math.min(end + INCREMENT, len);
isNode
? // @ts-expect-error unavailable or deprecated for browsers
setImmediate(step)
: setTimeout(step);
} else {
resolve(onFinish());
}
};
step();
});
}
export function asyncBasicMatch<U>(
this: AsyncFinder<ReadonlyArray<U>>,
query: string,
token: Token
): Promise<Array<FzfResultItem<U>>> {
const { queryRunes, caseSensitive } = buildPatternForBasicMatch(
query,
this.opts.casing,
this.opts.normalize
);
const scoreMap: Record<number, Array<FzfResultItem<U>>> = {};
return asyncMatcher(
token,
this.runesList.length,
getBasicMatchIter.bind(this as BaseFinder<ReadonlyArray<U>>)(
scoreMap,
queryRunes,
caseSensitive
),
() => getResultFromScoreMap(scoreMap, this.opts.limit)
);
}
export function asyncExtendedMatch<U>(
this: AsyncFinder<ReadonlyArray<U>>,
query: string,
token: Token
) {
const pattern = buildPatternForExtendedMatch(
Boolean(this.opts.fuzzy),
this.opts.casing,
this.opts.normalize,
query
);
const scoreMap: Record<number, Array<FzfResultItem<U>>> = {};
return asyncMatcher(
token,
this.runesList.length,
getExtendedMatchIter.bind(this as BaseFinder<ReadonlyArray<U>>)(scoreMap, pattern),
() => getResultFromScoreMap(scoreMap, this.opts.limit)
);
}
+237
View File
@@ -0,0 +1,237 @@
/* eslint-disable */
// @ts-nocheck
import type { Rune } from "./runes";
const normalized: Record<number, string> = {
0x00d8: "O",
0x00df: "s",
0x00f8: "o",
0x0111: "d",
0x0127: "h",
0x0131: "i",
0x0140: "l",
0x0142: "l",
0x0167: "t",
0x017f: "s",
0x0180: "b",
0x0181: "B",
0x0183: "b",
0x0186: "O",
0x0188: "c",
0x0189: "D",
0x018a: "D",
0x018c: "d",
0x018e: "E",
0x0190: "E",
0x0192: "f",
0x0193: "G",
0x0197: "I",
0x0199: "k",
0x019a: "l",
0x019c: "M",
0x019d: "N",
0x019e: "n",
0x019f: "O",
0x01a5: "p",
0x01ab: "t",
0x01ad: "t",
0x01ae: "T",
0x01b2: "V",
0x01b4: "y",
0x01b6: "z",
0x01dd: "e",
0x01e5: "g",
0x0220: "N",
0x0221: "d",
0x0225: "z",
0x0234: "l",
0x0235: "n",
0x0236: "t",
0x0237: "j",
0x023a: "A",
0x023b: "C",
0x023c: "c",
0x023d: "L",
0x023e: "T",
0x023f: "s",
0x0240: "z",
0x0243: "B",
0x0244: "U",
0x0245: "V",
0x0246: "E",
0x0247: "e",
0x0248: "J",
0x0249: "j",
0x024a: "Q",
0x024b: "q",
0x024c: "R",
0x024d: "r",
0x024e: "Y",
0x024f: "y",
0x0250: "a",
0x0251: "a",
0x0253: "b",
0x0254: "o",
0x0255: "c",
0x0256: "d",
0x0257: "d",
0x0258: "e",
0x025b: "e",
0x025c: "e",
0x025d: "e",
0x025e: "e",
0x025f: "j",
0x0260: "g",
0x0261: "g",
0x0262: "G",
0x0265: "h",
0x0266: "h",
0x0268: "i",
0x026a: "I",
0x026b: "l",
0x026c: "l",
0x026d: "l",
0x026f: "m",
0x0270: "m",
0x0271: "m",
0x0272: "n",
0x0273: "n",
0x0274: "N",
0x0275: "o",
0x0279: "r",
0x027a: "r",
0x027b: "r",
0x027c: "r",
0x027d: "r",
0x027e: "r",
0x027f: "r",
0x0280: "R",
0x0281: "R",
0x0282: "s",
0x0287: "t",
0x0288: "t",
0x0289: "u",
0x028b: "v",
0x028c: "v",
0x028d: "w",
0x028e: "y",
0x028f: "Y",
0x0290: "z",
0x0291: "z",
0x0297: "c",
0x0299: "B",
0x029a: "e",
0x029b: "G",
0x029c: "H",
0x029d: "j",
0x029e: "k",
0x029f: "L",
0x02a0: "q",
0x02ae: "h",
0x0363: "a",
0x0364: "e",
0x0365: "i",
0x0366: "o",
0x0367: "u",
0x0368: "c",
0x0369: "d",
0x036a: "h",
0x036b: "m",
0x036c: "r",
0x036d: "t",
0x036e: "v",
0x036f: "x",
0x1d00: "A",
0x1d03: "B",
0x1d04: "C",
0x1d05: "D",
0x1d07: "E",
0x1d08: "e",
0x1d09: "i",
0x1d0a: "J",
0x1d0b: "K",
0x1d0c: "L",
0x1d0d: "M",
0x1d0e: "N",
0x1d0f: "O",
0x1d10: "O",
0x1d11: "o",
0x1d12: "o",
0x1d13: "o",
0x1d16: "o",
0x1d17: "o",
0x1d18: "P",
0x1d19: "R",
0x1d1a: "R",
0x1d1b: "T",
0x1d1c: "U",
0x1d1d: "u",
0x1d1e: "u",
0x1d1f: "m",
0x1d20: "V",
0x1d21: "W",
0x1d22: "Z",
0x1d62: "i",
0x1d63: "r",
0x1d64: "u",
0x1d65: "v",
0x1e9a: "a",
0x1e9b: "s",
0x2071: "i",
0x2095: "h",
0x2096: "k",
0x2097: "l",
0x2098: "m",
0x2099: "n",
0x209a: "p",
0x209b: "s",
0x209c: "t",
0x2184: "c",
};
for (let i = "\u0300".codePointAt(0)!; i <= "\u036F".codePointAt(0)!; ++i) {
const diacritic = String.fromCodePoint(i);
for (const asciiChar of "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz") {
const withDiacritic = (asciiChar + diacritic).normalize();
const withDiacriticCodePoint = withDiacritic.codePointAt(0)!;
if (withDiacriticCodePoint > 126) {
normalized[withDiacriticCodePoint] = asciiChar;
}
}
}
const ranges: Record<string, [number, number]> = {
a: [7844, 7863],
e: [7870, 7879],
o: [7888, 7907],
u: [7912, 7921],
};
for (const lowerChar of Object.keys(ranges)) {
const upperChar = lowerChar.toUpperCase();
for (let i = ranges[lowerChar][0]; i <= ranges[lowerChar][1]; ++i) {
normalized[i] = i % 2 === 0 ? upperChar : lowerChar;
}
}
export function normalizeRune(rune: Rune): Rune {
if (rune < 0x00c0 || rune > 0x2184) {
return rune;
}
// while a char can be converted to hex using str.charCodeAt().toString(16), it is not needed
// because in `normalized` map those hex in keys will be converted to decimals.
// Also we are passing a number instead of a converting a char so the above line doesn't apply (and that
// we are using codePointAt instead of charCodeAt)
const normalizedChar = normalized[rune];
if (normalizedChar !== undefined) {
return normalizedChar.codePointAt(0)!;
}
return rune;
}
+40
View File
@@ -0,0 +1,40 @@
/* eslint-disable */
// @ts-nocheck
export type Int16 = Int16Array[0];
export type Int32 = Int32Array[0];
// for short, int, long naming convention https://docs.oracle.com/javase/tutorial/java/nutsandbolts/datatypes.html
export function toShort(number: number): Int16 {
//
// // with this implementation, I don't think it does anything
// // as it is returning a number only, not int16
// const int16 = new Int16Array(1);
// int16[0] = number;
// return int16[0];
//
return number;
}
export function toInt(number: number): Int32 {
//
// // with this implementation, I don't think it does anything
// // as it is returning a number only, not int32
// const int32 = new Int32Array(1);
// int32[0] = number;
// return int32[0];
//
return number;
}
export function maxInt16(num1: number, num2: number) {
//
// // with this implementation, I don't think it does anything
// // as it is returning a number only, not int16
// const arr = Int16Array.from([num1, num2]);
// return arr[0] > arr[1] ? arr[0] : arr[1];
// // also converting to int16 just for comparison accumulates
// // overhead if done thousands of times
//
return num1 > num2 ? num1 : num2;
}
+248
View File
@@ -0,0 +1,248 @@
/* eslint-disable */
// @ts-nocheck
import { equalMatch, exactMatchNaive, fuzzyMatchV2, prefixMatch, suffixMatch } from "./algo";
import { normalizeRune } from "./normalize";
import { runesToStr, strToRunes } from "./runes";
import type { Rune } from "./runes";
import type { Casing } from "./types";
export enum TermType {
Fuzzy,
Exact,
Prefix,
Suffix,
Equal,
}
export const termTypeMap = {
[TermType.Fuzzy]: fuzzyMatchV2,
[TermType.Exact]: exactMatchNaive,
[TermType.Prefix]: prefixMatch,
[TermType.Suffix]: suffixMatch,
[TermType.Equal]: equalMatch,
};
interface Term {
typ: TermType;
inv: boolean;
text: Array<Rune>;
caseSensitive: boolean;
normalize: boolean;
}
type TermSet = Array<Term>;
export function buildPatternForExtendedMatch(
fuzzy: boolean,
caseMode: Casing,
normalize: boolean,
str: string
) {
// TODO Implement caching here and below.
// cacheable is received from caller of this fn
let cacheable = true;
str = str.trimLeft();
// while(str.endsWith(' ') && !str.endsWith('\\ ')) {
// str= str.substring(0, str.length - 1)
// }
// ^^ simplified below:
{
const trimmedAtRightStr = str.trimRight();
if (trimmedAtRightStr.endsWith("\\") && str[trimmedAtRightStr.length] === " ") {
str = `${trimmedAtRightStr} `;
} else {
str = trimmedAtRightStr;
}
}
// TODO cache not implemented here
// https://github.com/junegunn/fzf/blob/7191ebb615f5d6ebbf51d598d8ec853a65e2274d/src/pattern.go#L100-L103
// to implement cache, search for all cache word uses in pattern.go
// sortable turns to false initially in junegunn/fzf for extended matches
let sortable = false;
let termSets: Array<TermSet> = [];
termSets = parseTerms(fuzzy, caseMode, normalize, str);
Loop: for (const termSet of termSets) {
for (const [idx, term] of termSet.entries()) {
if (!term.inv) {
sortable = true;
}
if (
!cacheable ||
idx > 0 ||
term.inv ||
(fuzzy && term.typ !== TermType.Fuzzy) ||
(!fuzzy && term.typ !== TermType.Exact)
) {
cacheable = false;
if (sortable) {
break Loop;
}
}
}
}
return {
// this modified str can be used as cache as pattern cache
// see https://github.com/junegunn/fzf/blob/7191ebb615f5d6ebbf51d598d8ec853a65e2274d/src/pattern.go#L100
//
// there is also buildCacheKey https://github.com/junegunn/fzf/blob/7191ebb615f5d6ebbf51d598d8ec853a65e2274d/src/pattern.go#L261
// which i believe has a different purpose
str,
// ^ this in junegunn/fzf is `text: []rune(asString)`
termSets,
sortable,
cacheable,
fuzzy,
};
}
function parseTerms(
fuzzy: boolean,
caseMode: Casing,
normalize: boolean,
str: string
): Array<TermSet> {
// <backslash><space> to a <tab>
str = str.replace(/\\ /g, "\t");
// split on space groups
const tokens = str.split(/ +/);
const sets: Array<TermSet> = [];
let set: TermSet = [];
let switchSet = false;
let afterBar = false;
for (const token of tokens) {
let typ = TermType.Fuzzy;
let inv = false;
let text = token.replace(/\t/g, " ");
const lowerText = text.toLowerCase();
const caseSensitive =
caseMode === "case-sensitive" || (caseMode === "smart-case" && text !== lowerText);
// TODO double conversion here, could be simplified
const normalizeTerm =
normalize && lowerText === runesToStr(strToRunes(lowerText).map(normalizeRune));
if (!caseSensitive) {
text = lowerText;
}
if (!fuzzy) {
typ = TermType.Exact;
}
if (set.length > 0 && !afterBar && text === "|") {
switchSet = false;
afterBar = true;
continue;
}
afterBar = false;
if (text.startsWith("!")) {
inv = true;
typ = TermType.Exact;
text = text.substring(1);
}
if (text !== "$" && text.endsWith("$")) {
typ = TermType.Suffix;
text = text.substring(0, text.length - 1);
}
if (text.startsWith("'")) {
if (fuzzy && !inv) {
typ = TermType.Exact;
} else {
typ = TermType.Fuzzy;
}
text = text.substring(1);
} else if (text.startsWith("^")) {
if (typ === TermType.Suffix) {
typ = TermType.Equal;
} else {
typ = TermType.Prefix;
}
text = text.substring(1);
}
if (text.length > 0) {
if (switchSet) {
sets.push(set);
set = [];
}
let textRunes = strToRunes(text);
if (normalizeTerm) {
textRunes = textRunes.map(normalizeRune);
}
set.push({
typ,
inv,
text: textRunes,
caseSensitive,
normalize: normalizeTerm,
});
switchSet = true;
}
}
if (set.length > 0) {
sets.push(set);
}
return sets;
}
export const buildPatternForBasicMatch = (query: string, casing: Casing, normalize: boolean) => {
let caseSensitive = false;
switch (casing) {
case "smart-case": {
if (query.toLowerCase() !== query) {
caseSensitive = true;
}
break;
}
case "case-sensitive": {
caseSensitive = true;
break;
}
case "case-insensitive": {
query = query.toLowerCase();
caseSensitive = false;
break;
}
}
let queryRunes = strToRunes(query);
if (normalize) {
queryRunes = queryRunes.map(normalizeRune);
}
return {
queryRunes,
caseSensitive,
};
};
+13
View File
@@ -0,0 +1,13 @@
/* eslint-disable */
// @ts-nocheck
import type { Int32 } from "./numerics";
export type Rune = Int32;
// This fn should give Int32[] not number[] but this is still okay in current
// state as not many times a rune array is intended to be subarray-ed in the
// code.
export const strToRunes = (str: string) => str.split("").map((s) => s.codePointAt(0)!);
export const runesToStr = (runes: Array<Rune>) =>
runes.map((r) => String.fromCodePoint(r)).join("");
+24
View File
@@ -0,0 +1,24 @@
/* eslint-disable */
// @ts-nocheck
export interface Slab {
i16: Int16Array;
i32: Int32Array;
}
// from https://github.com/junegunn/fzf/blob/764316a53d0eb60b315f0bbcd513de58ed57a876/src/constants.go#L40
const SLAB_16_SIZE = 100 * 1024; // 200KB * 32 = 12.8MB
const SLAB_32_SIZE = 2048; // 8KB * 32 = 256KB
function makeSlab(size16: number, size32: number): Slab {
return {
i16: new Int16Array(size16),
i32: new Int32Array(size32),
};
}
// TODO maybe: do not initialise slab unless an fzf algo that needs slab gets called
//
// seems like a slab can be reused **without** setting its arrs' values to 0
// everytime we call algo fn
export const slab = makeSlab(SLAB_16_SIZE, SLAB_32_SIZE);
+16
View File
@@ -0,0 +1,16 @@
/* eslint-disable */
// @ts-nocheck
import type { FzfResultItem, Selector } from "./types";
export function byLengthAsc<U>(
a: FzfResultItem<U>,
b: FzfResultItem<U>,
selector: Selector<U>
): number {
return selector(a.item).length - selector(b.item).length;
}
export function byStartAsc<U>(a: FzfResultItem<U>, b: FzfResultItem<U>): number {
return a.start - b.start;
}
+157
View File
@@ -0,0 +1,157 @@
/* eslint-disable */
// @ts-nocheck
import type { Result } from "./algo";
import type { SyncFinder, AsyncFinder } from "./finders";
export interface Token {
cancelled: boolean;
}
export type Casing = "case-insensitive" | "case-sensitive" | "smart-case";
export interface FzfResultItem<U = string> extends Result {
item: U;
positions: Set<number>;
}
export type Selector<U> = BaseOptions<U>["selector"];
export type Tiebreaker<U> = (
a: FzfResultItem<U>,
b: FzfResultItem<U>,
selector: Selector<U>
) => number;
export interface BaseOptions<U> {
/**
* If `limit` is 32, top 32 items that matches your query will be returned.
* By default all matched items are returned.
*
* @defaultValue `Infinity`
*/
limit: number;
/**
* For each item in the list, target a specific property of the item to search for.
*/
selector: (v: U) => string;
/**
* Defines what type of case sensitive search you want.
*
* @defaultValue `"smart-case"`
*/
casing: Casing;
/**
* If true, FZF will try to remove diacritics from list items.
* This is useful if the list contains items with diacritics but
* you want to query with plain A-Z letters.
*
* @example
* Zoë → Zoe
* blessèd → blessed
*
* @defaultValue `true`
*/
normalize: boolean;
/**
* Fuzzy algo to choose. Each algo has their own advantages, see here:
* https://github.com/junegunn/fzf/blob/4c9cab3f8ae7b55f7124d7c3cf7ac6b4cc3db210/src/algo/algo.go#L5
* If asssigned `false`, an exact match will be made instead of a fuzzy one.
*
* @defaultValue `"v2"`
*/
fuzzy: "v1" | "v2" | false;
/**
* A list of functions that act as fallback and help to
* sort result entries when the score between two entries is tied.
*
* Consider a tiebreaker to be a [JS array sort](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/sort)
* compare function with an added third argument which is `options.selector`.
*
* If multiple tiebreakers are given, they are evaluated left to right until
* one breaks the tie.
*
* Note that tiebreakers cannot be used if `sort=false`.
*
* FZF ships with these tiebreakers:
* - `byLengthAsc`
* - `byStartAsc`
*
* @defaultValue `[]`
*
* @example
* ```js
* function byLengthAsc(a, b, selector) {
* return selector(a.item).length - selector(b.item).length;
* }
*
* const fzf = new Fzf(list, { tiebreakers: [byLengthAsc] })
* ```
* This will result in following result entries having same score sorted like this:
* FROM TO
* axaa axaa
* bxbbbb bxbbbb
* cxcccccccccc dxddddddd
* dxddddddd cxcccccccccc
*/
tiebreakers: Array<Tiebreaker<U>>;
/**
* If `true`, result items will be sorted in descending order by their score.
*
* If `false`, result items won't be sorted and tiebreakers won't affect the
* sort order either. In this case, the items are returned in the same order
* as they are in the input list.
*
* @defaultValue `true`
*/
sort: boolean;
/**
* If `false`, matching will be done from backwards.
*
* @defaultValue `true`
*
* @example
* /breeds/pyrenees when queried with "re"
* with forward=true : /b**re**eds/pyrenees
* with forward=false : /breeds/py**re**nees
*
* Doing forward=false is useful, for example, if one needs to match a file
* path and they prefer querying for the file name over directory names
* present in the path.
*/
forward: boolean;
}
export type SyncOptions<U> = BaseOptions<U> & {
/**
* A function that is responsible for matching list items with the query.
*
* We ship with two match functions - `basicMatch` and `extendedMatch`.
*
* If `extendedMatch` is used, you can add special patterns to narrow down your search.
* To read about how they can be used, see [this section](https://github.com/junegunn/fzf/tree/7191ebb615f5d6ebbf51d598d8ec853a65e2274d#search-syntax).
* For a quick glance, see [this piece](https://github.com/junegunn/fzf/blob/764316a53d0eb60b315f0bbcd513de58ed57a876/src/pattern.go#L12-L19).
*
* @defaultValue `basicMatch`
*/
match: (this: SyncFinder<ReadonlyArray<U>>, query: string) => Array<FzfResultItem<U>>;
};
export type AsyncOptions<U> = BaseOptions<U> & {
/**
* A function that is responsible for matching list items with the query.
*
* We ship with two match functions - `asyncBasicMatch` and `asyncExtendedMatch`.
*
* If `asyncExtendedMatch` is used, you can add special patterns to narrow down your search.
* To read about how they can be used, see [this section](https://github.com/junegunn/fzf/tree/7191ebb615f5d6ebbf51d598d8ec853a65e2274d#search-syntax).
* For a quick glance, see [this piece](https://github.com/junegunn/fzf/blob/764316a53d0eb60b315f0bbcd513de58ed57a876/src/pattern.go#L12-L19).
*
* @defaultValue `asyncBasicMatch`
*/
match: (
this: AsyncFinder<ReadonlyArray<U>>,
query: string,
token: Token
) => Promise<Array<FzfResultItem<U>>>;
};
+1 -8
View File
@@ -1,12 +1,5 @@
/* eslint-disable import/first */
// see: https://www.npmjs.com/package/fix-esm
// fzf *mandates* esm, but we don't use it (ts transpiles to cjs)
// sick of this headache.
// eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires
require("fix-esm").register();
import { AsyncFzf } from "./fzf/main";
import db from "external/mongo/db";
import { AsyncFzf } from "fzf";
import CreateLogCtx from "lib/logger/logger";
import { TachiConfig } from "lib/setup/config";
import { CreateSongMap, SplitGPT } from "tachi-common";