add allWords database to account

This commit is contained in:
adueck 2023-01-29 22:46:27 +05:00
parent 091eacf6f0
commit a1a6194717
2 changed files with 45 additions and 7 deletions

View File

@ -2,6 +2,7 @@ import loki, { Collection, LokiMemoryAdapter } from "lokijs";
import fetch from "node-fetch"; import fetch from "node-fetch";
import { CronJob } from "cron"; import { CronJob } from "cron";
const collectionName = "ps-dictionary"; const collectionName = "ps-dictionary";
const allWordsCollectionName = "all-words";
import { import {
readDictionary, readDictionary,
readDictionaryInfo, readDictionaryInfo,
@ -11,7 +12,8 @@ import {
standardizePashto, standardizePashto,
} from "@lingdocs/inflect" } from "@lingdocs/inflect"
export let collection: Collection<any> | undefined = undefined; export let collection: Collection<T.DictionaryEntry> | undefined = undefined;
export let allWordsCollection: Collection<T.PsString> | undefined = undefined;
const adapter = new LokiMemoryAdapter(); const adapter = new LokiMemoryAdapter();
const lokidb = new loki("", { const lokidb = new loki("", {
adapter, adapter,
@ -30,6 +32,11 @@ async function fetchDictionary(): Promise<T.Dictionary> {
return readDictionary(buffer as Uint8Array); return readDictionary(buffer as Uint8Array);
} }
async function fetchAllWords(): Promise<T.AllWordsWithInflections> {
const res = await fetch(process.env.LINGDOCS_DICTIONARY_URL?.slice(0, -4) + "all-words.json");
return await res.json();
}
async function fetchDictionaryInfo(): Promise<T.DictionaryInfo> { async function fetchDictionaryInfo(): Promise<T.DictionaryInfo> {
const res = await fetch(process.env.LINGDOCS_DICTIONARY_URL + "-info" || ""); const res = await fetch(process.env.LINGDOCS_DICTIONARY_URL + "-info" || "");
const buffer = await res.arrayBuffer(); const buffer = await res.arrayBuffer();
@ -54,10 +61,22 @@ function getOneByTs(ts: number): T.DictionaryEntry {
throw new Error("dictionary not initialized"); throw new Error("dictionary not initialized");
} }
const r = collection.by("ts", ts); const r = collection.by("ts", ts);
// @ts-ignore
const { $loki, meta, ...entry } = r; const { $loki, meta, ...entry } = r;
return entry; return entry;
} }
export function findInAllWords(p: string | RegExp): T.PsWord[] | undefined {
if (!allWordsCollection) {
throw new Error("allWords not initialized");
}
return allWordsCollection.find({
p: typeof p === "string"
? p
: { $regex: p },
});
}
export async function getEntries(ids: (number | string)[]): Promise<{ export async function getEntries(ids: (number | string)[]): Promise<{
results: (T.DictionaryEntry | T.VerbEntry)[], results: (T.DictionaryEntry | T.VerbEntry)[],
notFound: (number | string)[], notFound: (number | string)[],
@ -100,15 +119,20 @@ export async function getEntries(ids: (number | string)[]): Promise<{
lokidb.loadDatabase({}, (err: Error) => { lokidb.loadDatabase({}, (err: Error) => {
lokidb.removeCollection(collectionName); lokidb.removeCollection(collectionName);
lokidb.removeCollection(allWordsCollectionName);
fetchDictionary().then((dictionary) => {
collection = lokidb.addCollection(collectionName, { collection = lokidb.addCollection(collectionName, {
// TODO: THIS ISN'T WORKING!
disableMeta: true,
indices: ["i", "p"], indices: ["i", "p"],
unique: ["ts"], unique: ["ts"],
}); });
fetchDictionary().then((dictionary) => {
version = dictionary.info.release; version = dictionary.info.release;
collection?.insert(dictionary.entries); collection?.insert(dictionary.entries);
updateJob.start(); updateJob.start();
}).catch(console.error); }).catch(console.error);
fetchAllWords().then((allWords) => {
allWordsCollection = lokidb.addCollection(allWordsCollectionName, {
indices: ["p"],
});
allWordsCollection?.insert(allWords.words);
});
}); });

View File

@ -1,6 +1,8 @@
import express from "express"; import express from "express";
import { import {
allWordsCollection,
collection, collection,
findInAllWords,
getEntries, getEntries,
updateDictionary, updateDictionary,
} from "../lib/dictionary"; } from "../lib/dictionary";
@ -12,6 +14,18 @@ dictionaryRouter.post("/update", async (req, res, next) => {
res.send({ ok: true, result }); res.send({ ok: true, result });
}); });
dictionaryRouter.post("/all-words", async (req, res, next) => {
if (!allWordsCollection) {
return res.send({ ok: false, message: "allWords not ready" });
}
const word = req.body.word as string;
if (!word) {
return res.status(400).send({ ok: false, error: "invalid query" });
}
const results = await findInAllWords(word);
res.send(results);
})
dictionaryRouter.post("/entries", async (req, res, next) => { dictionaryRouter.post("/entries", async (req, res, next) => {
if (!collection) { if (!collection) {
return res.send({ ok: false, message: "dictionary not ready" }); return res.send({ ok: false, message: "dictionary not ready" });