add allWords database to account
This commit is contained in:
parent
091eacf6f0
commit
a1a6194717
|
@ -2,6 +2,7 @@ import loki, { Collection, LokiMemoryAdapter } from "lokijs";
|
||||||
import fetch from "node-fetch";
|
import fetch from "node-fetch";
|
||||||
import { CronJob } from "cron";
|
import { CronJob } from "cron";
|
||||||
const collectionName = "ps-dictionary";
|
const collectionName = "ps-dictionary";
|
||||||
|
const allWordsCollectionName = "all-words";
|
||||||
import {
|
import {
|
||||||
readDictionary,
|
readDictionary,
|
||||||
readDictionaryInfo,
|
readDictionaryInfo,
|
||||||
|
@ -11,7 +12,8 @@ import {
|
||||||
standardizePashto,
|
standardizePashto,
|
||||||
} from "@lingdocs/inflect"
|
} from "@lingdocs/inflect"
|
||||||
|
|
||||||
export let collection: Collection<any> | undefined = undefined;
|
export let collection: Collection<T.DictionaryEntry> | undefined = undefined;
|
||||||
|
export let allWordsCollection: Collection<T.PsString> | undefined = undefined;
|
||||||
const adapter = new LokiMemoryAdapter();
|
const adapter = new LokiMemoryAdapter();
|
||||||
const lokidb = new loki("", {
|
const lokidb = new loki("", {
|
||||||
adapter,
|
adapter,
|
||||||
|
@ -30,6 +32,11 @@ async function fetchDictionary(): Promise<T.Dictionary> {
|
||||||
return readDictionary(buffer as Uint8Array);
|
return readDictionary(buffer as Uint8Array);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function fetchAllWords(): Promise<T.AllWordsWithInflections> {
|
||||||
|
const res = await fetch(process.env.LINGDOCS_DICTIONARY_URL?.slice(0, -4) + "all-words.json");
|
||||||
|
return await res.json();
|
||||||
|
}
|
||||||
|
|
||||||
async function fetchDictionaryInfo(): Promise<T.DictionaryInfo> {
|
async function fetchDictionaryInfo(): Promise<T.DictionaryInfo> {
|
||||||
const res = await fetch(process.env.LINGDOCS_DICTIONARY_URL + "-info" || "");
|
const res = await fetch(process.env.LINGDOCS_DICTIONARY_URL + "-info" || "");
|
||||||
const buffer = await res.arrayBuffer();
|
const buffer = await res.arrayBuffer();
|
||||||
|
@ -54,10 +61,22 @@ function getOneByTs(ts: number): T.DictionaryEntry {
|
||||||
throw new Error("dictionary not initialized");
|
throw new Error("dictionary not initialized");
|
||||||
}
|
}
|
||||||
const r = collection.by("ts", ts);
|
const r = collection.by("ts", ts);
|
||||||
|
// @ts-ignore
|
||||||
const { $loki, meta, ...entry } = r;
|
const { $loki, meta, ...entry } = r;
|
||||||
return entry;
|
return entry;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function findInAllWords(p: string | RegExp): T.PsWord[] | undefined {
|
||||||
|
if (!allWordsCollection) {
|
||||||
|
throw new Error("allWords not initialized");
|
||||||
|
}
|
||||||
|
return allWordsCollection.find({
|
||||||
|
p: typeof p === "string"
|
||||||
|
? p
|
||||||
|
: { $regex: p },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export async function getEntries(ids: (number | string)[]): Promise<{
|
export async function getEntries(ids: (number | string)[]): Promise<{
|
||||||
results: (T.DictionaryEntry | T.VerbEntry)[],
|
results: (T.DictionaryEntry | T.VerbEntry)[],
|
||||||
notFound: (number | string)[],
|
notFound: (number | string)[],
|
||||||
|
@ -100,15 +119,20 @@ export async function getEntries(ids: (number | string)[]): Promise<{
|
||||||
|
|
||||||
lokidb.loadDatabase({}, (err: Error) => {
|
lokidb.loadDatabase({}, (err: Error) => {
|
||||||
lokidb.removeCollection(collectionName);
|
lokidb.removeCollection(collectionName);
|
||||||
collection = lokidb.addCollection(collectionName, {
|
lokidb.removeCollection(allWordsCollectionName);
|
||||||
// TODO: THIS ISN'T WORKING!
|
|
||||||
disableMeta: true,
|
|
||||||
indices: ["i", "p"],
|
|
||||||
unique: ["ts"],
|
|
||||||
});
|
|
||||||
fetchDictionary().then((dictionary) => {
|
fetchDictionary().then((dictionary) => {
|
||||||
|
collection = lokidb.addCollection(collectionName, {
|
||||||
|
indices: ["i", "p"],
|
||||||
|
unique: ["ts"],
|
||||||
|
});
|
||||||
version = dictionary.info.release;
|
version = dictionary.info.release;
|
||||||
collection?.insert(dictionary.entries);
|
collection?.insert(dictionary.entries);
|
||||||
updateJob.start();
|
updateJob.start();
|
||||||
}).catch(console.error);
|
}).catch(console.error);
|
||||||
|
fetchAllWords().then((allWords) => {
|
||||||
|
allWordsCollection = lokidb.addCollection(allWordsCollectionName, {
|
||||||
|
indices: ["p"],
|
||||||
|
});
|
||||||
|
allWordsCollection?.insert(allWords.words);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
|
@ -1,6 +1,8 @@
|
||||||
import express from "express";
|
import express from "express";
|
||||||
import {
|
import {
|
||||||
|
allWordsCollection,
|
||||||
collection,
|
collection,
|
||||||
|
findInAllWords,
|
||||||
getEntries,
|
getEntries,
|
||||||
updateDictionary,
|
updateDictionary,
|
||||||
} from "../lib/dictionary";
|
} from "../lib/dictionary";
|
||||||
|
@ -12,6 +14,18 @@ dictionaryRouter.post("/update", async (req, res, next) => {
|
||||||
res.send({ ok: true, result });
|
res.send({ ok: true, result });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
dictionaryRouter.post("/all-words", async (req, res, next) => {
|
||||||
|
if (!allWordsCollection) {
|
||||||
|
return res.send({ ok: false, message: "allWords not ready" });
|
||||||
|
}
|
||||||
|
const word = req.body.word as string;
|
||||||
|
if (!word) {
|
||||||
|
return res.status(400).send({ ok: false, error: "invalid query" });
|
||||||
|
}
|
||||||
|
const results = await findInAllWords(word);
|
||||||
|
res.send(results);
|
||||||
|
})
|
||||||
|
|
||||||
dictionaryRouter.post("/entries", async (req, res, next) => {
|
dictionaryRouter.post("/entries", async (req, res, next) => {
|
||||||
if (!collection) {
|
if (!collection) {
|
||||||
return res.send({ ok: false, message: "dictionary not ready" });
|
return res.send({ ok: false, message: "dictionary not ready" });
|
||||||
|
|
Loading…
Reference in New Issue