From 1fd018a08aa2156a3384affe248a70b64754221b Mon Sep 17 00:00:00 2001 From: adueck Date: Wed, 2 Nov 2022 10:28:14 +0500 Subject: [PATCH] simpler ts query --- account/package-lock.json | 63 +++++++++++++ account/package.json | 2 + account/src/lib/dictionary.ts | 109 ++++++++++++++++++----- account/src/routers/dictionary-router.ts | 54 ++++------- account/yarn.lock | 25 ++++++ update-inflector.sh | 7 +- 6 files changed, 198 insertions(+), 62 deletions(-) diff --git a/account/package-lock.json b/account/package-lock.json index 906bf98..2dc0d41 100644 --- a/account/package-lock.json +++ b/account/package-lock.json @@ -14,6 +14,7 @@ "bcryptjs": "^2.4.3", "connect-redis": "^6.0.0", "cors": "^2.8.5", + "cron": "^2.1.0", "crypto": "^1.0.1", "ejs": "^3.1.6", "express-session": "^1.17.2", @@ -37,6 +38,7 @@ "@types/base64url": "^2.0.0", "@types/bcryptjs": "^2.4.2", "@types/cors": "^2.8.12", + "@types/cron": "^2.0.0", "@types/express": "^4.17.13", "@types/express-session": "^1.17.4", "@types/node": "^16.6.0", @@ -611,6 +613,16 @@ "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==", "dev": true }, + "node_modules/@types/cron": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@types/cron/-/cron-2.0.0.tgz", + "integrity": "sha512-xZM08fqvwIXgghtPVkSPKNgC+JoMQ2OHazEvyTKnNf7aWu1aB6/4lBbQFrb03Td2cUGG7ITzMv3mFYnMu6xRaQ==", + "dev": true, + "dependencies": { + "@types/luxon": "*", + "@types/node": "*" + } + }, "node_modules/@types/express": { "version": "4.17.13", "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", @@ -643,6 +655,12 @@ "@types/express": "*" } }, + "node_modules/@types/luxon": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.0.2.tgz", + "integrity": "sha512-HM2OVWckUMmXbWYZufmWT2XMURGDZ6XbyNyQ+Lx+gCFGFqbZaIjsz7b+AGeGP/AuVYHBiuGY+wXfweP1RremnA==", + "dev": true + }, "node_modules/@types/mime": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", @@ -1492,6 +1510,14 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "dev": true }, + "node_modules/cron": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cron/-/cron-2.1.0.tgz", + "integrity": "sha512-Hq7u3P8y7UWYvsZbSKHHJDVG0VO9O7tp2qljxzTScelcTODBfCme8AIhnZsFwmQ9NchZ3hr2uNr+s3DSms7q6w==", + "dependencies": { + "luxon": "^1.23.x" + } + }, "node_modules/crypto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/crypto/-/crypto-1.0.1.tgz", @@ -2837,6 +2863,14 @@ "loose-envify": "cli.js" } }, + "node_modules/luxon": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz", + "integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ==", + "engines": { + "node": "*" + } + }, "node_modules/make-dir": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", @@ -5443,6 +5477,16 @@ "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==", "dev": true }, + "@types/cron": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@types/cron/-/cron-2.0.0.tgz", + "integrity": "sha512-xZM08fqvwIXgghtPVkSPKNgC+JoMQ2OHazEvyTKnNf7aWu1aB6/4lBbQFrb03Td2cUGG7ITzMv3mFYnMu6xRaQ==", + "dev": true, + "requires": { + "@types/luxon": "*", + "@types/node": "*" + } + }, "@types/express": { "version": "4.17.13", "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz", @@ -5475,6 +5519,12 @@ "@types/express": "*" } }, + "@types/luxon": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.0.2.tgz", + "integrity": "sha512-HM2OVWckUMmXbWYZufmWT2XMURGDZ6XbyNyQ+Lx+gCFGFqbZaIjsz7b+AGeGP/AuVYHBiuGY+wXfweP1RremnA==", + "dev": true + }, "@types/mime": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", @@ -6217,6 +6267,14 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "dev": true }, + "cron": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cron/-/cron-2.1.0.tgz", + "integrity": "sha512-Hq7u3P8y7UWYvsZbSKHHJDVG0VO9O7tp2qljxzTScelcTODBfCme8AIhnZsFwmQ9NchZ3hr2uNr+s3DSms7q6w==", + "requires": { + "luxon": "^1.23.x" + } + }, "crypto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/crypto/-/crypto-1.0.1.tgz", @@ -7221,6 +7279,11 @@ "js-tokens": "^3.0.0 || ^4.0.0" } }, + "luxon": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz", + "integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ==" + }, "make-dir": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", diff --git a/account/package.json b/account/package.json index bdcf601..ad49cea 100644 --- a/account/package.json +++ b/account/package.json @@ -16,6 +16,7 @@ "bcryptjs": "^2.4.3", "connect-redis": "^6.0.0", "cors": "^2.8.5", + "cron": "^2.1.0", "crypto": "^1.0.1", "ejs": "^3.1.6", "express-session": "^1.17.2", @@ -39,6 +40,7 @@ "@types/base64url": "^2.0.0", "@types/bcryptjs": "^2.4.2", "@types/cors": "^2.8.12", + "@types/cron": "^2.0.0", "@types/express": "^4.17.13", "@types/express-session": "^1.17.4", "@types/node": "^16.6.0", diff --git a/account/src/lib/dictionary.ts b/account/src/lib/dictionary.ts index 3fffaac..d1079fb 100644 --- a/account/src/lib/dictionary.ts +++ b/account/src/lib/dictionary.ts @@ -1,9 +1,12 @@ import loki, { Collection, LokiMemoryAdapter } from "lokijs"; import fetch from "node-fetch"; +import { CronJob } from "cron"; const collectionName = "ps-dictionary"; import { readDictionary, + readDictionaryInfo, Types as T, + typePredicates as tp, } from "@lingdocs/inflect" export let collection: Collection | undefined = undefined; @@ -15,32 +18,90 @@ const lokidb = new loki("", { env: "NODEJS", }); -export let dictionary: T.Dictionary | undefined = undefined; +const updateJob = new CronJob("* * * * *", updateDictionary, null, false); -// TODO: Abstract dictionary fetch -export function updateDictionary() { - fetch(process.env.LINGDOCS_DICTIONARY_URL || "").then(res => res.arrayBuffer()).then(buffer => { - const dict = readDictionary(buffer as Uint8Array); - dictionary = dict; - collection?.clear(); - lokidb.removeCollection(collectionName); - collection?.insert(dictionary.entries); - }).catch(console.error); +let version: number = 0; + +async function fetchDictionary(): Promise { + const res = await fetch(process.env.LINGDOCS_DICTIONARY_URL || ""); + const buffer = await res.arrayBuffer(); + return readDictionary(buffer as Uint8Array); } -lokidb.loadDatabase({}, async (err: Error) => { - collection = lokidb.getCollection(collectionName); - if (!collection) { - collection = lokidb.addCollection(collectionName, { - // TODO: THIS ISN'T WORKING! - disableMeta: true, - indices: ["i", "p"], - unique: ["ts"], - }); - fetch(process.env.LINGDOCS_DICTIONARY_URL || "").then(res => res.arrayBuffer()).then(buffer => { - const dict = readDictionary(buffer as Uint8Array); - dictionary = dict; - collection?.insert(dictionary.entries); - }).catch(console.error); +async function fetchDictionaryInfo(): Promise { + const res = await fetch(process.env.LINGDOCS_DICTIONARY_URL + "-info" || ""); + const buffer = await res.arrayBuffer(); + return readDictionaryInfo(buffer as Uint8Array); +} + +export async function updateDictionary(): Promise<"no update" | "updated"> { + const info = await fetchDictionaryInfo(); + if (info.release === version) { + return "no update"; } + const dictionary = await fetchDictionary(); + version = dictionary.info.release; + collection?.clear(); + lokidb.removeCollection(collectionName); + collection?.insert(dictionary.entries); + return "updated"; +} + +function getOneByTs(ts: number): T.DictionaryEntry { + if (!collection) { + throw new Error("dictionary not initialized"); + } + const r = collection.by("ts", ts); + const { $loki, meta, ...entry } = r; + return entry; +} + +export async function getEntries(ids: number[]): Promise<{ + results: (T.DictionaryEntry | T.VerbEntry)[], + notFound: number[], +}> { + if (!collection) { + throw new Error("dictionary not initialized"); + } + const results: (T.DictionaryEntry | T.VerbEntry)[] = collection.find({ + "ts": { "$in": ids }, + }).map(x => { + const { $loki, meta, ...entry } = x; + return entry; + }).map((entry): T.DictionaryEntry | T.VerbEntry => { + if (tp.isVerbDictionaryEntry(entry)) { + if (entry.c?.includes("comp.") && entry.l) { + const complement = getOneByTs(entry.l); + if (!complement) throw new Error("Error getting complement "+entry.l); + return { + entry, + complement, + }; + } + return { entry }; + } else { + return entry; + } + }); + return { + results, + notFound: ids.filter(id => !results.find(x => ( + "entry" in x ? x.entry.ts === id : x.ts === id + ))), + }; +} + +lokidb.loadDatabase({}, (err: Error) => { + lokidb.removeCollection(collectionName); + collection = lokidb.addCollection(collectionName, { + // TODO: THIS ISN'T WORKING! + disableMeta: true, + indices: ["i", "p"], + unique: ["ts"], + }); + fetchDictionary().then((dictionary) => { + version = dictionary.info.release; + collection?.insert(dictionary.entries); + updateJob.start(); + }).catch(console.error); }); diff --git a/account/src/routers/dictionary-router.ts b/account/src/routers/dictionary-router.ts index 4ed0c55..5b464da 100644 --- a/account/src/routers/dictionary-router.ts +++ b/account/src/routers/dictionary-router.ts @@ -1,55 +1,37 @@ -import express, { Response } from "express"; +import express from "express"; import { collection, - dictionary, + getEntries, updateDictionary, } from "../lib/dictionary"; import { unary } from "froebel"; const dictionaryRouter = express.Router(); -// Guard all api with authentication -dictionaryRouter.get("/", async (req, res, next) => { - if (!dictionary) { - return res.send({ ok: false, message: "dictionary not ready" }); - } - res.send(dictionary); -}) - -dictionaryRouter.get("/info", async (req, res, next) => { - if (!dictionary) { - return res.send({ ok: false, message: "dictionary not ready" }); - } - res.send({ info: dictionary.info }) -}); - dictionaryRouter.post("/update", async (req, res, next) => { - updateDictionary(); - res.send({ ok: true }); + const result = await updateDictionary(); + res.send({ ok: true, result }); }); -dictionaryRouter.get("/entry/:id", async (req, res, next) => { - const id = req.params.id.includes(",") - ? req.params.id.split(",").map(unary(parseInt)) - : parseInt(req.params.id); +dictionaryRouter.post("/entry", async (req, res, next) => { if (!collection) { return res.send({ ok: false, message: "dictionary not ready" }); } - if (Array.isArray(id)) { - const results = collection.find({ - "ts": { - "$in": id, - }, - }); - return res.send({ results }); + const ids = req.body.ids as number[]; + if (!Array.isArray(ids)) { + return res.status(400).send({ ok: false, error: "invalid query" }); } - const r = collection.by("ts", id); - if (!r) { - return res.send({ result: [] }); + const results = await getEntries(ids); + return res.send(results); +}); + +dictionaryRouter.get("/entry/:id", async (req, res, next) => { + if (!collection) { + return res.send({ ok: false, message: "dictionary not ready" }); } - // remove $loki and meta - const { $loki, meta, ...word } = r; - return res.send({ result: word }); + const ids = req.params.id.split(",").map(unary(parseInt)); + const results = await getEntries(ids); + return res.send(results); }); export default dictionaryRouter; \ No newline at end of file diff --git a/account/yarn.lock b/account/yarn.lock index ed68fbe..d389b81 100644 --- a/account/yarn.lock +++ b/account/yarn.lock @@ -207,6 +207,14 @@ "resolved" "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz" "version" "2.8.12" +"@types/cron@^2.0.0": + "integrity" "sha512-xZM08fqvwIXgghtPVkSPKNgC+JoMQ2OHazEvyTKnNf7aWu1aB6/4lBbQFrb03Td2cUGG7ITzMv3mFYnMu6xRaQ==" + "resolved" "https://registry.npmjs.org/@types/cron/-/cron-2.0.0.tgz" + "version" "2.0.0" + dependencies: + "@types/luxon" "*" + "@types/node" "*" + "@types/express-serve-static-core@^4.17.18": "integrity" "sha512-3UJuW+Qxhzwjq3xhwXm2onQcFHn76frIYVbTu+kn24LFxI+dEhdfISDFovPB8VpEgW8oQCTpRuCe+0zJxB7NEA==" "resolved" "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.24.tgz" @@ -233,6 +241,11 @@ "@types/qs" "*" "@types/serve-static" "*" +"@types/luxon@*": + "integrity" "sha512-HM2OVWckUMmXbWYZufmWT2XMURGDZ6XbyNyQ+Lx+gCFGFqbZaIjsz7b+AGeGP/AuVYHBiuGY+wXfweP1RremnA==" + "resolved" "https://registry.npmjs.org/@types/luxon/-/luxon-3.0.2.tgz" + "version" "3.0.2" + "@types/mime@^1": "integrity" "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==" "resolved" "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz" @@ -898,6 +911,13 @@ "resolved" "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz" "version" "1.1.1" +"cron@^2.1.0": + "integrity" "sha512-Hq7u3P8y7UWYvsZbSKHHJDVG0VO9O7tp2qljxzTScelcTODBfCme8AIhnZsFwmQ9NchZ3hr2uNr+s3DSms7q6w==" + "resolved" "https://registry.npmjs.org/cron/-/cron-2.1.0.tgz" + "version" "2.1.0" + dependencies: + "luxon" "^1.23.x" + "crypto-browserify@3.12.0": "integrity" "sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==" "resolved" "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz" @@ -1749,6 +1769,11 @@ dependencies: "js-tokens" "^3.0.0 || ^4.0.0" +"luxon@^1.23.x": + "integrity" "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ==" + "resolved" "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz" + "version" "1.28.0" + "make-dir@^3.0.2": "integrity" "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==" "resolved" "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz" diff --git a/update-inflector.sh b/update-inflector.sh index b290ba6..334dece 100755 --- a/update-inflector.sh +++ b/update-inflector.sh @@ -9,8 +9,6 @@ if [ $# -eq 0 ] version=$(npm show @lingdocs/ps-react version) fi - - # update all instances of @lingdocs/inflect and @lingdocs/ps-react in various package.json files # in website tmp=$(mktemp) @@ -19,10 +17,15 @@ jq --arg version "$version" '.dependencies."@lingdocs/ps-react"=$version' "websi # in functions tmp=$(mktemp) jq --arg version "$version" '.dependencies."@lingdocs/inflect"=$version' "functions/package.json" > "$tmp" && mv "$tmp" "functions/package.json" +# in account +tmp=$(mktemp) +jq --arg version "$version" '.dependencies."@lingdocs/inflect"=$version' "account/package.json" > "$tmp" && mv "$tmp" "account/package.json" # install to update .lock files cd website && yarn install --legacy-peer-deps && cd ../functions && +npm install && +cd ../account && npm install