lingdocs
/
pashto-dictionary
mirror of https://github.com/lingdocs/pashto-dictionary.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

simpler ts query

This commit is contained in:
adueck 2022-11-02 10:28:14 +05:00
parent 85f972e7e4
commit 1fd018a08a
6 changed files with 198 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
account/package-lock.json generated
View File

@ -14,6 +14,7 @@
"bcryptjs": "^2.4.3",
"connect-redis": "^6.0.0",
"cors": "^2.8.5",
"cron": "^2.1.0",
"crypto": "^1.0.1",
"ejs": "^3.1.6",
"express-session": "^1.17.2",
@ -37,6 +38,7 @@
"@types/base64url": "^2.0.0",
"@types/bcryptjs": "^2.4.2",
"@types/cors": "^2.8.12",
"@types/cron": "^2.0.0",
"@types/express": "^4.17.13",
"@types/express-session": "^1.17.4",
"@types/node": "^16.6.0",
@ -611,6 +613,16 @@
"integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==",
"dev": true
},
"node_modules/@types/cron": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/@types/cron/-/cron-2.0.0.tgz",
"integrity": "sha512-xZM08fqvwIXgghtPVkSPKNgC+JoMQ2OHazEvyTKnNf7aWu1aB6/4lBbQFrb03Td2cUGG7ITzMv3mFYnMu6xRaQ==",
"dev": true,
"dependencies": {
"@types/luxon": "*",
"@types/node": "*"
}
},
"node_modules/@types/express": {
"version": "4.17.13",
"resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz",
@ -643,6 +655,12 @@
"@types/express": "*"
}
},
"node_modules/@types/luxon": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.0.2.tgz",
"integrity": "sha512-HM2OVWckUMmXbWYZufmWT2XMURGDZ6XbyNyQ+Lx+gCFGFqbZaIjsz7b+AGeGP/AuVYHBiuGY+wXfweP1RremnA==",
"dev": true
},
"node_modules/@types/mime": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz",
@ -1492,6 +1510,14 @@
"integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
"dev": true
},
"node_modules/cron": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/cron/-/cron-2.1.0.tgz",
"integrity": "sha512-Hq7u3P8y7UWYvsZbSKHHJDVG0VO9O7tp2qljxzTScelcTODBfCme8AIhnZsFwmQ9NchZ3hr2uNr+s3DSms7q6w==",
"dependencies": {
"luxon": "^1.23.x"
}
},
"node_modules/crypto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/crypto/-/crypto-1.0.1.tgz",
@ -2837,6 +2863,14 @@
"loose-envify": "cli.js"
}
},
"node_modules/luxon": {
"version": "1.28.0",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz",
"integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ==",
"engines": {
"node": "*"
}
},
"node_modules/make-dir": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz",
@ -5443,6 +5477,16 @@
"integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==",
"dev": true
},
"@types/cron": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/@types/cron/-/cron-2.0.0.tgz",
"integrity": "sha512-xZM08fqvwIXgghtPVkSPKNgC+JoMQ2OHazEvyTKnNf7aWu1aB6/4lBbQFrb03Td2cUGG7ITzMv3mFYnMu6xRaQ==",
"dev": true,
"requires": {
"@types/luxon": "*",
"@types/node": "*"
}
},
"@types/express": {
"version": "4.17.13",
"resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.13.tgz",
@ -5475,6 +5519,12 @@
"@types/express": "*"
}
},
"@types/luxon": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.0.2.tgz",
"integrity": "sha512-HM2OVWckUMmXbWYZufmWT2XMURGDZ6XbyNyQ+Lx+gCFGFqbZaIjsz7b+AGeGP/AuVYHBiuGY+wXfweP1RremnA==",
"dev": true
},
"@types/mime": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz",
@ -6217,6 +6267,14 @@
"integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
"dev": true
},
"cron": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/cron/-/cron-2.1.0.tgz",
"integrity": "sha512-Hq7u3P8y7UWYvsZbSKHHJDVG0VO9O7tp2qljxzTScelcTODBfCme8AIhnZsFwmQ9NchZ3hr2uNr+s3DSms7q6w==",
"requires": {
"luxon": "^1.23.x"
}
},
"crypto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/crypto/-/crypto-1.0.1.tgz",
@ -7221,6 +7279,11 @@
"js-tokens": "^3.0.0 || ^4.0.0"
}
},
"luxon": {
"version": "1.28.0",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz",
"integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ=="
},
"make-dir": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz",

2
account/package.json
View File

@ -16,6 +16,7 @@
"bcryptjs": "^2.4.3",
"connect-redis": "^6.0.0",
"cors": "^2.8.5",
"cron": "^2.1.0",
"crypto": "^1.0.1",
"ejs": "^3.1.6",
"express-session": "^1.17.2",
@ -39,6 +40,7 @@
"@types/base64url": "^2.0.0",
"@types/bcryptjs": "^2.4.2",
"@types/cors": "^2.8.12",
"@types/cron": "^2.0.0",
"@types/express": "^4.17.13",
"@types/express-session": "^1.17.4",
"@types/node": "^16.6.0",

109
account/src/lib/dictionary.ts
View File

@ -1,9 +1,12 @@
import loki, { Collection, LokiMemoryAdapter } from "lokijs";
import fetch from "node-fetch";
import { CronJob } from "cron";
const collectionName = "ps-dictionary";
import {
readDictionary,
readDictionaryInfo,
Types as T,
typePredicates as tp,
} from "@lingdocs/inflect"
export let collection: Collection<any> | undefined = undefined;
@ -15,32 +18,90 @@ const lokidb = new loki("", {
env: "NODEJS",
});
export let dictionary: T.Dictionary | undefined = undefined;
const updateJob = new CronJob("* * * * *", updateDictionary, null, false);
// TODO: Abstract dictionary fetch
export function updateDictionary() {
fetch(process.env.LINGDOCS_DICTIONARY_URL || "").then(res => res.arrayBuffer()).then(buffer => {
const dict = readDictionary(buffer as Uint8Array);
dictionary = dict;
collection?.clear();
lokidb.removeCollection(collectionName);
collection?.insert(dictionary.entries);
}).catch(console.error);
let version: number = 0;
async function fetchDictionary(): Promise<T.Dictionary> {
const res = await fetch(process.env.LINGDOCS_DICTIONARY_URL || "");
const buffer = await res.arrayBuffer();
return readDictionary(buffer as Uint8Array);
}
lokidb.loadDatabase({}, async (err: Error) => {
collection = lokidb.getCollection(collectionName);
if (!collection) {
collection = lokidb.addCollection(collectionName, {
// TODO: THIS ISN'T WORKING!
disableMeta: true,
indices: ["i", "p"],
unique: ["ts"],
});
fetch(process.env.LINGDOCS_DICTIONARY_URL || "").then(res => res.arrayBuffer()).then(buffer => {
const dict = readDictionary(buffer as Uint8Array);
dictionary = dict;
collection?.insert(dictionary.entries);
}).catch(console.error);
async function fetchDictionaryInfo(): Promise<T.DictionaryInfo> {
const res = await fetch(process.env.LINGDOCS_DICTIONARY_URL + "-info" || "");
const buffer = await res.arrayBuffer();
return readDictionaryInfo(buffer as Uint8Array);
}
export async function updateDictionary(): Promise<"no update" | "updated"> {
const info = await fetchDictionaryInfo();
if (info.release === version) {
return "no update";
}
const dictionary = await fetchDictionary();
version = dictionary.info.release;
collection?.clear();
lokidb.removeCollection(collectionName);
collection?.insert(dictionary.entries);
return "updated";
}
function getOneByTs(ts: number): T.DictionaryEntry {
if (!collection) {
throw new Error("dictionary not initialized");
}
const r = collection.by("ts", ts);
const { $loki, meta, ...entry } = r;
return entry;
}
export async function getEntries(ids: number[]): Promise<{
results: (T.DictionaryEntry | T.VerbEntry)[],
notFound: number[],
}> {
if (!collection) {
throw new Error("dictionary not initialized");
}
const results: (T.DictionaryEntry | T.VerbEntry)[] = collection.find({
"ts": { "$in": ids },
}).map(x => {
const { $loki, meta, ...entry } = x;
return entry;
}).map((entry): T.DictionaryEntry | T.VerbEntry => {
if (tp.isVerbDictionaryEntry(entry)) {
if (entry.c?.includes("comp.") && entry.l) {
const complement = getOneByTs(entry.l);
if (!complement) throw new Error("Error getting complement "+entry.l);
return {
entry,
complement,
};
}
return { entry };
} else {
return entry;
}
});
return {
results,
notFound: ids.filter(id => !results.find(x => (
"entry" in x ? x.entry.ts === id : x.ts === id
))),
};
}
lokidb.loadDatabase({}, (err: Error) => {
lokidb.removeCollection(collectionName);
collection = lokidb.addCollection(collectionName, {
// TODO: THIS ISN'T WORKING!
disableMeta: true,
indices: ["i", "p"],
unique: ["ts"],
});
fetchDictionary().then((dictionary) => {
version = dictionary.info.release;
collection?.insert(dictionary.entries);
updateJob.start();
}).catch(console.error);
});

54
account/src/routers/dictionary-router.ts
View File

@ -1,55 +1,37 @@
import express, { Response } from "express";
import express from "express";
import {
collection,
dictionary,
getEntries,
updateDictionary,
} from "../lib/dictionary";
import { unary } from "froebel";
const dictionaryRouter = express.Router();
// Guard all api with authentication
dictionaryRouter.get("/", async (req, res, next) => {
if (!dictionary) {
return res.send({ ok: false, message: "dictionary not ready" });
}
res.send(dictionary);
})
dictionaryRouter.get("/info", async (req, res, next) => {
if (!dictionary) {
return res.send({ ok: false, message: "dictionary not ready" });
}
res.send({ info: dictionary.info })
});
dictionaryRouter.post("/update", async (req, res, next) => {
updateDictionary();
res.send({ ok: true });
const result = await updateDictionary();
res.send({ ok: true, result });
});
dictionaryRouter.get("/entry/:id", async (req, res, next) => {
const id = req.params.id.includes(",")
? req.params.id.split(",").map(unary(parseInt))
: parseInt(req.params.id);
dictionaryRouter.post("/entry", async (req, res, next) => {
if (!collection) {
return res.send({ ok: false, message: "dictionary not ready" });
}
if (Array.isArray(id)) {
const results = collection.find({
"ts": {
"$in": id,
},
});
return res.send({ results });
const ids = req.body.ids as number[];
if (!Array.isArray(ids)) {
return res.status(400).send({ ok: false, error: "invalid query" });
}
const r = collection.by("ts", id);
if (!r) {
return res.send({ result: [] });
const results = await getEntries(ids);
return res.send(results);
});
dictionaryRouter.get("/entry/:id", async (req, res, next) => {
if (!collection) {
return res.send({ ok: false, message: "dictionary not ready" });
}
// remove $loki and meta
const { $loki, meta, ...word } = r;
return res.send({ result: word });
const ids = req.params.id.split(",").map(unary(parseInt));
const results = await getEntries(ids);
return res.send(results);
});
export default dictionaryRouter;

25
account/yarn.lock
View File

@ -207,6 +207,14 @@
"resolved" "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz"
"version" "2.8.12"
"@types/cron@^2.0.0":
"integrity" "sha512-xZM08fqvwIXgghtPVkSPKNgC+JoMQ2OHazEvyTKnNf7aWu1aB6/4lBbQFrb03Td2cUGG7ITzMv3mFYnMu6xRaQ=="
"resolved" "https://registry.npmjs.org/@types/cron/-/cron-2.0.0.tgz"
"version" "2.0.0"
dependencies:
"@types/luxon" "*"
"@types/node" "*"
"@types/express-serve-static-core@^4.17.18":
"integrity" "sha512-3UJuW+Qxhzwjq3xhwXm2onQcFHn76frIYVbTu+kn24LFxI+dEhdfISDFovPB8VpEgW8oQCTpRuCe+0zJxB7NEA=="
"resolved" "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.24.tgz"
@ -233,6 +241,11 @@
"@types/qs" "*"
"@types/serve-static" "*"
"@types/luxon@*":
"integrity" "sha512-HM2OVWckUMmXbWYZufmWT2XMURGDZ6XbyNyQ+Lx+gCFGFqbZaIjsz7b+AGeGP/AuVYHBiuGY+wXfweP1RremnA=="
"resolved" "https://registry.npmjs.org/@types/luxon/-/luxon-3.0.2.tgz"
"version" "3.0.2"
"@types/mime@^1":
"integrity" "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw=="
"resolved" "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz"
@ -898,6 +911,13 @@
"resolved" "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz"
"version" "1.1.1"
"cron@^2.1.0":
"integrity" "sha512-Hq7u3P8y7UWYvsZbSKHHJDVG0VO9O7tp2qljxzTScelcTODBfCme8AIhnZsFwmQ9NchZ3hr2uNr+s3DSms7q6w=="
"resolved" "https://registry.npmjs.org/cron/-/cron-2.1.0.tgz"
"version" "2.1.0"
dependencies:
"luxon" "^1.23.x"
"crypto-browserify@3.12.0":
"integrity" "sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg=="
"resolved" "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz"
@ -1749,6 +1769,11 @@
dependencies:
"js-tokens" "^3.0.0 || ^4.0.0"
"luxon@^1.23.x":
"integrity" "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ=="
"resolved" "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz"
"version" "1.28.0"
"make-dir@^3.0.2":
"integrity" "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw=="
"resolved" "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz"

7
update-inflector.sh
View File

@ -9,8 +9,6 @@ if [ $# -eq 0 ]
version=$(npm show @lingdocs/ps-react version)
fi
# update all instances of @lingdocs/inflect and @lingdocs/ps-react in various package.json files
# in website
tmp=$(mktemp)
@ -19,10 +17,15 @@ jq --arg version "$version" '.dependencies."@lingdocs/ps-react"=$version' "websi
# in functions
tmp=$(mktemp)
jq --arg version "$version" '.dependencies."@lingdocs/inflect"=$version' "functions/package.json" > "$tmp" && mv "$tmp" "functions/package.json"
# in account
tmp=$(mktemp)
jq --arg version "$version" '.dependencies."@lingdocs/inflect"=$version' "account/package.json" > "$tmp" && mv "$tmp" "account/package.json"
# install to update .lock files
cd website &&
yarn install --legacy-peer-deps &&
cd ../functions &&
npm install &&
cd ../account &&
npm install