lingdocs
/
pashto-dictionary
mirror of https://github.com/lingdocs/pashto-dictionary.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

release new phonetics!

This commit is contained in:
adueck 2023-07-27 18:18:01 +04:00
parent 8dd63ad9c4
commit 54fb2050c1
21 changed files with 3376 additions and 2910 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -67,6 +67,10 @@ npm install
#### Development #### Development
```sh ```sh
firebase login
# get envars locally
firebase functions:config:get > .runtimeconfig.json
# start functions emulator
npm run serve npm run serve
``` ```

14
account/package-lock.json generated
View File

@ -9,7 +9,7 @@
"version": "1.0.0", "version": "1.0.0",
"license": "ISC", "license": "ISC",
"dependencies": { "dependencies": {
"@lingdocs/inflect": "5.10.1", "@lingdocs/inflect": "6.0.0",
"base64url": "^3.0.1", "base64url": "^3.0.1",
"bcryptjs": "^2.4.3", "bcryptjs": "^2.4.3",
"connect-redis": "^6.0.0", "connect-redis": "^6.0.0",
@ -124,9 +124,9 @@
} }
}, },
"node_modules/@lingdocs/inflect": { "node_modules/@lingdocs/inflect": {
"version": "5.10.1", "version": "6.0.0",
"resolved": "https://npm.lingdocs.com/@lingdocs%2finflect/-/inflect-5.10.1.tgz", "resolved": "https://npm.lingdocs.com/@lingdocs%2finflect/-/inflect-6.0.0.tgz",
"integrity": "sha512-8MPsfQzeerlyT02dz7D7L+AYFrjGOrQB7nMBUXutnLw3/RKhvW99dLImFZKSnCr8DZsEONEp0IVeqxeIUczxog==", "integrity": "sha512-aPvjqOkeKhu60Inbk7uuLooR/9hvUS4rDHyqR5JJPziZMLJ05U5fBTUvehit7stHSRGivskR00uU3liWbXce6g==",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"fp-ts": "^2.16.0", "fp-ts": "^2.16.0",
@ -2747,9 +2747,9 @@
} }
}, },
"@lingdocs/inflect": { "@lingdocs/inflect": {
"version": "5.10.1", "version": "6.0.0",
"resolved": "https://npm.lingdocs.com/@lingdocs%2finflect/-/inflect-5.10.1.tgz", "resolved": "https://npm.lingdocs.com/@lingdocs%2finflect/-/inflect-6.0.0.tgz",
"integrity": "sha512-8MPsfQzeerlyT02dz7D7L+AYFrjGOrQB7nMBUXutnLw3/RKhvW99dLImFZKSnCr8DZsEONEp0IVeqxeIUczxog==", "integrity": "sha512-aPvjqOkeKhu60Inbk7uuLooR/9hvUS4rDHyqR5JJPziZMLJ05U5fBTUvehit7stHSRGivskR00uU3liWbXce6g==",
"requires": { "requires": {
"fp-ts": "^2.16.0", "fp-ts": "^2.16.0",
"pbf": "^3.2.1", "pbf": "^3.2.1",

4
account/package.json
View File

@ -11,7 +11,7 @@
"author": "", "author": "",
"license": "ISC", "license": "ISC",
"dependencies": { "dependencies": {
"@lingdocs/inflect": "5.10.1", "@lingdocs/inflect": "6.0.0",
"base64url": "^3.0.1", "base64url": "^3.0.1",
"bcryptjs": "^2.4.3", "bcryptjs": "^2.4.3",
"connect-redis": "^6.0.0", "connect-redis": "^6.0.0",
@ -22,6 +22,7 @@
"express-session": "^1.17.2", "express-session": "^1.17.2",
"lokijs": "^1.5.12", "lokijs": "^1.5.12",
"nano": "^9.0.3", "nano": "^9.0.3",
"next": "^13.4.12",
"node-fetch": "^2.6.7", "node-fetch": "^2.6.7",
"nodemailer": "^6.6.3", "nodemailer": "^6.6.3",
"passport": "^0.4.1", "passport": "^0.4.1",
@ -42,6 +43,7 @@
"@types/cron": "^2.0.0", "@types/cron": "^2.0.0",
"@types/express": "^4.17.13", "@types/express": "^4.17.13",
"@types/express-session": "^1.17.4", "@types/express-session": "^1.17.4",
"@types/lokijs": "^1.5.8",
"@types/node": "^16.6.0", "@types/node": "^16.6.0",
"@types/node-fetch": "^2.5.12", "@types/node-fetch": "^2.5.12",
"@types/nodemailer": "^6.4.4", "@types/nodemailer": "^6.4.4",

207
account/src/lib/dictionary.ts
View File

@ -4,22 +4,22 @@ import { CronJob } from "cron";
const collectionName = "ps-dictionary"; const collectionName = "ps-dictionary";
const allWordsCollectionName = "all-words"; const allWordsCollectionName = "all-words";
import { import {
readDictionary, readDictionary,
readDictionaryInfo, readDictionaryInfo,
Types as T, Types as T,
typePredicates as tp, typePredicates as tp,
entryOfFull, entryOfFull,
standardizePashto, standardizePashto,
} from "@lingdocs/inflect" } from "@lingdocs/inflect";
export let collection: Collection<T.DictionaryEntry> | undefined = undefined; export let collection: Collection<T.DictionaryEntry> | undefined = undefined;
export let allWordsCollection: Collection<T.PsString> | undefined = undefined; export let allWordsCollection: Collection<T.PsString> | undefined = undefined;
const adapter = new LokiMemoryAdapter(); const adapter = new LokiMemoryAdapter();
const lokidb = new loki("", { const lokidb = new loki("", {
adapter, adapter,
autoload: false, autoload: false,
autosave: false, autosave: false,
env: "NODEJS", env: "NODEJS",
}); });
const updateJob = new CronJob("* * * * *", updateDictionary, null, false); const updateJob = new CronJob("* * * * *", updateDictionary, null, false);
@ -27,117 +27,126 @@ const updateJob = new CronJob("* * * * *", updateDictionary, null, false);
let version: number = 0; let version: number = 0;
async function fetchDictionary(): Promise<T.Dictionary> { async function fetchDictionary(): Promise<T.Dictionary> {
const res = await fetch(process.env.LINGDOCS_DICTIONARY_URL || ""); const res = await fetch(process.env.LINGDOCS_DICTIONARY_URL || "");
const buffer = await res.arrayBuffer(); const buffer = await res.arrayBuffer();
return readDictionary(buffer as Uint8Array); return readDictionary(buffer as Uint8Array);
} }
async function fetchAllWords(): Promise<T.AllWordsWithInflections> { async function fetchAllWords(): Promise<T.AllWordsWithInflections> {
// TODO: this is really ugly // TODO: this is really ugly
const res = await fetch(process.env.LINGDOCS_DICTIONARY_URL?.slice(0, -4) + "all-words.json"); const res = await fetch(
return await res.json(); process.env.LINGDOCS_DICTIONARY_URL?.slice(0, -10) +
"all-words-dictionary.json"
);
return await res.json();
} }
async function fetchDictionaryInfo(): Promise<T.DictionaryInfo> { async function fetchDictionaryInfo(): Promise<T.DictionaryInfo> {
const res = await fetch(process.env.LINGDOCS_DICTIONARY_URL + "-info" || ""); const res = await fetch(process.env.LINGDOCS_DICTIONARY_URL + "-info" || "");
const buffer = await res.arrayBuffer(); const buffer = await res.arrayBuffer();
return readDictionaryInfo(buffer as Uint8Array); return readDictionaryInfo(buffer as Uint8Array);
} }
export async function updateDictionary(): Promise<"no update" | "updated"> { export async function updateDictionary(): Promise<"no update" | "updated"> {
const info = await fetchDictionaryInfo(); const info = await fetchDictionaryInfo();
if (info.release === version) { if (info.release === version) {
return "no update"; return "no update";
} }
const dictionary = await fetchDictionary(); const dictionary = await fetchDictionary();
version = dictionary.info.release; version = dictionary.info.release;
collection?.clear(); collection?.clear();
lokidb.removeCollection(collectionName); lokidb.removeCollection(collectionName);
collection?.insert(dictionary.entries); collection?.insert(dictionary.entries);
const allWords = await fetchAllWords(); const allWords = await fetchAllWords();
allWordsCollection?.clear(); allWordsCollection?.clear();
lokidb.removeCollection(allWordsCollectionName); lokidb.removeCollection(allWordsCollectionName);
allWordsCollection?.insert(allWords.words); allWordsCollection?.insert(allWords.words);
return "updated"; return "updated";
} }
function getOneByTs(ts: number): T.DictionaryEntry { function getOneByTs(ts: number): T.DictionaryEntry {
if (!collection) { if (!collection) {
throw new Error("dictionary not initialized"); throw new Error("dictionary not initialized");
} }
const r = collection.by("ts", ts); const r = collection.by("ts", ts);
// @ts-ignore // @ts-ignore
const { $loki, meta, ...entry } = r; const { $loki, meta, ...entry } = r;
return entry; return entry;
} }
export function findInAllWords(p: string | RegExp): T.PsWord[] | undefined { export function findInAllWords(p: string | RegExp): T.PsWord[] | undefined {
if (!allWordsCollection) { if (!allWordsCollection) {
throw new Error("allWords not initialized"); throw new Error("allWords not initialized");
} }
return allWordsCollection.find({ return allWordsCollection.find({
p: typeof p === "string" p: typeof p === "string" ? p : { $regex: p },
? p });
: { $regex: p },
});
} }
export async function getEntries(ids: (number | string)[]): Promise<{ export async function getEntries(ids: (number | string)[]): Promise<{
results: (T.DictionaryEntry | T.VerbEntry)[], results: (T.DictionaryEntry | T.VerbEntry)[];
notFound: (number | string)[], notFound: (number | string)[];
}> { }> {
if (!collection) { if (!collection) {
throw new Error("dictionary not initialized"); throw new Error("dictionary not initialized");
} }
const idsP = ids.map(x => typeof x === "number" ? x : standardizePashto(x)) const idsP = ids.map((x) =>
const results: (T.DictionaryEntry | T.VerbEntry)[] = collection.find({ typeof x === "number" ? x : standardizePashto(x)
"$or": [ );
{ "ts": { "$in": idsP }}, const results: (T.DictionaryEntry | T.VerbEntry)[] = collection
{ "p": { "$in": idsP }}, .find({
], $or: [{ ts: { $in: idsP } }, { p: { $in: idsP } }],
}).map(x => { })
const { $loki, meta, ...entry } = x; .map((x) => {
return entry; const { $loki, meta, ...entry } = x;
}).map((entry): T.DictionaryEntry | T.VerbEntry => { return entry;
if (tp.isVerbDictionaryEntry(entry)) { })
if (entry.c?.includes("comp.") && entry.l) { .map((entry): T.DictionaryEntry | T.VerbEntry => {
const complement = getOneByTs(entry.l); if (tp.isVerbDictionaryEntry(entry)) {
if (!complement) throw new Error("Error getting complement "+entry.l); if (entry.c?.includes("comp.") && entry.l) {
return { const complement = getOneByTs(entry.l);
entry, if (!complement)
complement, throw new Error("Error getting complement " + entry.l);
}; return {
} entry,
return { entry }; complement,
} else { };
return entry;
} }
return { entry };
} else {
return entry;
}
}); });
return { return {
results, results,
notFound: ids.filter(id => !results.find(x => { notFound: ids.filter(
const entry = entryOfFull(x); (id) =>
return entry.p === id || entry.ts === id; !results.find((x) => {
})), const entry = entryOfFull(x);
}; return entry.p === id || entry.ts === id;
})
),
};
} }
lokidb.loadDatabase({}, (err: Error) => { lokidb.loadDatabase({}, (err: Error) => {
lokidb.removeCollection(collectionName); lokidb.removeCollection(collectionName);
lokidb.removeCollection(allWordsCollectionName); lokidb.removeCollection(allWordsCollectionName);
fetchDictionary().then((dictionary) => { fetchDictionary()
collection = lokidb.addCollection(collectionName, { .then((dictionary) => {
indices: ["i", "p"], collection = lokidb.addCollection(collectionName, {
unique: ["ts"], indices: ["i", "p"],
}); unique: ["ts"],
version = dictionary.info.release; });
collection?.insert(dictionary.entries); version = dictionary.info.release;
updateJob.start(); collection?.insert(dictionary.entries);
}).catch(console.error); updateJob.start();
fetchAllWords().then((allWords) => { })
allWordsCollection = lokidb.addCollection(allWordsCollectionName, { .catch(console.error);
indices: ["p"], fetchAllWords().then((allWords) => {
}); allWordsCollection = lokidb.addCollection(allWordsCollectionName, {
allWordsCollection?.insert(allWords.words); indices: ["p"],
}); });
allWordsCollection?.insert(allWords.words);
});
}); });

73
account/src/routers/dictionary-router.ts
View File

@ -1,55 +1,54 @@
import express from "express"; import express from "express";
import { import {
allWordsCollection, allWordsCollection,
collection, collection,
findInAllWords, getEntries,
getEntries, updateDictionary,
updateDictionary,
} from "../lib/dictionary"; } from "../lib/dictionary";
import { scriptToPhonetics } from "../lib/scriptToPhonetics"; import { scriptToPhonetics } from "../lib/scriptToPhonetics";
const dictionaryRouter = express.Router(); const dictionaryRouter = express.Router();
dictionaryRouter.post("/update", async (req, res, next) => { dictionaryRouter.post("/update", async (req, res, next) => {
const result = await updateDictionary(); const result = await updateDictionary();
res.send({ ok: true, result }); res.send({ ok: true, result });
}); });
dictionaryRouter.post("/script-to-phonetics", async (req, res, next) => { dictionaryRouter.post("/script-to-phonetics", async (req, res, next) => {
if (!allWordsCollection) { if (!allWordsCollection) {
return res.send({ ok: false, message: "allWords not ready" }); return res.send({ ok: false, message: "allWords not ready" });
} }
const text = req.body.text as unknown; const text = req.body.text as unknown;
const accents = req.body.accents as unknown; const accents = req.body.accents as unknown;
if (!text || typeof text !== "string" || typeof accents !== "boolean") { if (!text || typeof text !== "string" || typeof accents !== "boolean") {
return res.status(400).send({ ok: false, error: "invalid query" }); return res.status(400).send({ ok: false, error: "invalid query" });
} }
const results = await scriptToPhonetics(text, accents); const results = await scriptToPhonetics(text, accents);
res.send({ ok: true, results }); res.send({ ok: true, results });
}) });
dictionaryRouter.post("/entries", async (req, res, next) => { dictionaryRouter.post("/entries", async (req, res, next) => {
if (!collection) { if (!collection) {
return res.send({ ok: false, message: "dictionary not ready" }); return res.send({ ok: false, message: "dictionary not ready" });
} }
const ids = req.body.ids as (number | string)[]; const ids = req.body.ids as (number | string)[];
if (!Array.isArray(ids)) { if (!Array.isArray(ids)) {
return res.status(400).send({ ok: false, error: "invalid query" }); return res.status(400).send({ ok: false, error: "invalid query" });
} }
const results = await getEntries(ids); const results = await getEntries(ids);
return res.send(results); return res.send(results);
}); });
dictionaryRouter.get("/entries/:id", async (req, res, next) => { dictionaryRouter.get("/entries/:id", async (req, res, next) => {
if (!collection) { if (!collection) {
return res.send({ ok: false, message: "dictionary not ready" }); return res.send({ ok: false, message: "dictionary not ready" });
} }
const ids = req.params.id.split(",").map(x => { const ids = req.params.id.split(",").map((x) => {
const n = parseInt(x); const n = parseInt(x);
return Number.isNaN(n) ? x : n; return Number.isNaN(n) ? x : n;
}); });
const results = await getEntries(ids); const results = await getEntries(ids);
return res.send(results); return res.send(results);
}); });
export default dictionaryRouter; export default dictionaryRouter;

2865
account/yarn.lock
View File

File diff suppressed because it is too large Load Diff

14
functions/package-lock.json generated
View File

@ -7,7 +7,7 @@
"name": "functions", "name": "functions",
"dependencies": { "dependencies": {
"@google-cloud/storage": "^5.8.1", "@google-cloud/storage": "^5.8.1",
"@lingdocs/inflect": "5.10.1", "@lingdocs/inflect": "6.0.0",
"@types/cors": "^2.8.10", "@types/cors": "^2.8.10",
"@types/google-spreadsheet": "^3.0.2", "@types/google-spreadsheet": "^3.0.2",
"@types/react": "^18.0.21", "@types/react": "^18.0.21",
@ -1468,9 +1468,9 @@
} }
}, },
"node_modules/@lingdocs/inflect": { "node_modules/@lingdocs/inflect": {
"version": "5.10.1", "version": "6.0.0",
"resolved": "https://npm.lingdocs.com/@lingdocs%2finflect/-/inflect-5.10.1.tgz", "resolved": "https://npm.lingdocs.com/@lingdocs%2finflect/-/inflect-6.0.0.tgz",
"integrity": "sha512-8MPsfQzeerlyT02dz7D7L+AYFrjGOrQB7nMBUXutnLw3/RKhvW99dLImFZKSnCr8DZsEONEp0IVeqxeIUczxog==", "integrity": "sha512-aPvjqOkeKhu60Inbk7uuLooR/9hvUS4rDHyqR5JJPziZMLJ05U5fBTUvehit7stHSRGivskR00uU3liWbXce6g==",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"fp-ts": "^2.16.0", "fp-ts": "^2.16.0",
@ -8056,9 +8056,9 @@
} }
}, },
"@lingdocs/inflect": { "@lingdocs/inflect": {
"version": "5.10.1", "version": "6.0.0",
"resolved": "https://npm.lingdocs.com/@lingdocs%2finflect/-/inflect-5.10.1.tgz", "resolved": "https://npm.lingdocs.com/@lingdocs%2finflect/-/inflect-6.0.0.tgz",
"integrity": "sha512-8MPsfQzeerlyT02dz7D7L+AYFrjGOrQB7nMBUXutnLw3/RKhvW99dLImFZKSnCr8DZsEONEp0IVeqxeIUczxog==", "integrity": "sha512-aPvjqOkeKhu60Inbk7uuLooR/9hvUS4rDHyqR5JJPziZMLJ05U5fBTUvehit7stHSRGivskR00uU3liWbXce6g==",
"requires": { "requires": {
"fp-ts": "^2.16.0", "fp-ts": "^2.16.0",
"pbf": "^3.2.1", "pbf": "^3.2.1",

2
functions/package.json
View File

@ -15,7 +15,7 @@
"main": "lib/functions/src/index.js", "main": "lib/functions/src/index.js",
"dependencies": { "dependencies": {
"@google-cloud/storage": "^5.8.1", "@google-cloud/storage": "^5.8.1",
"@lingdocs/inflect": "5.10.1", "@lingdocs/inflect": "6.0.0",
"@types/cors": "^2.8.10", "@types/cors": "^2.8.10",
"@types/google-spreadsheet": "^3.0.2", "@types/google-spreadsheet": "^3.0.2",
"@types/react": "^18.0.21", "@types/react": "^18.0.21",

78
functions/src/index.ts
View File

@ -4,47 +4,59 @@ import { receiveSubmissions } from "./submissions";
import lingdocsAuth from "./middleware/lingdocs-auth"; import lingdocsAuth from "./middleware/lingdocs-auth";
import publish from "./publish"; import publish from "./publish";
export const publishDictionary = functions.runWith({ export const publishDictionary = functions
.runWith({
timeoutSeconds: 525, timeoutSeconds: 525,
memory: "2GB" memory: "2GB",
}).https.onRequest( })
.https.onRequest(
lingdocsAuth( lingdocsAuth(
async (req, res: functions.Response<FT.PublishDictionaryResponse | FT.FunctionError>) => { async (
if (req.user.level !== "editor") { req,
res.status(403).send({ ok: false, error: "403 forbidden" }); res: functions.Response<FT.PublishDictionaryResponse | FT.FunctionError>
return; ) => {
} if (req.user.level !== "editor") {
try { res.status(403).send({ ok: false, error: "403 forbidden" });
const response = await publish(); return;
res.send(response);
} catch (e) {
// @ts-ignore
res.status(500).send({ ok: false, error: e.message });
}
} }
try {
const response = await publish();
res.send(response);
} catch (e) {
// @ts-ignore
res.status(500).send({ ok: false, error: e.message });
}
}
) )
); );
export const submissions = functions.runWith({ export const submissions = functions
.runWith({
timeoutSeconds: 60, timeoutSeconds: 60,
memory: "1GB", memory: "1GB",
}).https.onRequest(lingdocsAuth( })
async (req, res: functions.Response<FT.SubmissionsResponse | FT.FunctionError>) => { .https.onRequest(
lingdocsAuth(
async (
req,
res: functions.Response<FT.SubmissionsResponse | FT.FunctionError>
) => {
if (!Array.isArray(req.body)) { if (!Array.isArray(req.body)) {
res.status(400).send({ res.status(400).send({
ok: false, ok: false,
error: "invalid submission", error: "invalid submission",
}); });
return; return;
} }
const suggestions = req.body as FT.SubmissionsRequest; const suggestions = req.body as FT.SubmissionsRequest;
try { try {
const response = await receiveSubmissions(suggestions, true);// req.user.level === "editor"); const response = await receiveSubmissions(suggestions, true); // req.user.level === "editor");
// TODO: WARN IF ANY OF THE EDITS DIDN'T HAPPEN // TODO: WARN IF ANY OF THE EDITS DIDN'T HAPPEN
res.send(response); res.send(response);
} catch (e) { } catch (e) {
// @ts-ignore // @ts-ignore
res.status(500).send({ ok: false, error: e.message }); res.status(500).send({ ok: false, error: e.message });
}; }
}) }
); )
);

373
functions/src/publish.ts
View File

@ -1,36 +1,33 @@
import { GoogleSpreadsheet } from "google-spreadsheet"; import { GoogleSpreadsheet } from "google-spreadsheet";
import * as functions from "firebase-functions"; import * as functions from "firebase-functions";
import { import {
Types as T, Types as T,
dictionaryEntryBooleanFields, dictionaryEntryBooleanFields,
dictionaryEntryNumberFields, dictionaryEntryNumberFields,
dictionaryEntryTextFields, dictionaryEntryTextFields,
validateEntry, validateEntry,
writeDictionary, writeDictionary,
writeDictionaryInfo, writeDictionaryInfo,
simplifyPhonetics, simplifyPhonetics,
standardizeEntry, standardizeEntry,
} from "@lingdocs/inflect"; } from "@lingdocs/inflect";
import { import { getWordList } from "./word-list-maker";
getWordList, import { PublishDictionaryResponse } from "../../website/src/types/functions-types";
} from "./word-list-maker";
import {
PublishDictionaryResponse,
} from "../../website/src/types/functions-types";
import { Storage } from "@google-cloud/storage"; import { Storage } from "@google-cloud/storage";
const storage = new Storage({ const storage = new Storage({
projectId: "lingdocs", projectId: "lingdocs",
}); });
const title = "LingDocs Pashto Dictionary" const title = "LingDocs Pashto Dictionary";
const license = "Copyright © 2021 lingdocs.com All Rights Reserved - Licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License - https://creativecommons.org/licenses/by-nc-sa/4.0/"; const license =
"Copyright © 2021 lingdocs.com All Rights Reserved - Licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License - https://creativecommons.org/licenses/by-nc-sa/4.0/";
const bucketName = "lingdocs"; const bucketName = "lingdocs";
const baseUrl = `https://storage.googleapis.com/${bucketName}/`; const baseUrl = `https://storage.googleapis.com/${bucketName}/`;
const dictionaryFilename = "dict"; const dictionaryFilename = "dictionary";
const dictionaryInfoFilename = "dict-info"; const dictionaryInfoFilename = "dictionary-info";
// const hunspellAffFileFilename = "ps_AFF.aff"; // const hunspellAffFileFilename = "ps_AFF.aff";
// const hunspellDicFileFilename = "ps_AFF.dic"; // const hunspellDicFileFilename = "ps_AFF.dic";
const allWordsJsonFilename = "all-words.json"; const allWordsJsonFilename = "all-words-dictionary.json";
const url = `${baseUrl}${dictionaryFilename}`; const url = `${baseUrl}${dictionaryFilename}`;
const infoUrl = `${baseUrl}${dictionaryInfoFilename}`; const infoUrl = `${baseUrl}${dictionaryInfoFilename}`;
@ -38,159 +35,173 @@ const infoUrl = `${baseUrl}${dictionaryInfoFilename}`;
// to keep the publish function time down // to keep the publish function time down
export default async function publish(): Promise<PublishDictionaryResponse> { export default async function publish(): Promise<PublishDictionaryResponse> {
const entries = await getRawEntries(); const entries = await getRawEntries();
const errors = checkForErrors(entries); const errors = checkForErrors(entries);
if (errors.length) { if (errors.length) {
return({ ok: false, errors }); return { ok: false, errors };
} }
// const duplicates = findDuplicates(entries); // const duplicates = findDuplicates(entries);
// duplicates.forEach((duplicate) => { // duplicates.forEach((duplicate) => {
// const index = entries.findIndex(e => e.ts === duplicate.ts); // const index = entries.findIndex(e => e.ts === duplicate.ts);
// if (index > -1) entries.splice(index, 1); // if (index > -1) entries.splice(index, 1);
// }) // })
const dictionary: T.Dictionary = { const dictionary: T.Dictionary = {
info: { info: {
title, title,
license, license,
url, url,
infoUrl, infoUrl,
release: new Date().getTime(), release: new Date().getTime(),
numberOfEntries: entries.length, numberOfEntries: entries.length,
}, },
entries, entries,
} };
uploadDictionaryToStorage(dictionary).catch(console.error); uploadDictionaryToStorage(dictionary).catch(console.error);
// TODO: make this async and run after publish response // TODO: make this async and run after publish response
doHunspellEtc(dictionary.info, entries).catch(console.error); doHunspellEtc(dictionary.info, entries).catch(console.error);
return { return {
ok: true, ok: true,
info: dictionary.info info: dictionary.info,
}; };
} }
async function doHunspellEtc(info: T.DictionaryInfo, entries: T.DictionaryEntry[]) { async function doHunspellEtc(
const wordlistResponse = getWordList(entries); info: T.DictionaryInfo,
if (!wordlistResponse.ok) { entries: T.DictionaryEntry[]
throw new Error(JSON.stringify(wordlistResponse.errors)); ) {
} const wordlistResponse = getWordList(entries);
// const hunspell = makeHunspell(wordlistResponse.wordlist); if (!wordlistResponse.ok) {
// await uploadHunspellToStorage(hunspell); throw new Error(JSON.stringify(wordlistResponse.errors));
await uploadAllWordsToStoarage(info, wordlistResponse.wordlist) }
// const hunspell = makeHunspell(wordlistResponse.wordlist);
// await uploadHunspellToStorage(hunspell);
await uploadAllWordsToStoarage(info, wordlistResponse.wordlist);
} }
/** /**
* Gets the entries from the spreadsheet, and also deletes duplicate * Gets the entries from the spreadsheet, and also deletes duplicate
* entries that are sometimes annoyingly created by the GoogleSheets API * entries that are sometimes annoyingly created by the GoogleSheets API
* when adding entries programmatically * when adding entries programmatically
* *
* @returns * @returns
* *
*/ */
async function getRows() { async function getRows() {
const doc = new GoogleSpreadsheet( const doc = new GoogleSpreadsheet(functions.config().sheet.id);
functions.config().sheet.id, await doc.useServiceAccountAuth({
); client_email: functions.config().serviceacct.email,
await doc.useServiceAccountAuth({ private_key: functions.config().serviceacct.key,
client_email: functions.config().serviceacct.email, });
private_key: functions.config().serviceacct.key, await doc.loadInfo();
}); const sheet = doc.sheetsByIndex[0];
await doc.loadInfo(); const rows = await sheet.getRows();
const sheet = doc.sheetsByIndex[0]; rows.sort((a, b) => (a.ts > b.ts ? -1 : a.ts < b.ts ? 1 : 0));
const rows = await sheet.getRows(); return rows;
rows.sort((a, b) => a.ts > b.ts ? -1 : a.ts < b.ts ? 1 : 0);
return rows;
} }
async function getRawEntries(): Promise<T.DictionaryEntry[]> { async function getRawEntries(): Promise<T.DictionaryEntry[]> {
const rows = await getRows(); const rows = await getRows();
async function deleteRow(i: number) { async function deleteRow(i: number) {
console.log("WILL DELETE ROW", rows[i].p, rows[i].ts, rows[i].f); console.log("WILL DELETE ROW", rows[i].p, rows[i].ts, rows[i].f);
await rows[i].delete(); await rows[i].delete();
}
const entries: T.DictionaryEntry[] = [];
let sheetIndex = 0;
// get the rows in order of ts for easy detection of duplicate entries
for (let i = 0; i < rows.length; i++) {
function sameEntry(a: any, b: any): boolean {
return a.p === b.p && a.f === b.f && a.e === b.e;
} }
const entries: T.DictionaryEntry[] = []; sheetIndex++;
let sheetIndex = 0; const row = rows[i];
// get the rows in order of ts for easy detection of duplicate entries const nextRow = rows[i + 1] || undefined;
for (let i = 0; i < rows.length; i++) { if (row.ts === nextRow?.ts) {
function sameEntry(a: any, b: any): boolean { if (sameEntry(row, nextRow)) {
return a.p === b.p && a.f === b.f && a.e === b.e; // this looks like a duplicate entry made by the sheets api
} // delete it and keep going
sheetIndex++; await deleteRow(sheetIndex);
const row = rows[i]; sheetIndex--;
const nextRow = rows[i+1] || undefined; continue;
if (row.ts === nextRow?.ts) { } else {
if (sameEntry(row, nextRow)) { throw new Error(`ts ${row.ts} is a duplicate ts of a different entry`);
// this looks like a duplicate entry made by the sheets api }
// delete it and keep going
await deleteRow(sheetIndex);
sheetIndex--;
continue;
} else {
throw new Error(`ts ${row.ts} is a duplicate ts of a different entry`);
}
}
const e: T.DictionaryEntry = {
i: 1,
ts: parseInt(row.ts),
p: row.p,
f: row.f,
g: simplifyPhonetics(row.f),
e: row.e,
};
dictionaryEntryNumberFields.forEach((field: T.DictionaryEntryNumberField) => {
if (row[field]) e[field] = parseInt(row[field]);
});
dictionaryEntryTextFields.forEach((field: T.DictionaryEntryTextField) => {
if (row[field]) e[field] = row[field].trim();
});
dictionaryEntryBooleanFields.forEach((field: T.DictionaryEntryBooleanField) => {
if (row[field]) e[field] = true;
});
entries.push(standardizeEntry(e));
} }
// add alphabetical index const e: T.DictionaryEntry = {
entries.sort((a, b) => a.p.localeCompare(b.p, "ps")); i: 1,
const entriesLength = entries.length; ts: parseInt(row.ts),
// add index p: row.p,
for (let i = 0; i < entriesLength; i++) { f: row.f,
entries[i].i = i; g: simplifyPhonetics(row.f),
} e: row.e,
return entries; };
dictionaryEntryNumberFields.forEach(
(field: T.DictionaryEntryNumberField) => {
if (row[field]) e[field] = parseInt(row[field]);
}
);
dictionaryEntryTextFields.forEach((field: T.DictionaryEntryTextField) => {
if (row[field]) e[field] = row[field].trim();
});
dictionaryEntryBooleanFields.forEach(
(field: T.DictionaryEntryBooleanField) => {
if (row[field]) e[field] = true;
}
);
entries.push(standardizeEntry(e));
}
// add alphabetical index
entries.sort((a, b) => a.p.localeCompare(b.p, "ps"));
const entriesLength = entries.length;
// add index
for (let i = 0; i < entriesLength; i++) {
entries[i].i = i;
}
return entries;
} }
function checkForErrors(entries: T.DictionaryEntry[]): T.DictionaryEntryError[] { function checkForErrors(
return entries.reduce((errors: T.DictionaryEntryError[], entry: T.DictionaryEntry) => { entries: T.DictionaryEntry[]
const response = validateEntry(entry); ): T.DictionaryEntryError[] {
if ("errors" in response && response.errors.length) { return entries.reduce(
return [...errors, response]; (errors: T.DictionaryEntryError[], entry: T.DictionaryEntry) => {
const response = validateEntry(entry);
if ("errors" in response && response.errors.length) {
return [...errors, response];
}
if ("checkComplement" in response) {
const complement = entries.find((e) => e.ts === entry.l);
if (!complement) {
const error: T.DictionaryEntryError = {
errors: ["complement link not found in dictonary"],
ts: entry.ts,
p: entry.p,
f: entry.f,
e: entry.e,
erroneousFields: ["l"],
};
return [...errors, error];
} }
if ("checkComplement" in response) { if (
const complement = entries.find((e) => e.ts === entry.l); !complement.c?.includes("n.") &&
if (!complement) { !complement.c?.includes("adj.") &&
const error: T.DictionaryEntryError = { !complement.c?.includes("adv.")
errors: ["complement link not found in dictonary"], ) {
ts: entry.ts, const error: T.DictionaryEntryError = {
p: entry.p, errors: ["complement link to invalid complement"],
f: entry.f, ts: entry.ts,
e: entry.e, p: entry.p,
erroneousFields: ["l"], f: entry.f,
}; e: entry.e,
return [...errors, error]; erroneousFields: ["l"],
} };
if (!complement.c?.includes("n.") && !complement.c?.includes("adj.") && !complement.c?.includes("adv.")) { return [...errors, error];
const error: T.DictionaryEntryError = {
errors: ["complement link to invalid complement"],
ts: entry.ts,
p: entry.p,
f: entry.f,
e: entry.e,
erroneousFields: ["l"],
};
return [...errors, error];
}
} }
return errors; }
}, []); return errors;
},
[]
);
} }
// function findDuplicates(entries: T.DictionaryEntry[]): T.DictionaryEntry[] { // function findDuplicates(entries: T.DictionaryEntry[]): T.DictionaryEntry[] {
@ -208,20 +219,20 @@ function checkForErrors(entries: T.DictionaryEntry[]): T.DictionaryEntryError[]
// } // }
async function upload(content: Buffer | string, filename: string) { async function upload(content: Buffer | string, filename: string) {
const isBuffer = typeof content !== "string"; const isBuffer = typeof content !== "string";
const file = storage.bucket(bucketName).file(filename); const file = storage.bucket(bucketName).file(filename);
await file.save(content, { await file.save(content, {
gzip: isBuffer ? false : true, gzip: isBuffer ? false : true,
predefinedAcl: "publicRead", predefinedAcl: "publicRead",
metadata: { metadata: {
contentType: isBuffer contentType: isBuffer
? "application/octet-stream" ? "application/octet-stream"
: filename.slice(-5) === ".json" : filename.slice(-5) === ".json"
? "application/json" ? "application/json"
: "text/plain; charset=UTF-8", : "text/plain; charset=UTF-8",
cacheControl: "no-cache", cacheControl: "no-cache",
}, },
}); });
} }
// async function uploadHunspellToStorage(wordlist: { // async function uploadHunspellToStorage(wordlist: {
@ -234,19 +245,25 @@ async function upload(content: Buffer | string, filename: string) {
// ]); // ]);
// } // }
async function uploadAllWordsToStoarage(info: T.DictionaryInfo, words: T.PsString[]) { async function uploadAllWordsToStoarage(
await upload(JSON.stringify({ info, words } as T.AllWordsWithInflections), allWordsJsonFilename); info: T.DictionaryInfo,
words: T.PsString[]
) {
await upload(
JSON.stringify({ info, words } as T.AllWordsWithInflections),
allWordsJsonFilename
);
} }
async function uploadDictionaryToStorage(dictionary: T.Dictionary) { async function uploadDictionaryToStorage(dictionary: T.Dictionary) {
const dictionaryBuffer = writeDictionary(dictionary); const dictionaryBuffer = writeDictionary(dictionary);
const dictionaryInfoBuffer = writeDictionaryInfo(dictionary.info); const dictionaryInfoBuffer = writeDictionaryInfo(dictionary.info);
await Promise.all([ await Promise.all([
upload(JSON.stringify(dictionary), `${dictionaryFilename}.json`), upload(JSON.stringify(dictionary), `${dictionaryFilename}.json`),
upload(JSON.stringify(dictionary.info), `${dictionaryInfoFilename}.json`), upload(JSON.stringify(dictionary.info), `${dictionaryInfoFilename}.json`),
upload(dictionaryBuffer as Buffer, dictionaryFilename), upload(dictionaryBuffer as Buffer, dictionaryFilename),
upload(dictionaryInfoBuffer as Buffer, dictionaryInfoFilename), upload(dictionaryInfoBuffer as Buffer, dictionaryInfoFilename),
]); ]);
} }
// function makeHunspell(wordlist: string[]) { // function makeHunspell(wordlist: string[]) {

2
website/package.json
View File

@ -7,7 +7,7 @@
"private": true, "private": true,
"dependencies": { "dependencies": {
"@fortawesome/fontawesome-free": "^5.15.2", "@fortawesome/fontawesome-free": "^5.15.2",
"@lingdocs/ps-react": "5.10.1", "@lingdocs/ps-react": "6.0.0",
"@testing-library/jest-dom": "^5.11.4", "@testing-library/jest-dom": "^5.11.4",
"@testing-library/react": "^11.1.0", "@testing-library/react": "^11.1.0",
"@testing-library/user-event": "^12.1.10", "@testing-library/user-event": "^12.1.10",

96
website/src/App.css
View File

@ -9,7 +9,7 @@
* { * {
font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Oxygen, font-family: -apple-system, BlinkMacSystemFont, Segoe UI, Roboto, Oxygen,
Ubuntu, Cantarell, Fira Sans, Droid Sans, Helvetica Neue, sans-serif; Ubuntu, Cantarell, Fira Sans, Droid Sans, Helvetica Neue, sans-serif;
} }
:root { :root {
@ -36,7 +36,7 @@
--farther: #bbb; --farther: #bbb;
--farthest: #999; --farthest: #999;
--high-contrast: #cfcfcf; --high-contrast: #cfcfcf;
--input-bg: #ccc; --input-bg: #ccc;
} }
@ -99,13 +99,23 @@ hr {
background-color: var(--closer) !important; background-color: var(--closer) !important;
color: var(--high-contrast); color: var(--high-contrast);
} }
.bg-white { .bg-white {
background-color: var(--theme-shade) !important; background-color: var(--theme-shade) !important;
} }
/* TODO: better handling of modals across light and dark modes */ /* TODO: better handling of modals across light and dark modes */
.modal-body, .modal-title { .modal-body,
color:#1d1f25; .modal-title {
color: var(--high-contrast);
}
.modal-content {
background-color: var(--theme-shade);
}
.modal-content .table {
color: var(--high-contrast);
} }
.table { .table {
@ -310,6 +320,7 @@ input {
.entry-suggestion-button { .entry-suggestion-button {
right: 15px; right: 15px;
} }
.conjugation-search-button { .conjugation-search-button {
right: 15px; right: 15px;
} }
@ -339,6 +350,7 @@ input {
text-decoration: none; text-decoration: none;
color: var(--farther); color: var(--farther);
} }
.clickable:hover { .clickable:hover {
color: var(--farther); color: var(--farther);
} }
@ -356,13 +368,13 @@ input {
.btn.bg-white:active, .btn.bg-white:active,
.btn.bg-white:hover { .btn.bg-white:hover {
color: #555 !important; color: #555 !important;
} }
.btn-group.full-width { .btn-group.full-width {
display: flex; display: flex;
} }
.full-width .btn { .full-width .btn {
flex: 1; flex: 1;
} }
@ -376,43 +388,49 @@ input {
/* Loding animation from https://projects.lukehaas.me/css-loaders/ */ /* Loding animation from https://projects.lukehaas.me/css-loaders/ */
.loader, .loader,
.loader:after { .loader:after {
border-radius: 50%; border-radius: 50%;
width: 10em; width: 10em;
height: 10em; height: 10em;
} }
.loader { .loader {
margin: 60px auto; margin: 60px auto;
font-size: 10px; font-size: 10px;
position: relative; position: relative;
text-indent: -9999em; text-indent: -9999em;
border-top: 1.1em solid var(--closer); border-top: 1.1em solid var(--closer);
border-right: 1.1em solid var(--closer); border-right: 1.1em solid var(--closer);
border-bottom: 1.1em solid var(--closer); border-bottom: 1.1em solid var(--closer);
border-left: 1.1em solid var(--farthest); border-left: 1.1em solid var(--farthest);
-webkit-transform: translateZ(0); -webkit-transform: translateZ(0);
-ms-transform: translateZ(0); -ms-transform: translateZ(0);
transform: translateZ(0); transform: translateZ(0);
-webkit-animation: load8 1.1s infinite linear; -webkit-animation: load8 1.1s infinite linear;
animation: load8 1.1s infinite linear; animation: load8 1.1s infinite linear;
} }
@-webkit-keyframes load8 { @-webkit-keyframes load8 {
0% { 0% {
-webkit-transform: rotate(0deg); -webkit-transform: rotate(0deg);
transform: rotate(0deg); transform: rotate(0deg);
} }
100% {
-webkit-transform: rotate(360deg); 100% {
transform: rotate(360deg); -webkit-transform: rotate(360deg);
} transform: rotate(360deg);
}
} }
@keyframes load8 { @keyframes load8 {
0% { 0% {
-webkit-transform: rotate(0deg); -webkit-transform: rotate(0deg);
transform: rotate(0deg); transform: rotate(0deg);
} }
100% {
-webkit-transform: rotate(360deg); 100% {
transform: rotate(360deg); -webkit-transform: rotate(360deg);
} transform: rotate(360deg);
}
} }
/* End of loading animation from https://projects.lukehaas.me/css-loaders/ */
/* End of loading animation from https://projects.lukehaas.me/css-loaders/ */

101
website/src/App.tsx
View File

@ -72,6 +72,7 @@ import PhraseBuilder from "./screens/PhraseBuilder";
import { searchAllInflections } from "./lib/search-all-inflections"; import { searchAllInflections } from "./lib/search-all-inflections";
import { addToWordlist } from "./lib/wordlist-database"; import { addToWordlist } from "./lib/wordlist-database";
import ScriptToPhonetics from "./screens/ScriptToPhonetics"; import ScriptToPhonetics from "./screens/ScriptToPhonetics";
import { Modal, Button } from "react-bootstrap";
// to allow Moustrap key combos even when input fields are in focus // to allow Moustrap key combos even when input fields are in focus
Mousetrap.prototype.stopCallback = function () { Mousetrap.prototype.stopCallback = function () {
@ -107,6 +108,7 @@ class App extends Component<RouteComponentProps, State> {
this.state = { this.state = {
dictionaryStatus: "loading", dictionaryStatus: "loading",
dictionaryInfo: undefined, dictionaryInfo: undefined,
showModal: false,
// TODO: Choose between the saved options and the options in the saved user // TODO: Choose between the saved options and the options in the saved user
options: savedOptions options: savedOptions
? savedOptions ? savedOptions
@ -146,6 +148,8 @@ class App extends Component<RouteComponentProps, State> {
this.handleRefreshReviewTasks = this.handleRefreshReviewTasks.bind(this); this.handleRefreshReviewTasks = this.handleRefreshReviewTasks.bind(this);
this.handleDictionaryUpdate = this.handleDictionaryUpdate.bind(this); this.handleDictionaryUpdate = this.handleDictionaryUpdate.bind(this);
this.handleInflectionSearch = this.handleInflectionSearch.bind(this); this.handleInflectionSearch = this.handleInflectionSearch.bind(this);
this.handleShowModal = this.handleShowModal.bind(this);
this.handleCloseModal = this.handleCloseModal.bind(this);
} }
public componentDidMount() { public componentDidMount() {
@ -583,6 +587,14 @@ class App extends Component<RouteComponentProps, State> {
}); });
} }
private handleCloseModal() {
this.setState({ showModal: false });
}
private handleShowModal() {
this.setState({ showModal: true });
}
render() { render() {
return ( return (
<div <div
@ -641,7 +653,7 @@ class App extends Component<RouteComponentProps, State> {
> >
<div className="my-4">New words this month</div> <div className="my-4">New words this month</div>
</Link> </Link>
<div className="mt-4 pt-3"> <div className="my-4 pt-3">
<Link <Link
to="/phrase-builder" to="/phrase-builder"
className="plain-link h5 font-weight-light" className="plain-link h5 font-weight-light"
@ -656,6 +668,12 @@ class App extends Component<RouteComponentProps, State> {
Grammar Grammar
</a> </a>
</div> </div>
<button
onClick={this.handleShowModal}
className="mt-2 btn btn-lg btn-secondary"
>
✨ New Phonetics for ی's!! 👀
</button>
</div> </div>
</Route> </Route>
<Route path="/about"> <Route path="/about">
@ -816,6 +834,87 @@ class App extends Component<RouteComponentProps, State> {
/> />
)} )}
</footer> </footer>
<Modal
show={this.state.showModal}
onHide={this.handleCloseModal}
centered
>
<Modal.Header closeButton>
<Modal.Title>Phonetics Update! 📰</Modal.Title>
</Modal.Header>
<Modal.Body>
<p>
The phonetics for{" "}
<span style={{ backgroundColor: "rgba(255,255,0,0.4)" }}>
two of the five ی's have been updated
</span>{" "}
to something much more logical and helpful for pronunciation.
</p>
<h5>Pure Vowels (mouth stays still)</h5>
<table className="table">
<thead>
<tr>
<th scope="col">Letter</th>
<th scope="col">Phonetics</th>
<th scope="col">Sound</th>
</tr>
</thead>
<tbody>
<tr>
<td>ي</td>
<td>ee</td>
<td>long "ee" like "bee"</td>
</tr>
<tr>
<td>ې</td>
<td>e</td>
<td>
<div>
like "ee" but <em>with a slightly more open mouth</em>
</div>
<div className="small">
This is a special vowel <em>not found in English</em>
</div>
</td>
</tr>
</tbody>
</table>
<h5>Dipthongs (pure vowel + y)</h5>
<table className="table">
<thead>
<tr>
<th scope="col">Letter</th>
<th scope="col">Phonetics</th>
<th scope="col">Sound</th>
</tr>
</thead>
<tbody>
<tr style={{ backgroundColor: "rgba(255,255,0,0.4)" }}>
<td>ی</td>
<td>ay</td>
<td>short 'a' + y</td>
</tr>
<tr>
<td>ۍ</td>
<td>uy</td>
<td>'u' shwa (ə) + y</td>
</tr>
<tr style={{ backgroundColor: "rgba(255,255,0,0.4)" }}>
<td>ئ</td>
<td>ey</td>
<td>
<div>'e' (ې) + y</div>
</td>
</tr>
</tbody>
</table>
</Modal.Body>
<Modal.Footer>
<Button variant="secondary" onClick={this.handleCloseModal}>
Close
</Button>
</Modal.Footer>
</Modal>
</div> </div>
); );
} }

834
website/src/lib/dictionary.ts
View File

@ -10,10 +10,10 @@ import { DictionaryDb } from "./dictionary-core";
import sanitizePashto from "./sanitize-pashto"; import sanitizePashto from "./sanitize-pashto";
import fillerWords from "./filler-words"; import fillerWords from "./filler-words";
import { import {
Types as T, Types as T,
simplifyPhonetics, simplifyPhonetics,
typePredicates as tp, typePredicates as tp,
revertSpelling, revertSpelling,
} from "@lingdocs/ps-react"; } from "@lingdocs/ps-react";
import { isPashtoScript } from "./is-pashto"; import { isPashtoScript } from "./is-pashto";
import { fuzzifyPashto } from "./fuzzify-pashto/fuzzify-pashto"; import { fuzzifyPashto } from "./fuzzify-pashto/fuzzify-pashto";
@ -21,14 +21,11 @@ import { fuzzifyPashto } from "./fuzzify-pashto/fuzzify-pashto";
import relevancy from "relevancy"; import relevancy from "relevancy";
import { makeAWeeBitFuzzy } from "./wee-bit-fuzzy"; import { makeAWeeBitFuzzy } from "./wee-bit-fuzzy";
import { getTextOptions } from "./get-text-options"; import { getTextOptions } from "./get-text-options";
import { import { DictionaryAPI, State } from "../types/dictionary-types";
DictionaryAPI,
State,
} from "../types/dictionary-types";
// const dictionaryBaseUrl = "https://storage.googleapis.com/lingdocs/"; // const dictionaryBaseUrl = "https://storage.googleapis.com/lingdocs/";
const dictionaryUrl = `https://storage.googleapis.com/lingdocs/dict`; const dictionaryUrl = `https://storage.googleapis.com/lingdocs/dictionary`;
const dictionaryInfoUrl = `https://storage.googleapis.com/lingdocs/dict-info`; const dictionaryInfoUrl = `https://storage.googleapis.com/lingdocs/dictionary-info`;
const dictionaryInfoLocalStorageKey = "dictionaryInfo5"; const dictionaryInfoLocalStorageKey = "dictionaryInfo5";
const dictionaryCollectionName = "dictionary3"; const dictionaryCollectionName = "dictionary3";
@ -37,17 +34,19 @@ export const pageSize = 35;
const relevancySorter = new relevancy.Sorter(); const relevancySorter = new relevancy.Sorter();
const db = indexedDB.open('inPrivate'); const db = indexedDB.open("inPrivate");
db.onerror = (e) => { db.onerror = (e) => {
console.error(e); console.error(e);
alert("Your browser does not have IndexedDB enabled. This might be because you are using private mode. Please use regular mode or enable IndexedDB to use this dictionary"); alert(
} "Your browser does not have IndexedDB enabled. This might be because you are using private mode. Please use regular mode or enable IndexedDB to use this dictionary"
);
};
const dictDb = new DictionaryDb({ const dictDb = new DictionaryDb({
url: dictionaryUrl, url: dictionaryUrl,
infoUrl: dictionaryInfoUrl, infoUrl: dictionaryInfoUrl,
collectionName: dictionaryCollectionName, collectionName: dictionaryCollectionName,
infoLocalStorageKey: dictionaryInfoLocalStorageKey, infoLocalStorageKey: dictionaryInfoLocalStorageKey,
}); });
function makeSearchStringSafe(searchString: string): string { function makeSearchStringSafe(searchString: string): string {
@ -57,9 +56,10 @@ function makeSearchStringSafe(searchString: string): string {
function fuzzifyEnglish(input: string): string { function fuzzifyEnglish(input: string): string {
const safeInput = input.trim().replace(/[#-.]|[[-^]|[?|{}]/g, ""); const safeInput = input.trim().replace(/[#-.]|[[-^]|[?|{}]/g, "");
// TODO: Could do: cover british/american things like offense / offence // TODO: Could do: cover british/american things like offense / offence
return safeInput.replace("to ", "") return safeInput
.replace(/our/g, "ou?r") .replace("to ", "")
.replace(/or/g, "ou?r"); .replace(/our/g, "ou?r")
.replace(/or/g, "ou?r");
} }
function chunkOutArray<T>(arr: T[], chunkSize: number): T[][] { function chunkOutArray<T>(arr: T[], chunkSize: number): T[][] {
@ -73,415 +73,465 @@ function chunkOutArray<T>(arr: T[], chunkSize: number): T[][] {
function getExpForInflections(input: string, index: "p" | "f"): RegExp { function getExpForInflections(input: string, index: "p" | "f"): RegExp {
let base = input; let base = input;
if (index === "f") { if (index === "f") {
if (["e", "é", "a", "á", "ó", "o"].includes(input.slice(-1))) { if (["e", "é", "a", "á", "ó", "o"].includes(input.slice(-1))) {
base = input.slice(0, -1); base = input.slice(0, -1);
} }
return new RegExp(`\\b${base}`); return new RegExp(`\\b${base}`);
} }
if (["ه", "ې", "و"].includes(input.slice(-1))) { if (["ه", "ې", "و"].includes(input.slice(-1))) {
base = input.slice(0, -1); base = input.slice(0, -1);
} }
return new RegExp(`^${base}[و|ې|ه]?`); return new RegExp(`^${base}[و|ې|ه]?`);
} }
function tsOneMonthBack(): number { function tsOneMonthBack(): number {
// https://stackoverflow.com/a/24049314/8620945 // https://stackoverflow.com/a/24049314/8620945
const d = new Date(); const d = new Date();
const m = d.getMonth(); const m = d.getMonth();
d.setMonth(d.getMonth() - 1); d.setMonth(d.getMonth() - 1);
// If still in same month, set date to last day of // If still in same month, set date to last day of
// previous month // previous month
if (d.getMonth() === m) d.setDate(0); if (d.getMonth() === m) d.setDate(0);
d.setHours(0, 0, 0); d.setHours(0, 0, 0);
d.setMilliseconds(0); d.setMilliseconds(0);
// Get the time value in milliseconds and convert to seconds // Get the time value in milliseconds and convert to seconds
return d.getTime(); return d.getTime();
} }
function alphabeticalLookup({ searchString, page }: { function alphabeticalLookup({
searchString: string, searchString,
page: number, page,
}: {
searchString: string;
page: number;
}): T.DictionaryEntry[] { }): T.DictionaryEntry[] {
const r = new RegExp("^" + sanitizePashto(makeSearchStringSafe(searchString))); const r = new RegExp(
const regexResults: T.DictionaryEntry[] = dictDb.collection.find({ "^" + sanitizePashto(makeSearchStringSafe(searchString))
$or: [ );
{p: { $regex: r }}, const regexResults: T.DictionaryEntry[] = dictDb.collection.find({
{g: { $regex: r }}, $or: [{ p: { $regex: r } }, { g: { $regex: r } }],
], });
}); const indexNumbers = regexResults.map((mpd: any) => mpd.i);
const indexNumbers = regexResults.map((mpd: any) => mpd.i); // Find the first matching word occuring first in the Pashto Index
// Find the first matching word occuring first in the Pashto Index let firstIndexNumber = null;
let firstIndexNumber = null; if (indexNumbers.length) {
if (indexNumbers.length) { firstIndexNumber = Math.min(...indexNumbers);
firstIndexNumber = Math.min(...indexNumbers); }
} // $gt query from that first occurance
// $gt query from that first occurance if (firstIndexNumber !== null) {
if (firstIndexNumber !== null) { return dictDb.collection
return dictDb.collection.chain() .chain()
.find({ i: { $gt: firstIndexNumber - 1 }}) .find({ i: { $gt: firstIndexNumber - 1 } })
.simplesort("i") .simplesort("i")
.limit(page * pageSize) .limit(page * pageSize)
.data(); .data();
} }
return []; return [];
} }
function fuzzyLookup<S extends T.DictionaryEntry>({ searchString, language, page, tpFilter }: { function fuzzyLookup<S extends T.DictionaryEntry>({
searchString: string, searchString,
language: "Pashto" | "English" | "Both", language,
page: number, page,
tpFilter?: (e: T.DictionaryEntry) => e is S, tpFilter,
}: {
searchString: string;
language: "Pashto" | "English" | "Both";
page: number;
tpFilter?: (e: T.DictionaryEntry) => e is S;
}): S[] { }): S[] {
// TODO: Implement working with both // TODO: Implement working with both
if (Number(searchString)) { if (Number(searchString)) {
const entry = dictionary.findOneByTs(Number(searchString)); const entry = dictionary.findOneByTs(Number(searchString));
// @ts-ignore; // @ts-ignore;
return entry ? [entry] : [] as S[]; return entry ? [entry] : ([] as S[]);
} }
return language === "Pashto" return language === "Pashto"
? pashtoFuzzyLookup({ searchString, page, tpFilter }) ? pashtoFuzzyLookup({ searchString, page, tpFilter })
: englishLookup({ searchString, page, tpFilter }) : englishLookup({ searchString, page, tpFilter });
} }
function englishLookup<S extends T.DictionaryEntry>({ searchString, page, tpFilter }: { function englishLookup<S extends T.DictionaryEntry>({
searchString: string, searchString,
page: number, page,
tpFilter?: (e: T.DictionaryEntry) => e is S, tpFilter,
}: {
searchString: string;
page: number;
tpFilter?: (e: T.DictionaryEntry) => e is S;
}): S[] { }): S[] {
function sortByR(a: T.DictionaryEntry, b: T.DictionaryEntry) { function sortByR(a: T.DictionaryEntry, b: T.DictionaryEntry) {
return (b.r || 3) - (a.r || 3); return (b.r || 3) - (a.r || 3);
}; }
let resultsGiven: number[] = []; let resultsGiven: number[] = [];
// get exact results // get exact results
const exactQuery = { const exactQuery = {
e: { e: {
$regex: new RegExp(`^${fuzzifyEnglish(searchString)}$`, "i"), $regex: new RegExp(`^${fuzzifyEnglish(searchString)}$`, "i"),
}, },
}; };
const exactResultsLimit = pageSize < 10 ? Math.floor(pageSize / 2) : 10; const exactResultsLimit = pageSize < 10 ? Math.floor(pageSize / 2) : 10;
const exactResults = dictDb.collection.chain() const exactResults = dictDb.collection
.find(exactQuery) .chain()
.limit(exactResultsLimit) .find(exactQuery)
.simplesort("i") .limit(exactResultsLimit)
.data(); .simplesort("i")
exactResults.sort(sortByR); .data();
resultsGiven = exactResults.map((mpd: any) => mpd.$loki); exactResults.sort(sortByR);
// get results with full word match at beginning of string resultsGiven = exactResults.map((mpd: any) => mpd.$loki);
const startingQuery = { // get results with full word match at beginning of string
e: { const startingQuery = {
$regex: new RegExp(`^${fuzzifyEnglish(searchString)}\\b`, "i"), e: {
}, $regex: new RegExp(`^${fuzzifyEnglish(searchString)}\\b`, "i"),
$loki: { $nin: resultsGiven }, },
}; $loki: { $nin: resultsGiven },
const startingResultsLimit = (pageSize * page) - resultsGiven.length; };
const startingResults = dictDb.collection.chain() const startingResultsLimit = pageSize * page - resultsGiven.length;
.find(startingQuery) const startingResults = dictDb.collection
.limit(startingResultsLimit) .chain()
.simplesort("i") .find(startingQuery)
.data(); .limit(startingResultsLimit)
startingResults.sort(sortByR); .simplesort("i")
resultsGiven = [...resultsGiven, ...startingResults.map((mpd: any) => mpd.$loki)]; .data();
// get results with full word match anywhere startingResults.sort(sortByR);
const fullWordQuery = { resultsGiven = [
e: { ...resultsGiven,
$regex: new RegExp(`\\b${fuzzifyEnglish(searchString)}\\b`, "i"), ...startingResults.map((mpd: any) => mpd.$loki),
}, ];
$loki: { $nin: resultsGiven }, // get results with full word match anywhere
}; const fullWordQuery = {
const fullWordResultsLimit = (pageSize * page) - resultsGiven.length; e: {
const fullWordResults = dictDb.collection.chain() $regex: new RegExp(`\\b${fuzzifyEnglish(searchString)}\\b`, "i"),
.find(fullWordQuery) },
.limit(fullWordResultsLimit) $loki: { $nin: resultsGiven },
.simplesort("i") };
.data(); const fullWordResultsLimit = pageSize * page - resultsGiven.length;
fullWordResults.sort(sortByR); const fullWordResults = dictDb.collection
resultsGiven = [...resultsGiven, ...fullWordResults.map((mpd: any) => mpd.$loki)] .chain()
// get results with partial match anywhere .find(fullWordQuery)
const partialMatchQuery = { .limit(fullWordResultsLimit)
e: { .simplesort("i")
$regex: new RegExp(`${fuzzifyEnglish(searchString)}`, "i"), .data();
}, fullWordResults.sort(sortByR);
$loki: { $nin: resultsGiven }, resultsGiven = [
}; ...resultsGiven,
const partialMatchLimit = (pageSize * page) - resultsGiven.length; ...fullWordResults.map((mpd: any) => mpd.$loki),
const partialMatchResults = dictDb.collection.chain() ];
.where(tpFilter ? tpFilter : () => true) // get results with partial match anywhere
.find(partialMatchQuery) const partialMatchQuery = {
.limit(partialMatchLimit) e: {
.simplesort("i") $regex: new RegExp(`${fuzzifyEnglish(searchString)}`, "i"),
.data(); },
partialMatchResults.sort(sortByR); $loki: { $nin: resultsGiven },
const results = [ };
...exactResults, const partialMatchLimit = pageSize * page - resultsGiven.length;
...startingResults, const partialMatchResults = dictDb.collection
...fullWordResults, .chain()
...partialMatchResults, .where(tpFilter ? tpFilter : () => true)
]; .find(partialMatchQuery)
if (tpFilter) { .limit(partialMatchLimit)
return results.filter(tpFilter); .simplesort("i")
} .data();
return results; partialMatchResults.sort(sortByR);
const results = [
...exactResults,
...startingResults,
...fullWordResults,
...partialMatchResults,
];
if (tpFilter) {
return results.filter(tpFilter);
}
return results;
} }
function pashtoExactLookup(searchString: string): T.DictionaryEntry[] { function pashtoExactLookup(searchString: string): T.DictionaryEntry[] {
const index = isPashtoScript(searchString) ? "p" : "g"; const index = isPashtoScript(searchString) ? "p" : "g";
const search = index === "g" ? simplifyPhonetics(searchString) : searchString; const search = index === "g" ? simplifyPhonetics(searchString) : searchString;
return dictDb.collection.find({ return dictDb.collection.find({
[index]: search, [index]: search,
}); });
} }
function pashtoFuzzyLookup<S extends T.DictionaryEntry>({ searchString, page, tpFilter }: { function pashtoFuzzyLookup<S extends T.DictionaryEntry>({
searchString: string, searchString,
page: number, page,
tpFilter?: (e: T.DictionaryEntry) => e is S, tpFilter,
}: {
searchString: string;
page: number;
tpFilter?: (e: T.DictionaryEntry) => e is S;
}): S[] { }): S[] {
let resultsGiven: number[] = []; let resultsGiven: number[] = [];
// Check if it's in Pashto or Latin script // Check if it's in Pashto or Latin script
const searchStringToUse = sanitizePashto(makeSearchStringSafe(searchString)); const searchStringToUse = sanitizePashto(makeSearchStringSafe(searchString));
const index = isPashtoScript(searchStringToUse) ? "p" : "g"; const index = isPashtoScript(searchStringToUse) ? "p" : "g";
const search = index === "g" ? simplifyPhonetics(searchStringToUse) : searchStringToUse; const search =
const infIndex = index === "p" ? "p" : "f"; index === "g" ? simplifyPhonetics(searchStringToUse) : searchStringToUse;
// Get exact matches const infIndex = index === "p" ? "p" : "f";
const exactExpression = new RegExp("^" + search); // Get exact matches
const weeBitFuzzy = new RegExp("^" + makeAWeeBitFuzzy(search, infIndex)); const exactExpression = new RegExp("^" + search);
// prepare exact expression for special matching const weeBitFuzzy = new RegExp("^" + makeAWeeBitFuzzy(search, infIndex));
// TODO: This is all a bit messy and could be done without regex // prepare exact expression for special matching
const expressionForInflections = getExpForInflections(search, infIndex); // TODO: This is all a bit messy and could be done without regex
const arabicPluralIndex = `ap${infIndex}`; const expressionForInflections = getExpForInflections(search, infIndex);
const pashtoPluralIndex = `pp${infIndex}`; const arabicPluralIndex = `ap${infIndex}`;
const presentStemIndex = `ps${infIndex}`; const pashtoPluralIndex = `pp${infIndex}`;
const firstInfIndex = `infa${infIndex}`; const presentStemIndex = `ps${infIndex}`;
const secondInfIndex = `infb${infIndex}`; const firstInfIndex = `infa${infIndex}`;
const pashtoExactResultFields = [ const secondInfIndex = `infb${infIndex}`;
{ const pashtoExactResultFields = [
[index]: { $regex: exactExpression }, {
}, { [index]: { $regex: exactExpression },
[arabicPluralIndex]: { $regex: weeBitFuzzy }, },
}, { {
[pashtoPluralIndex]: { $regex: weeBitFuzzy }, [arabicPluralIndex]: { $regex: weeBitFuzzy },
}, { },
[presentStemIndex]: { $regex: weeBitFuzzy }, {
}, [pashtoPluralIndex]: { $regex: weeBitFuzzy },
{ },
[firstInfIndex]: { $regex: expressionForInflections }, {
}, [presentStemIndex]: { $regex: weeBitFuzzy },
{ },
[secondInfIndex]: { $regex: expressionForInflections }, {
}, [firstInfIndex]: { $regex: expressionForInflections },
]; },
const exactQuery = { $or: [...pashtoExactResultFields] }; {
// just special incase using really small limits [secondInfIndex]: { $regex: expressionForInflections },
// multiple times scrolling / chunking / sorting might get a bit messed up if using a limit of less than 10 },
const exactResultsLimit = pageSize < 10 ? Math.floor(pageSize / 2) : 10; ];
const exactResults = dictDb.collection.chain() const exactQuery = { $or: [...pashtoExactResultFields] };
.find(exactQuery) // just special incase using really small limits
.limit(exactResultsLimit) // multiple times scrolling / chunking / sorting might get a bit messed up if using a limit of less than 10
.simplesort("i") const exactResultsLimit = pageSize < 10 ? Math.floor(pageSize / 2) : 10;
.data(); const exactResults = dictDb.collection
resultsGiven = exactResults.map((mpd: any) => mpd.$loki); .chain()
// Get slightly fuzzy matches .find(exactQuery)
const slightlyFuzzy = new RegExp(makeAWeeBitFuzzy(search, infIndex), "i"); .limit(exactResultsLimit)
const slightlyFuzzyQuery = { .simplesort("i")
[index]: { $regex: slightlyFuzzy }, .data();
$loki: { $nin: resultsGiven }, resultsGiven = exactResults.map((mpd: any) => mpd.$loki);
}; // Get slightly fuzzy matches
const slightlyFuzzyResultsLimit = (pageSize * page) - resultsGiven.length; const slightlyFuzzy = new RegExp(makeAWeeBitFuzzy(search, infIndex), "i");
const slightlyFuzzyResults = dictDb.collection.chain() const slightlyFuzzyQuery = {
.find(slightlyFuzzyQuery) [index]: { $regex: slightlyFuzzy },
.limit(slightlyFuzzyResultsLimit) $loki: { $nin: resultsGiven },
.data(); };
resultsGiven.push(...slightlyFuzzyResults.map((mpd: any) => mpd.$loki)); const slightlyFuzzyResultsLimit = pageSize * page - resultsGiven.length;
// Get fuzzy matches const slightlyFuzzyResults = dictDb.collection
const pashtoRegExLogic = fuzzifyPashto(search, { .chain()
script: index === "p" ? "Pashto" : "Latin", .find(slightlyFuzzyQuery)
simplifiedLatin: index === "g", .limit(slightlyFuzzyResultsLimit)
allowSpacesInWords: true, .data();
matchStart: "word", resultsGiven.push(...slightlyFuzzyResults.map((mpd: any) => mpd.$loki));
}); // Get fuzzy matches
const fuzzyPashtoExperssion = new RegExp(pashtoRegExLogic); const pashtoRegExLogic = fuzzifyPashto(search, {
const pashtoFuzzyQuery = [ script: index === "p" ? "Pashto" : "Latin",
{ simplifiedLatin: index === "g",
[index]: { $regex: fuzzyPashtoExperssion }, allowSpacesInWords: true,
}, { // TODO: Issue, this fuzzy doesn't line up well because it's not the simplified phonetics - still has 's etc matchStart: "word",
[arabicPluralIndex]: { $regex: fuzzyPashtoExperssion }, });
}, { const fuzzyPashtoExperssion = new RegExp(pashtoRegExLogic);
[presentStemIndex]: { $regex: fuzzyPashtoExperssion }, const pashtoFuzzyQuery = [
} {
]; [index]: { $regex: fuzzyPashtoExperssion },
// fuzzy results should be allowed to take up the rest of the limit (not used up by exact results) },
const fuzzyResultsLimit = (pageSize * page) - resultsGiven.length; {
// don't get these fuzzy results if searching in only English // TODO: Issue, this fuzzy doesn't line up well because it's not the simplified phonetics - still has 's etc
const fuzzyQuery = { [arabicPluralIndex]: { $regex: fuzzyPashtoExperssion },
$or: pashtoFuzzyQuery, },
$loki: { $nin: resultsGiven }, {
}; [presentStemIndex]: { $regex: fuzzyPashtoExperssion },
const fuzzyResults = dictDb.collection.chain() },
.find(fuzzyQuery) ];
.limit(fuzzyResultsLimit) // fuzzy results should be allowed to take up the rest of the limit (not used up by exact results)
.data(); const fuzzyResultsLimit = pageSize * page - resultsGiven.length;
const results = tpFilter // don't get these fuzzy results if searching in only English
? [...exactResults, ...slightlyFuzzyResults, ...fuzzyResults].filter(tpFilter) const fuzzyQuery = {
: [...exactResults, ...slightlyFuzzyResults, ...fuzzyResults]; $or: pashtoFuzzyQuery,
// sort out each chunk (based on limit used multiple times by infinite scroll) $loki: { $nin: resultsGiven },
// so that when infinite scrolling, it doesn't re-sort the previous chunks given };
const closeResultsLength = exactResults.length + slightlyFuzzyResults.length; const fuzzyResults = dictDb.collection
const chunksToSort = chunkOutArray(results, pageSize); .chain()
return chunksToSort .find(fuzzyQuery)
.reduce((acc, cur, i) => ((i === 0) .limit(fuzzyResultsLimit)
? [ .data();
...sortByRelevancy(cur.slice(0, closeResultsLength), search, index), const results = tpFilter
...sortByRelevancy(cur.slice(closeResultsLength), search, index), ? [...exactResults, ...slightlyFuzzyResults, ...fuzzyResults].filter(
] tpFilter
: [ )
...acc, : [...exactResults, ...slightlyFuzzyResults, ...fuzzyResults];
...sortByRelevancy(cur, search, index), // sort out each chunk (based on limit used multiple times by infinite scroll)
]), []); // so that when infinite scrolling, it doesn't re-sort the previous chunks given
const closeResultsLength = exactResults.length + slightlyFuzzyResults.length;
const chunksToSort = chunkOutArray(results, pageSize);
return chunksToSort.reduce(
(acc, cur, i) =>
i === 0
? [
...sortByRelevancy(cur.slice(0, closeResultsLength), search, index),
...sortByRelevancy(cur.slice(closeResultsLength), search, index),
]
: [...acc, ...sortByRelevancy(cur, search, index)],
[]
);
} }
function sortByRelevancy<T>(arr: T[], searchI: string, index: string): T[] { function sortByRelevancy<T>(arr: T[], searchI: string, index: string): T[] {
return relevancySorter.sort(arr, searchI, (obj: any, calc: any) => calc(obj[index])); return relevancySorter.sort(arr, searchI, (obj: any, calc: any) =>
calc(obj[index])
);
} }
function relatedWordsLookup(word: T.DictionaryEntry): T.DictionaryEntry[] { function relatedWordsLookup(word: T.DictionaryEntry): T.DictionaryEntry[] {
const wordArray = word.e.trim() const wordArray = word.e
.replace(/\?/g, "") .trim()
.replace(/( |,|\.|!|;|\(|\))/g, " ") .replace(/\?/g, "")
.split(/ +/) .replace(/( |,|\.|!|;|\(|\))/g, " ")
.filter((w: string) => !fillerWords.includes(w)); .split(/ +/)
let results: T.DictionaryEntry[] = []; .filter((w: string) => !fillerWords.includes(w));
wordArray.forEach((w: string) => { let results: T.DictionaryEntry[] = [];
let r: RegExp; wordArray.forEach((w: string) => {
try { let r: RegExp;
r = new RegExp(`\\b${w}\\b`, "i"); try {
const relatedToWord = dictDb.collection.chain() r = new RegExp(`\\b${w}\\b`, "i");
.find({ const relatedToWord = dictDb.collection
// don't include the original word .chain()
ts: { $ne: word.ts }, .find({
e: { $regex: r }, // don't include the original word
}) ts: { $ne: word.ts },
.limit(5) e: { $regex: r },
.data(); })
results = [...results, ...relatedToWord]; .limit(5)
// In case there's some weird regex fail .data();
} catch (error) { results = [...results, ...relatedToWord];
/* istanbul ignore next */ // In case there's some weird regex fail
console.error(error); } catch (error) {
} /* istanbul ignore next */
}); console.error(error);
// Remove duplicate items - https://stackoverflow.com/questions/40811451/remove-duplicates-from-a-array-of-objects }
results = results.filter(function(a) { });
// @ts-ignore // Remove duplicate items - https://stackoverflow.com/questions/40811451/remove-duplicates-from-a-array-of-objects
return !this[a.$loki] && (this[a.$loki] = true); results = results.filter(function (a) {
}, Object.create(null)); // @ts-ignore
return(results); return !this[a.$loki] && (this[a.$loki] = true);
}, Object.create(null));
return results;
} }
export function allEntries() { export function allEntries() {
return dictDb.collection.find(); return dictDb.collection.find();
} }
function makeLookupPortal<X extends T.DictionaryEntry>(tpFilter: (x: T.DictionaryEntry) => x is X): T.EntryLookupPortal<X> { function makeLookupPortal<X extends T.DictionaryEntry>(
return { tpFilter: (x: T.DictionaryEntry) => x is X
search: (s: string) => fuzzyLookup({ ): T.EntryLookupPortal<X> {
searchString: s, return {
language: "Pashto", search: (s: string) =>
page: 1, fuzzyLookup({
tpFilter, searchString: s,
}), language: "Pashto",
getByTs: (ts: number) => { page: 1,
const res = dictDb.findOneByTs(ts); tpFilter,
if (!res) return undefined; }),
return tpFilter(res) ? res : undefined; getByTs: (ts: number) => {
}, const res = dictDb.findOneByTs(ts);
} if (!res) return undefined;
return tpFilter(res) ? res : undefined;
},
};
} }
function makeVerbLookupPortal(): T.EntryLookupPortal<T.VerbEntry> { function makeVerbLookupPortal(): T.EntryLookupPortal<T.VerbEntry> {
return { return {
search: (s: string) => { search: (s: string) => {
const vEntries = fuzzyLookup({ const vEntries = fuzzyLookup({
searchString: s, searchString: s,
language: "Pashto", language: "Pashto",
page: 1, page: 1,
tpFilter: tp.isVerbDictionaryEntry, tpFilter: tp.isVerbDictionaryEntry,
}); });
return vEntries.map((entry): T.VerbEntry => ({ return vEntries.map(
entry, (entry): T.VerbEntry => ({
complement: (entry.c?.includes("comp.") && entry.l) entry,
? dictionary.findOneByTs(entry.l) complement:
: undefined, entry.c?.includes("comp.") && entry.l
})); ? dictionary.findOneByTs(entry.l)
}, : undefined,
getByTs: (ts: number): T.VerbEntry | undefined => { })
const entry = dictDb.findOneByTs(ts); );
if (!entry) return undefined; },
if (!tp.isVerbDictionaryEntry(entry)) { getByTs: (ts: number): T.VerbEntry | undefined => {
console.error("not valid verb entry"); const entry = dictDb.findOneByTs(ts);
return undefined; if (!entry) return undefined;
} if (!tp.isVerbDictionaryEntry(entry)) {
const complement = (() => { console.error("not valid verb entry");
if (entry.c?.includes("comp") && entry.l) { return undefined;
const comp = dictDb.findOneByTs(entry.l); }
if (!comp) { const complement = (() => {
console.error("complement not found for", entry); if (entry.c?.includes("comp") && entry.l) {
} const comp = dictDb.findOneByTs(entry.l);
return comp; if (!comp) {
} else { console.error("complement not found for", entry);
return undefined; }
} return comp;
})(); } else {
return { entry, complement }; return undefined;
}, }
} })();
return { entry, complement };
},
};
} }
export const entryFeeder: T.EntryFeeder = { export const entryFeeder: T.EntryFeeder = {
nouns: makeLookupPortal(tp.isNounEntry), nouns: makeLookupPortal(tp.isNounEntry),
verbs: makeVerbLookupPortal(), verbs: makeVerbLookupPortal(),
adjectives: makeLookupPortal(tp.isAdjectiveEntry), adjectives: makeLookupPortal(tp.isAdjectiveEntry),
locativeAdverbs: makeLookupPortal(tp.isLocativeAdverbEntry), locativeAdverbs: makeLookupPortal(tp.isLocativeAdverbEntry),
adverbs: makeLookupPortal(tp.isAdverbEntry), adverbs: makeLookupPortal(tp.isAdverbEntry),
} };
export const dictionary: DictionaryAPI = { export const dictionary: DictionaryAPI = {
// NOTE: For some reason that I do not understand you have to pass the functions from the // NOTE: For some reason that I do not understand you have to pass the functions from the
// dictionary core class in like this... ie. initialize: dictDb.initialize will mess up the this usage // dictionary core class in like this... ie. initialize: dictDb.initialize will mess up the this usage
// in the dictionary core class // in the dictionary core class
initialize: async () => await dictDb.initialize(), initialize: async () => await dictDb.initialize(),
update: async (notifyUpdateComing: () => void) => await dictDb.updateDictionary(notifyUpdateComing), update: async (notifyUpdateComing: () => void) =>
search: function(state: State): T.DictionaryEntry[] { await dictDb.updateDictionary(notifyUpdateComing),
const searchString = revertSpelling( search: function (state: State): T.DictionaryEntry[] {
state.searchValue, const searchString = revertSpelling(
getTextOptions(state).spelling, state.searchValue,
); getTextOptions(state).spelling
if (state.searchValue === "") { );
return []; if (state.searchValue === "") {
} return [];
return (state.options.searchType === "alphabetical" && state.options.language === "Pashto") }
? alphabeticalLookup({ return state.options.searchType === "alphabetical" &&
searchString, state.options.language === "Pashto"
page: state.page ? alphabeticalLookup({
}) searchString,
: fuzzyLookup({ page: state.page,
searchString, })
language: state.options.language, : fuzzyLookup({
page: state.page, searchString,
}); language: state.options.language,
}, page: state.page,
exactPashtoSearch: pashtoExactLookup, });
getNewWordsThisMonth: function(): T.DictionaryEntry[] { },
return dictDb.collection.chain() exactPashtoSearch: pashtoExactLookup,
.find({ ts: { $gt: tsOneMonthBack() }}) getNewWordsThisMonth: function (): T.DictionaryEntry[] {
.simplesort("ts") return dictDb.collection
.data() .chain()
.reverse(); .find({ ts: { $gt: tsOneMonthBack() } })
}, .simplesort("ts")
findOneByTs: (ts: number) => dictDb.findOneByTs(ts), .data()
findRelatedEntries: function(entry: T.DictionaryEntry): T.DictionaryEntry[] { .reverse();
return relatedWordsLookup(entry); },
}, findOneByTs: (ts: number) => dictDb.findOneByTs(ts),
} findRelatedEntries: function (entry: T.DictionaryEntry): T.DictionaryEntry[] {
return relatedWordsLookup(entry);
},
};

806
website/src/lib/fuzzify-pashto/fuzzify-pashto.test.ts
View File

@ -10,186 +10,186 @@ import { fuzzifyPashto } from "./fuzzify-pashto";
type match = [string, string]; type match = [string, string];
interface IDefaultInfoBlock { interface IDefaultInfoBlock {
matches: match[]; matches: match[];
nonMatches: match[]; nonMatches: match[];
} }
const defaultInfo: IDefaultInfoBlock = { const defaultInfo: IDefaultInfoBlock = {
matches: [ matches: [
["اوسېدل", "وسېدل"], ["اوسېدل", "وسېدل"],
["انبیه", "امبیه"], ["انبیه", "امبیه"],
["سرک", "صړق"], ["سرک", "صړق"],
["انطذاړ", "انتظار"], ["انطذاړ", "انتظار"],
["مالوم", "معلوم"], ["مالوم", "معلوم"],
["معلوم", "مالوم"], ["معلوم", "مالوم"],
["قېصا", "کيسه"], ["قېصا", "کيسه"],
["کور", "قوړ"], ["کور", "قوړ"],
["گرزيدل", "ګرځېدل"], ["گرزيدل", "ګرځېدل"],
["سنگہ", "څنګه"], ["سنگہ", "څنګه"],
["کار", "قهر"], ["کار", "قهر"],
["زبا", "ژبه"], ["زبا", "ژبه"],
["سڑے", "سړی"], ["سڑے", "سړی"],
["استمال", "استعمال"], ["استمال", "استعمال"],
["اعمل", "عمل"], ["اعمل", "عمل"],
["جنگل", "ځنګل"], ["جنگل", "ځنګل"],
["ځال", "جال"], ["ځال", "جال"],
["زنگل", "ځنګل"], ["زنگل", "ځنګل"],
["جرل", "ژړل"], ["جرل", "ژړل"],
["فرمائيل", "فرمايل"], ["فرمائيل", "فرمايل"],
["مرمنه", "مېرمنه"], ["مرمنه", "مېرمنه"],
// using هٔ as two characters // using هٔ as two characters
["وارېدهٔ", "وارېده"], ["وارېدهٔ", "وارېده"],
// using as one character // using as one character
["واريدۀ", "وارېده"], ["واريدۀ", "وارېده"],
["زوی", "زوئے"], ["زوی", "زوئے"],
["ئے", "يې"], ["ئے", "يې"],
// optional ا s in middle // optional ا s in middle
["توقف", "تواقف"], ["توقف", "تواقف"],
// option ي s in middle // option ي s in middle
["مناظره", "مناظيره"], ["مناظره", "مناظيره"],
["بلکل", "بالکل"], ["بلکل", "بالکل"],
["مهرب", "محراب"], ["مهرب", "محراب"],
["مسول", "مسوول"], ["مسول", "مسوول"],
["ډارونکي", "ډاروونکي"], ["ډارونکي", "ډاروونکي"],
["ډانګره", "ډانګوره"], ["ډانګره", "ډانګوره"],
["هنداره", "هینداره"], ["هنداره", "هینداره"],
["متأصفانه", "متاسفانه"], ["متأصفانه", "متاسفانه"],
["وازف", "واظیف"], ["وازف", "واظیف"],
["شوریٰ", "شورا"], ["شوریٰ", "شورا"],
["ځنبېدل", "ځمبېدل"], ["ځنبېدل", "ځمبېدل"],
// consonant swap // TODO: more?? // consonant swap // TODO: more??
["مچلوغزه", "مچلوزغه"], ["مچلوغزه", "مچلوزغه"],
["رکشه", "رشکه"], ["رکشه", "رشکه"],
["پښه", "ښپه"], ["پښه", "ښپه"],
], ],
nonMatches: [ nonMatches: [
["سرک", "ترک"], ["سرک", "ترک"],
["کار", "بېکاري"], ["کار", "بېکاري"],
// ا should not be optional in the beginning or end // ا should not be optional in the beginning or end
["اړتیا", "اړتی"], ["اړتیا", "اړتی"],
["ړتیا", "اړتیا"], ["ړتیا", "اړتیا"],
// و should not be optional in the begenning or end // و should not be optional in the begenning or end
["ورور", "رور"], ["ورور", "رور"],
], ],
}; };
const defaultLatinInfo: IDefaultInfoBlock = { const defaultLatinInfo: IDefaultInfoBlock = {
matches: [ matches: [
// TODO: // TODO:
["anbiya", "ambiya"], ["anbiya", "ambiya"],
["lootfun", "lUtfan"], ["lootfun", "lUtfan"],
["sarey", "saRey"], ["saray", "saRay"],
["senga", "tsanga"], ["senga", "tsanga"],
["daktur", "DakTar"], ["daktur", "DakTar"],
["iteebar", "itibaar"], ["iteebar", "itibaar"],
["dzaal", "jaal"], ["dzaal", "jaal"],
["bekaar", "bekáar"], ["bekaar", "bekáar"],
["bekár", "bekaar"], ["bekár", "bekaar"],
["chaai", "cháai"], ["chaai", "cháai"],
["day", "daai"], ["day", "daai"],
["dai", "dey"], ["dai", "day"],
["daktar", "Daktár"], ["daktar", "Daktár"],
["sarái", "saRey"], ["sarái", "saRay"],
["beter", "bahtár"], ["beter", "bahtár"],
["doosti", "dostee"], ["doosti", "dostee"],
["dắraghlum", "deraghlum"], // using the ă along with a combining ́ ["dắraghlum", "deraghlum"], // using the ă along with a combining ́
["dar", "dăr"], ["dar", "dăr"],
["der", "dăr"], ["der", "dăr"],
["dur", "dăr"], ["dur", "dăr"],
["chee", "che"], ["chee", "che"],
["dzooy", "zooy"], ["dzooy", "zooy"],
["delta", "dalta"], ["delta", "dalta"],
["koorbaani", "qUrbaanee"], ["koorbaani", "qUrbaanee"],
["jamaat", "jamaa'at"], ["jamaat", "jamaa'at"],
["taaroof", "ta'aarÚf"], ["taaroof", "ta'aarÚf"],
["xudza", "xúdza"], ["xudza", "xúdza"],
["ishaak", "is`haaq"], ["ishaak", "is`haaq"],
["lUtfun", "lootfan"], ["lUtfun", "lootfan"],
["miraab", "mihraab"], ["miraab", "mihraab"],
["taamul", "tahamul"], ["taamul", "tahamul"],
["otsedul", "osedul"], ["otsedul", "osedul"],
["ghaara", "ghaaRa"], ["ghaara", "ghaaRa"],
["maafiat", "maafiyat"], ["maafiat", "maafiyat"],
["tasalUt", "tassalUt"], ["tasalUt", "tassalUt"],
], ],
nonMatches: [ nonMatches: [
["kor", "por"], ["kor", "por"],
["intizaar", "intizaam"], ["intizaar", "intizaam"],
["ishaat", "shaat"], // i should not be optional at the beginning ["ishaat", "shaat"], // i should not be optional at the beginning
], ],
}; };
const withDiacritics: match[] = [ const withDiacritics: match[] = [
["تتتت", "تِتّتّت"], ["تتتت", "تِتّتّت"],
["بببب", "بّبّبَب"], ["بببب", "بّبّبَب"],
]; ];
const matchesWithAn: match[] = [ const matchesWithAn: match[] = [
["حتمن", "حتماً"], ["حتمن", "حتماً"],
["لتفن", "لطفاً"], ["لتفن", "لطفاً"],
["کاملا", "کاملاً"], ["کاملا", "کاملاً"],
]; ];
const matchesWithSpaces: match[] = [ const matchesWithSpaces: match[] = [
["دپاره", "د پاره"], ["دپاره", "د پاره"],
["بېکار", "بې کار"], ["بېکار", "بې کار"],
["د پاره", "دپاره"], ["د پاره", "دپاره"],
["بې کار", "بېکار"], ["بې کار", "بېکار"],
["کار مند", "کارمند"], ["کار مند", "کارمند"],
["همنشین", "هم نشین"], ["همنشین", "هم نشین"],
["بغل کشي", "بغلکشي"], ["بغل کشي", "بغلکشي"],
]; ];
const matchesWithSpacesLatin: match[] = [ const matchesWithSpacesLatin: match[] = [
["dupaara", "du paara"], ["dupaara", "du paara"],
["bekaara", "be kaara"], ["bekaara", "be kaara"],
["du paara", "dupaara"], ["du paara", "dupaara"],
["be kaara", "bekaara"], ["be kaara", "bekaara"],
["oreckbgqjxmroe", "or ec kb gq jxmr oe"], ["oreckbgqjxmroe", "or ec kb gq jxmr oe"],
["cc cc c", "ccccc"], ["cc cc c", "ccccc"],
]; ];
const defaultSimpleLatinInfo: IDefaultInfoBlock = { const defaultSimpleLatinInfo: IDefaultInfoBlock = {
matches: [ matches: [
// TODO: // TODO:
["anbiya", "ambiya"], ["anbiya", "ambiya"],
["lootfun", "lUtfan"], ["lootfun", "lUtfan"],
["sarey", "saRey"], ["saray", "saRay"],
["senga", "tsanga"], ["senga", "tsanga"],
["daktur", "DakTar"], ["daktur", "DakTar"],
["iteebar", "itibaar"], ["iteebar", "itibaar"],
["dzaal", "jaal"], ["dzaal", "jaal"],
["bekaar", "bekaar"], ["bekaar", "bekaar"],
["bekar", "bekaar"], ["bekar", "bekaar"],
["chaai", "chaai"], ["chaai", "chaai"],
["day", "daai"], ["day", "daai"],
["dai", "dey"], ["dai", "day"],
["daktar", "Daktar"], ["daktar", "Daktar"],
["sarai", "saRey"], ["sarai", "saRay"],
["beter", "bahtar"], ["beter", "bahtar"],
["doosti", "dostee"], ["doosti", "dostee"],
["daraghlum", "deraghlum"], // using the ă along with a combining ́ ["daraghlum", "deraghlum"], // using the ă along with a combining ́
["dar", "dar"], ["dar", "dar"],
["der", "dar"], ["der", "dar"],
["dur", "dar"], ["dur", "dar"],
["chee", "che"], ["chee", "che"],
["dzooy", "zooy"], ["dzooy", "zooy"],
["delta", "dalta"], ["delta", "dalta"],
["koorbaani", "qUrbaanee"], ["koorbaani", "qUrbaanee"],
["taaroof", "taaarUf"], ["taaroof", "taaarUf"],
["xudza", "xudza"], ["xudza", "xudza"],
["ishaak", "ishaaq"], ["ishaak", "ishaaq"],
["lUtfun", "lootfan"], ["lUtfun", "lootfan"],
["miraab", "mihraab"], ["miraab", "mihraab"],
["taamul", "tahamul"], ["taamul", "tahamul"],
["otsedul", "osedul"], ["otsedul", "osedul"],
["ghaara", "ghaaRa"], ["ghaara", "ghaaRa"],
], ],
nonMatches: [ nonMatches: [
["kor", "por"], ["kor", "por"],
["intizaar", "intizaam"], ["intizaar", "intizaam"],
["ishaat", "shaat"], // i should not be optional at the beginning ["ishaat", "shaat"], // i should not be optional at the beginning
], ],
}; };
interface ITestOptions { interface ITestOptions {
@ -200,267 +200,301 @@ interface ITestOptions {
} }
const optionsPossibilities: ITestOptions[] = [ const optionsPossibilities: ITestOptions[] = [
{ {
options: {}, // default options: {}, // default
...defaultInfo, ...defaultInfo,
viceVersaMatches: true, viceVersaMatches: true,
}, },
{ {
options: { script: "Latin" }, options: { script: "Latin" },
...defaultLatinInfo, ...defaultLatinInfo,
viceVersaMatches: true, viceVersaMatches: true,
}, },
{ {
options: {matchStart: "word"}, // same as default options: { matchStart: "word" }, // same as default
...defaultInfo, ...defaultInfo,
viceVersaMatches: true, viceVersaMatches: true,
}, },
{ {
options: { script: "Latin", simplifiedLatin: true }, options: { script: "Latin", simplifiedLatin: true },
...defaultSimpleLatinInfo, ...defaultSimpleLatinInfo,
viceVersaMatches: true, viceVersaMatches: true,
}, },
{ {
matches: [ matches: [...matchesWithSpaces],
...matchesWithSpaces, nonMatches: [],
], options: { allowSpacesInWords: true },
nonMatches: [], viceVersaMatches: true,
options: {allowSpacesInWords: true}, },
viceVersaMatches: true, {
}, matches: [...matchesWithSpacesLatin],
{ nonMatches: [],
matches: [ options: { allowSpacesInWords: true, script: "Latin" },
...matchesWithSpacesLatin, viceVersaMatches: true,
], },
nonMatches: [], {
options: {allowSpacesInWords: true, script: "Latin"}, matches: [],
viceVersaMatches: true, nonMatches: matchesWithSpaces,
}, options: { allowSpacesInWords: false },
{ },
matches: [], {
nonMatches: matchesWithSpaces, matches: [],
options: {allowSpacesInWords: false}, nonMatches: matchesWithSpacesLatin,
}, options: { allowSpacesInWords: false, script: "Latin" },
{ },
matches: [], {
nonMatches: matchesWithSpacesLatin, matches: [["کار", "بېکاري"]],
options: {allowSpacesInWords: false, script: "Latin"}, nonMatches: [["سرک", "بېترک"]],
}, options: { matchStart: "anywhere" },
{ },
matches: [ {
["کار", "بېکاري"], matches: [
], ["کور", "کور"],
nonMatches: [ ["سری", "سړی"],
["سرک", "بېترک"], ],
], nonMatches: [
options: {matchStart: "anywhere"}, ["سړي", "سړيتوب"],
}, ["کور", "کورونه"],
{ ],
matches: [ options: { matchWholeWordOnly: true },
["کور", "کور"], viceVersaMatches: true,
["سری", "سړی"], },
], {
nonMatches: [ matches: [
["سړي", "سړيتوب"], ["کور", "کور ته ځم"],
["کور", "کورونه"], ["سری", "سړی دی"],
], ],
options: {matchWholeWordOnly: true}, nonMatches: [
viceVersaMatches: true, ["سړي", " سړيتوب"],
}, ["کور", "خټين کورونه"],
{ ],
matches: [ options: { matchStart: "string" },
["کور", "کور ته ځم"], },
["سری", "سړی دی"], {
], matches: [
nonMatches: [ ["کور", "کور ته ځم"],
["سړي", " سړيتوب"], ["سری", "سړی دی"],
["کور", "خټين کورونه"], ],
], nonMatches: [
options: {matchStart: "string"}, ["سړي", " سړيتوب"],
}, ["کور", "خټين کورونه"],
{ ],
matches: [ options: { matchStart: "string" },
["کور", "کور ته ځم"], },
["سری", "سړی دی"],
],
nonMatches: [
["سړي", " سړيتوب"],
["کور", "خټين کورونه"],
],
options: {matchStart: "string"},
},
]; ];
const punctuationToExclude = [ const punctuationToExclude = [
"،", "؟", "؛", "۔", "۲", "۹", "۰", "»", "«", "٫", "!", ".", "؋", "٪", "٬", "×", ")", "(", " ", "\t", "،",
"؟",
"؛",
"۔",
"۲",
"۹",
"۰",
"»",
"«",
"٫",
"!",
".",
"؋",
"٪",
"٬",
"×",
")",
"(",
" ",
"\t",
]; ];
optionsPossibilities.forEach((o) => { optionsPossibilities.forEach((o) => {
o.matches.forEach((m: any) => {
test(`${m[0]} should match ${m[1]}`, () => {
const re = fuzzifyPashto(m[0], o.options);
// eslint-disable-next-line
const result = m[1].match(new RegExp(re));
expect(result).toBeTruthy();
});
});
if (o.viceVersaMatches === true) {
o.matches.forEach((m: any) => { o.matches.forEach((m: any) => {
test(`${m[0]} should match ${m[1]}`, () => { test(`${m[1]} should match ${m[0]}`, () => {
const re = fuzzifyPashto(m[0], o.options); const re = fuzzifyPashto(m[1], o.options);
// eslint-disable-next-line // eslint-disable-next-line
const result = m[1].match(new RegExp(re)); const result = m[0].match(new RegExp(re));
expect(result).toBeTruthy(); expect(result).toBeTruthy();
}); });
}); });
if (o.viceVersaMatches === true) { }
o.matches.forEach((m: any) => { o.nonMatches.forEach((m: any) => {
test(`${m[1]} should match ${m[0]}`, () => { test(`${m[0]} should not match ${m[1]}`, () => {
const re = fuzzifyPashto(m[1], o.options); const re = fuzzifyPashto(m[0], o.options);
// eslint-disable-next-line // eslint-disable-next-line
const result = m[0].match(new RegExp(re)); const result = m[1].match(new RegExp(re));
expect(result).toBeTruthy(); expect(result).toBeNull();
});
});
}
o.nonMatches.forEach((m: any) => {
test(`${m[0]} should not match ${m[1]}`, () => {
const re = fuzzifyPashto(m[0], o.options);
// eslint-disable-next-line
const result = m[1].match(new RegExp(re));
expect(result).toBeNull();
});
}); });
});
}); });
matchesWithAn.forEach((m: any) => { matchesWithAn.forEach((m: any) => {
test(`matching ${m[0]} should work with ${m[1]}`, () => { test(`matching ${m[0]} should work with ${m[1]}`, () => {
const re = fuzzifyPashto(m[0], { matchWholeWordOnly: true }); const re = fuzzifyPashto(m[0], { matchWholeWordOnly: true });
// eslint-disable-next-line // eslint-disable-next-line
const result = m[1].match(new RegExp(re)); const result = m[1].match(new RegExp(re));
expect(result).toBeTruthy(); expect(result).toBeTruthy();
}); });
test(`matching ${m[1]} should work with ${m[0]}`, () => { test(`matching ${m[1]} should work with ${m[0]}`, () => {
const re = fuzzifyPashto(m[1], { matchWholeWordOnly: true }); const re = fuzzifyPashto(m[1], { matchWholeWordOnly: true });
// eslint-disable-next-line // eslint-disable-next-line
const result = m[0].match(new RegExp(re)); const result = m[0].match(new RegExp(re));
expect(result).toBeTruthy(); expect(result).toBeTruthy();
}); });
}); });
withDiacritics.forEach((m: any) => { withDiacritics.forEach((m: any) => {
test(`matich ${m[0]} should ignore the diactritics in ${m[1]}`, () => { test(`matich ${m[0]} should ignore the diactritics in ${m[1]}`, () => {
const re = fuzzifyPashto(m[0], { ignoreDiacritics: true }); const re = fuzzifyPashto(m[0], { ignoreDiacritics: true });
// eslint-disable-next-line // eslint-disable-next-line
const result = m[1].match(new RegExp(re)); const result = m[1].match(new RegExp(re));
expect(result).toBeTruthy(); expect(result).toBeTruthy();
}); });
test(`the diacritics should in ${m[0]} should be ignored when matching with ${m[1]}`, () => { test(`the diacritics should in ${m[0]} should be ignored when matching with ${m[1]}`, () => {
const re = fuzzifyPashto(m[1], { ignoreDiacritics: true }); const re = fuzzifyPashto(m[1], { ignoreDiacritics: true });
// eslint-disable-next-line // eslint-disable-next-line
const result = m[0].match(new RegExp(re)); const result = m[0].match(new RegExp(re));
expect(result).toBeTruthy(); expect(result).toBeTruthy();
}); });
}); });
test(`وs should be optional if entered in search string`, () => { test(`وs should be optional if entered in search string`, () => {
const re = fuzzifyPashto("لوتفن"); const re = fuzzifyPashto("لوتفن");
// eslint-disable-next-line // eslint-disable-next-line
const result = "لطفاً".match(new RegExp(re)); const result = "لطفاً".match(new RegExp(re));
expect(result).toBeTruthy(); expect(result).toBeTruthy();
}); });
test(`matchWholeWordOnly should override matchStart = "anywhere"`, () => { test(`matchWholeWordOnly should override matchStart = "anywhere"`, () => {
const re = fuzzifyPashto("کار", { matchWholeWordOnly: true, matchStart: "anywhere" }); const re = fuzzifyPashto("کار", {
// eslint-disable-next-line matchWholeWordOnly: true,
const result = "کار کوه، بېکاره مه ګرځه".match(new RegExp(re)); matchStart: "anywhere",
expect(result).toHaveLength(1); });
expect(result).toEqual(expect.not.arrayContaining(["بېکاره"])); // eslint-disable-next-line
const result = "کار کوه، بېکاره مه ګرځه".match(new RegExp(re));
expect(result).toHaveLength(1);
expect(result).toEqual(expect.not.arrayContaining(["بېکاره"]));
}); });
test(`returnWholeWord should return the whole word`, () => { test(`returnWholeWord should return the whole word`, () => {
// With Pashto Script // With Pashto Script
const re = fuzzifyPashto("کار", { returnWholeWord: true }); const re = fuzzifyPashto("کار", { returnWholeWord: true });
// eslint-disable-next-line // eslint-disable-next-line
const result = "کارونه کوه، بېکاره مه شه".match(new RegExp(re)); const result = "کارونه کوه، بېکاره مه شه".match(new RegExp(re));
expect(result).toHaveLength(1); expect(result).toHaveLength(1);
expect(result).toContain("کارونه"); expect(result).toContain("کارونه");
// With Latin Script // With Latin Script
const reLatin = fuzzifyPashto("kaar", { const reLatin = fuzzifyPashto("kaar", {
returnWholeWord: true, returnWholeWord: true,
script: "Latin", script: "Latin",
}); });
// eslint-disable-next-line // eslint-disable-next-line
const resultLatin = "kaaroona kawa, bekaara ma gurdza.".match(new RegExp(reLatin)); const resultLatin = "kaaroona kawa, bekaara ma gurdza.".match(
expect(resultLatin).toHaveLength(1); new RegExp(reLatin)
expect(resultLatin).toContain("kaaroona"); );
expect(resultLatin).toHaveLength(1);
expect(resultLatin).toContain("kaaroona");
}); });
test(`returnWholeWord should return the whole word even when starting the matching in the middle`, () => { test(`returnWholeWord should return the whole word even when starting the matching in the middle`, () => {
// With Pashto Script // With Pashto Script
const re = fuzzifyPashto("کار", { returnWholeWord: true, matchStart: "anywhere" }); const re = fuzzifyPashto("کار", {
// eslint-disable-next-line returnWholeWord: true,
const result = "کارونه کوه، بېکاره مه شه".match(new RegExp(re, "g")); matchStart: "anywhere",
expect(result).toHaveLength(2); });
expect(result).toContain(" بېکاره"); // eslint-disable-next-line
const result = "کارونه کوه، بېکاره مه شه".match(new RegExp(re, "g"));
expect(result).toHaveLength(2);
expect(result).toContain(" بېکاره");
// With Latin Script // With Latin Script
const reLatin = fuzzifyPashto("kaar", { const reLatin = fuzzifyPashto("kaar", {
matchStart: "anywhere", matchStart: "anywhere",
returnWholeWord: true, returnWholeWord: true,
script: "Latin", script: "Latin",
}); });
// eslint-disable-next-line // eslint-disable-next-line
const resultLatin = "kaaroona kawa bekaara ma gurdza".match(new RegExp(reLatin, "g")); const resultLatin = "kaaroona kawa bekaara ma gurdza".match(
expect(resultLatin).toHaveLength(2); new RegExp(reLatin, "g")
expect(resultLatin).toContain("bekaara"); );
expect(resultLatin).toHaveLength(2);
expect(resultLatin).toContain("bekaara");
}); });
test(`returnWholeWord should should not return partial matches if matchWholeWordOnly is true`, () => { test(`returnWholeWord should should not return partial matches if matchWholeWordOnly is true`, () => {
// With Pashto Script // With Pashto Script
const re = fuzzifyPashto("کار", { returnWholeWord: true, matchStart: "anywhere", matchWholeWordOnly: true }); const re = fuzzifyPashto("کار", {
// eslint-disable-next-line returnWholeWord: true,
const result = "کارونه کوه، بېکاره مه ګرځه".match(new RegExp(re)); matchStart: "anywhere",
expect(result).toBeNull(); matchWholeWordOnly: true,
});
// eslint-disable-next-line
const result = "کارونه کوه، بېکاره مه ګرځه".match(new RegExp(re));
expect(result).toBeNull();
// With Latin Script // With Latin Script
const reLatin = fuzzifyPashto("kaar", { const reLatin = fuzzifyPashto("kaar", {
matchStart: "anywhere", matchStart: "anywhere",
matchWholeWordOnly: true, matchWholeWordOnly: true,
returnWholeWord: true,
script: "Latin",
});
// eslint-disable-next-line
const resultLatin = "kaaroona kawa bekaara ma gurdza".match(
new RegExp(reLatin)
);
expect(resultLatin).toBeNull();
});
punctuationToExclude.forEach((m) => {
test(`${m} should not be considered part of a Pashto word`, () => {
const re = fuzzifyPashto("کور", {
returnWholeWord: true,
matchStart: "word",
});
// ISSUE: This should also work when the word is PRECEDED by the punctuation
// Need to work with a lookbehind equivalent
// eslint-disable-next-line
const result = `زمونږ کورونه${m} دي`.match(new RegExp(re));
expect(result).toHaveLength(1);
expect(result).toContain(" کورونه");
// Matches will unfortunately have a space on the front of the word, issue with missing es2018 lookbehinds
});
});
punctuationToExclude.forEach((m) => {
// tslint:disable-next-line
test(`${m} should not be considered part of a Pashto word (front or back with es2018) - or should fail if using a non es2018 environment`, () => {
let result: any;
let failed = false;
// if environment is not es2018 with lookbehind support (like node 6, 8) this will fail
try {
const re = fuzzifyPashto("کور", {
returnWholeWord: true, returnWholeWord: true,
script: "Latin", matchStart: "word",
}); es2018: true,
// eslint-disable-next-line });
const resultLatin = "kaaroona kawa bekaara ma gurdza".match(new RegExp(reLatin)); // eslint-disable-next-line
expect(resultLatin).toBeNull(); result = `زمونږ ${m}کورونه${m} دي`.match(new RegExp(re));
}); } catch (error) {
failed = true;
punctuationToExclude.forEach((m) => { }
test(`${m} should not be considered part of a Pashto word`, () => { const worked = failed || (result.length === 1 && result.includes("کورونه"));
const re = fuzzifyPashto("کور", { returnWholeWord: true, matchStart: "word" }); expect(worked).toBe(true);
// ISSUE: This should also work when the word is PRECEDED by the punctuation });
// Need to work with a lookbehind equivalent
// eslint-disable-next-line
const result = `زمونږ کورونه${m} دي`.match(new RegExp(re));
expect(result).toHaveLength(1);
expect(result).toContain(" کورونه");
// Matches will unfortunately have a space on the front of the word, issue with missing es2018 lookbehinds
});
});
punctuationToExclude.forEach((m) => {
// tslint:disable-next-line
test(`${m} should not be considered part of a Pashto word (front or back with es2018) - or should fail if using a non es2018 environment`, () => {
let result: any;
let failed = false;
// if environment is not es2018 with lookbehind support (like node 6, 8) this will fail
try {
const re = fuzzifyPashto("کور", { returnWholeWord: true, matchStart: "word", es2018: true });
// eslint-disable-next-line
result = `زمونږ ${m}کورونه${m} دي`.match(new RegExp(re));
} catch (error) {
failed = true;
}
const worked = failed || (result.length === 1 && result.includes("کورونه"));
expect(worked).toBe(true);
});
}); });
test(`Arabic punctuation or numbers should not be considered part of a Pashto word`, () => { test(`Arabic punctuation or numbers should not be considered part of a Pashto word`, () => {
const re = fuzzifyPashto("کار", { returnWholeWord: true }); const re = fuzzifyPashto("کار", { returnWholeWord: true });
// eslint-disable-next-line // eslint-disable-next-line
const result = "کارونه کوه، بېکاره مه ګرځه".match(new RegExp(re)); const result = "کارونه کوه، بېکاره مه ګرځه".match(new RegExp(re));
expect(result).toHaveLength(1); expect(result).toHaveLength(1);
expect(result).toContain("کارونه"); expect(result).toContain("کارونه");
}); });

254
website/src/lib/fuzzify-pashto/replacer.ts
View File

@ -14,7 +14,7 @@ const velarPlosives = "ګغږکقگك";
const rLikeSounds = "رړڑڼ"; const rLikeSounds = "رړڑڼ";
const labialPlosivesAndFricatives = "فپب"; const labialPlosivesAndFricatives = "فپب";
// Includes Arabic ى \u0649 // Includes Arabic ى \u0649
const theFiveYeys = "ېۍیيئےى"; const theFiveYays = "ېۍیيئےى";
const guttural = "ښخشخهحغګ"; const guttural = "ښخشخهحغګ";
interface IReplacerInfoItem { interface IReplacerInfoItem {
@ -38,7 +38,6 @@ const ghzCombo = ["غز", "زغ"];
const pxCombo = ["پښ", "ښپ"]; const pxCombo = ["پښ", "ښپ"];
const kshCombo = ["کش", "شک", "کښ", "کش"]; const kshCombo = ["کش", "شک", "کښ", "کش"];
export const pashtoReplacerInfo: IPashtoReplacerInfoItem[] = [ export const pashtoReplacerInfo: IPashtoReplacerInfoItem[] = [
{ char: "اً", range: "ان" }, { char: "اً", range: "ان" },
{ {
@ -54,15 +53,25 @@ export const pashtoReplacerInfo: IPashtoReplacerInfoItem[] = [
{ char: "ٳ", range: "اآهأ" }, { char: "ٳ", range: "اآهأ" },
{ char: "یٰ", range: "ای", plus: ["یٰ"] }, { char: "یٰ", range: "ای", plus: ["یٰ"] },
{ char: "ی", range: theFiveYeys, plus: ["ئی", "ئي", "یٰ"], ignorableIfInMiddle: true }, {
{ char: "ي", range: theFiveYeys, plus: ["ئی", "ئي", "یٰ"], ignorableIfInMiddle: true }, char: "ی",
{ char: "ې", range: theFiveYeys, ignorableIfInMiddle: true }, range: theFiveYays,
{ char: "ۍ", range: theFiveYeys }, plus: ["ئی", "ئي", "یٰ"],
{ char: "ئي", range: theFiveYeys, plus: ["ئی", "ئي"] }, ignorableIfInMiddle: true,
{ char: "ئی", range: theFiveYeys, plus: ["ئی", "ئي"] }, },
{ char: "ئے", range: theFiveYeys, plus: ["ئی", "ئي", "يې"]}, {
{ char: "ئ", range: theFiveYeys, ignorableIfInMiddle: true }, char: "ي",
{ char: "ے", range: theFiveYeys }, range: theFiveYays,
plus: ["ئی", "ئي", "یٰ"],
ignorableIfInMiddle: true,
},
{ char: "ې", range: theFiveYays, ignorableIfInMiddle: true },
{ char: "ۍ", range: theFiveYays },
{ char: "ئي", range: theFiveYays, plus: ["ئی", "ئي"] },
{ char: "ئی", range: theFiveYays, plus: ["ئی", "ئي"] },
{ char: "ئے", range: theFiveYays, plus: ["ئی", "ئي", "يې"] },
{ char: "ئ", range: theFiveYays, ignorableIfInMiddle: true },
{ char: "ے", range: theFiveYays },
{ char: "س", range: sSounds }, { char: "س", range: sSounds },
{ char: "ص", range: sSounds }, { char: "ص", range: sSounds },
@ -79,7 +88,7 @@ export const pashtoReplacerInfo: IPashtoReplacerInfoItem[] = [
{ char: "ع", range: "اوع", ignorable: true }, { char: "ع", range: "اوع", ignorable: true },
{ char: "و", range: "وع", plus: ["وو"], ignorableIfInMiddle: true }, { char: "و", range: "وع", plus: ["وو"], ignorableIfInMiddle: true },
{ char: "ؤ", range: "وع"}, { char: "ؤ", range: "وع" },
{ char: "ښ", range: guttural }, { char: "ښ", range: guttural },
{ char: "غ", range: guttural }, { char: "غ", range: guttural },
@ -91,7 +100,7 @@ export const pashtoReplacerInfo: IPashtoReplacerInfoItem[] = [
{ char: "ز", range: zSounds }, { char: "ز", range: zSounds },
{ char: "ض", range: zSounds }, { char: "ض", range: zSounds },
{ char: "ذ", range: zSounds }, { char: "ذ", range: zSounds },
{ char: "ځ", range: zSounds + "جڅ"}, { char: "ځ", range: zSounds + "جڅ" },
{ char: "ظ", range: zSounds }, { char: "ظ", range: zSounds },
{ char: "ژ", range: "زضظژذځږج" }, { char: "ژ", range: "زضظژذځږج" },
@ -133,11 +142,12 @@ export const pashtoReplacerInfo: IPashtoReplacerInfoItem[] = [
]; ];
// tslint:disable-next-line // tslint:disable-next-line
export const pashtoReplacerRegex = /اً|أ|ا|آ|ٱ|ٲ|ٳ|ئی|ئي|ئے|یٰ|ی|ي|ې|ۍ|ئ|ے|س|ص|ث|څ|ج|چ|هٔ|ه|ۀ|غز|زغ|کش|شک|ښک|ښک|پښ|ښپ|ہ|ع|و|ؤ|ښ|غ|خ|ح|ش|ز|ض|ذ|ځ|ظ|ژ|ر|ړ|ڑ|ت|ټ|ٹ|ط|د|ډ|ڈ|مب|م|نب|ن|ڼ|ک|ګ|گ|ل|ق|ږ|ب|پ|ف/g; export const pashtoReplacerRegex =
/اً|أ|ا|آ|ٱ|ٲ|ٳ|ئی|ئي|ئے|یٰ|ی|ي|ې|ۍ|ئ|ے|س|ص|ث|څ|ج|چ|هٔ|ه|ۀ|غز|زغ|کش|شک|ښک|ښک|پښ|ښپ|ہ|ع|و|ؤ|ښ|غ|خ|ح|ش|ز|ض|ذ|ځ|ظ|ژ|ر|ړ|ڑ|ت|ټ|ٹ|ط|د|ډ|ڈ|مب|م|نب|ن|ڼ|ک|ګ|گ|ل|ق|ږ|ب|پ|ف/g;
// TODO: I removed the h? 's at the beginning and ends. was that a good idea? // TODO: I removed the h? 's at the beginning and ends. was that a good idea?
const aaySoundLatin = "(?:[aá]a?i|[eé]y|[aá]a?y|[aá]h?i)"; const aaySoundLatin = "(?:[aá]a?i|[eé]y|[aá]a?y|[aá]h?i)";
const aaySoundSimpleLatin = "(?:aa?i|ey|aa?y|ah?i)"; const aaySoundSimpleLatin = "(?:aa?i|ay|aa?y|ah?i)";
const longASoundLatin = "(?:[aá]{1,2}'?h?a{0,2}?)h?"; const longASoundLatin = "(?:[aá]{1,2}'?h?a{0,2}?)h?";
const longASoundSimpleLatin = "(?:a{1,2}'?h?a{0,2}?)h?"; const longASoundSimpleLatin = "(?:a{1,2}'?h?a{0,2}?)h?";
const shortASoundLatin = "(?:[aáă][a|́]?|au|áu|[uú]|[UÚ]|[ií]|[eé])?h?"; const shortASoundLatin = "(?:[aáă][a|́]?|au|áu|[uú]|[UÚ]|[ií]|[eé])?h?";
@ -146,8 +156,8 @@ const shwaSoundLatin = "(?:[uú]|[oó]o?|w[uú]|[aáă]|[ií]|[UÚ])?";
const shwaSoundSimpleLatin = "(?:u|oo?|wu|a|i|U)?"; const shwaSoundSimpleLatin = "(?:u|oo?|wu|a|i|U)?";
const ooSoundLatin = "(?:[oó]o?|[áa]u|w[uú]|[aá]w|[uú]|[UÚ])(?:h|w)?"; const ooSoundLatin = "(?:[oó]o?|[áa]u|w[uú]|[aá]w|[uú]|[UÚ])(?:h|w)?";
const ooSoundSimpleLatin = "(?:oo?|au|wu|aw|u|U)(?:h|w)?"; const ooSoundSimpleLatin = "(?:oo?|au|wu|aw|u|U)(?:h|w)?";
const eySoundLatin = "(?:[eé]y|[eé]e?|[uú]y|[aá]y|[ií])"; const aySoundLatin = "(?:[eé]y|[eé]e?|[uú]y|[aá]y|[ií])";
const eySoundSimpleLatin = "(?:ey|ee?|uy|ay|i)"; const aySoundSimpleLatin = "(?:ay|ee?|uy|ay|i)";
const middleESoundLatin = "(?:[eé]e?|[ií]|[aáă]|[eé])[h|y|́]?"; const middleESoundLatin = "(?:[eé]e?|[ií]|[aáă]|[eé])[h|y|́]?";
const middleESoundSimpleLatin = "(?:ee?|i|a|e)[h|y]?"; const middleESoundSimpleLatin = "(?:ee?|i|a|e)[h|y]?";
const iSoundLatin = "-?(?:[uú]|[aáă]|[ií]|[eé]e?)?h?-?"; const iSoundLatin = "-?(?:[uú]|[aáă]|[ií]|[eé]e?)?h?-?";
@ -180,67 +190,67 @@ export const latinReplacerInfo: IPhoneticsReplacerInfoItem[] = [
{ char: "óo", repl: ooSoundLatin }, { char: "óo", repl: ooSoundLatin },
{ char: "i", repl: iSoundLatin, replWhenBeginning: iSoundLatinBeginning }, { char: "i", repl: iSoundLatin, replWhenBeginning: iSoundLatinBeginning },
{ char: "í", repl: iSoundLatin, replWhenBeginning: iSoundLatinBeginning }, { char: "í", repl: iSoundLatin, replWhenBeginning: iSoundLatinBeginning },
{ char: "ey", repl: eySoundLatin }, { char: "ay", repl: aySoundLatin },
{ char: "éy", repl: eySoundLatin }, { char: "áy", repl: aySoundLatin },
{ char: "ee", repl: eySoundLatin }, { char: "ee", repl: aySoundLatin },
{ char: "ée", repl: eySoundLatin }, { char: "ée", repl: aySoundLatin },
{ char: "uy", repl: eySoundLatin }, { char: "uy", repl: aySoundLatin },
{ char: "úy", repl: eySoundLatin }, { char: "úy", repl: aySoundLatin },
{ char: "e", repl: middleESoundLatin }, { char: "e", repl: middleESoundLatin },
{ char: "é", repl: middleESoundLatin }, { char: "é", repl: middleESoundLatin },
{ char: "w", repl: "(?:w{1,2}?[UÚ]?|b)"}, { char: "w", repl: "(?:w{1,2}?[UÚ]?|b)" },
{ char: "y", repl: "[ií]?y?"}, { char: "y", repl: "[ií]?y?" },
{ char: "ts", repl: "(?:s{1,2}|z{1,2|ts|c)"}, { char: "ts", repl: "(?:s{1,2}|z{1,2|ts|c)" },
{ char: "s", repl: "(?:s{1,2}|z{1,2|ts|c)"}, { char: "s", repl: "(?:s{1,2}|z{1,2|ts|c)" },
{ char: "ss", repl: "(?:s{1,2}|z{1,2|ts|c)"}, { char: "ss", repl: "(?:s{1,2}|z{1,2|ts|c)" },
{ char: "c", repl: "(?:s{1,2}|z{1,2|ts|c)"}, { char: "c", repl: "(?:s{1,2}|z{1,2|ts|c)" },
{ char: "dz", repl: "(?:dz|z{1,2}|j)"}, { char: "dz", repl: "(?:dz|z{1,2}|j)" },
{ char: "z", repl: "(?:s{1,2}|dz|z{1,2}|ts)"}, { char: "z", repl: "(?:s{1,2}|dz|z{1,2}|ts)" },
{ char: "t", repl: "(?:t{1,2}|T|d{1,2}|D)"}, { char: "t", repl: "(?:t{1,2}|T|d{1,2}|D)" },
{ char: "tt", repl: "(?:t{1,2}|T|d{1,2}|D)"}, { char: "tt", repl: "(?:t{1,2}|T|d{1,2}|D)" },
{ char: "T", repl: "(?:t{1,2}|T|d{1,2}|D)"}, { char: "T", repl: "(?:t{1,2}|T|d{1,2}|D)" },
{ char: "d", repl: "(?:t{1,2}|T|d{1,2}|D)"}, { char: "d", repl: "(?:t{1,2}|T|d{1,2}|D)" },
{ char: "dd", repl: "(?:t{1,2}|T|d{1,2}|D)"}, { char: "dd", repl: "(?:t{1,2}|T|d{1,2}|D)" },
{ char: "D", repl: "(?:t{1,2}|T|d{1,2}|D)"}, { char: "D", repl: "(?:t{1,2}|T|d{1,2}|D)" },
{ char: "r", repl: "(?:R|r{1,2}|N)"}, { char: "r", repl: "(?:R|r{1,2}|N)" },
{ char: "rr", repl: "(?:R|r{1,2}|N)"}, { char: "rr", repl: "(?:R|r{1,2}|N)" },
{ char: "R", repl: "(?:R|r{1,2}|N)"}, { char: "R", repl: "(?:R|r{1,2}|N)" },
{ char: "nb", repl: "(?:nb|mb)"}, { char: "nb", repl: "(?:nb|mb)" },
{ char: "mb", repl: "(?:nb|mb)"}, { char: "mb", repl: "(?:nb|mb)" },
{ char: "n", repl: "(?:n{1,2}|N)"}, { char: "n", repl: "(?:n{1,2}|N)" },
{ char: "N", repl: "(?:R|r{1,2}|N)"}, { char: "N", repl: "(?:R|r{1,2}|N)" },
{ char: "f", repl: "(?:f{1,2}|p{1,2})"}, { char: "f", repl: "(?:f{1,2}|p{1,2})" },
{ char: "ff", repl: "(?:f{1,2}|p{1,2})"}, { char: "ff", repl: "(?:f{1,2}|p{1,2})" },
{ char: "b", repl: "(?:b{1,2}|p{1,2})"}, { char: "b", repl: "(?:b{1,2}|p{1,2})" },
{ char: "bb", repl: "(?:b{1,2}|p{1,2})"}, { char: "bb", repl: "(?:b{1,2}|p{1,2})" },
{ char: "p", repl: "(?:b{1,2}|p{1,2}|f{1,2})"}, { char: "p", repl: "(?:b{1,2}|p{1,2}|f{1,2})" },
{ char: "pp", repl: "(?:b{1,2}|p{1,2}|f{1,2})"}, { char: "pp", repl: "(?:b{1,2}|p{1,2}|f{1,2})" },
{ char: "sh", repl: "(?:x|sh|s`h)"}, { char: "sh", repl: "(?:x|sh|s`h)" },
{ char: "x", repl: "(?:kh|gh|x|h){1,2}"}, { char: "x", repl: "(?:kh|gh|x|h){1,2}" },
{ char: "kh", repl: "(?:kh|gh|x|h){1,2}"}, { char: "kh", repl: "(?:kh|gh|x|h){1,2}" },
{ char: "k", repl: "(?:k{1,2}|q{1,2})"}, { char: "k", repl: "(?:k{1,2}|q{1,2})" },
{ char: "q", repl: "(?:k{1,2}|q{1,2})"}, { char: "q", repl: "(?:k{1,2}|q{1,2})" },
{ char: "jz", repl: "(?:G|jz)"}, { char: "jz", repl: "(?:G|jz)" },
{ char: "G", repl: "(?:jz|G|g)"}, { char: "G", repl: "(?:jz|G|g)" },
{ char: "g", repl: "(?:gh?|k{1,2}|G)"}, { char: "g", repl: "(?:gh?|k{1,2}|G)" },
{ char: "gh", repl: "(?:g|gh|kh|G)"}, { char: "gh", repl: "(?:g|gh|kh|G)" },
{ char: "j", repl: "(?:j{1,2}|ch|dz)"}, { char: "j", repl: "(?:j{1,2}|ch|dz)" },
{ char: "ch", repl: "(?:j{1,2}|ch)"}, { char: "ch", repl: "(?:j{1,2}|ch)" },
{ char: "l", repl: "l{1,2}"}, { char: "l", repl: "l{1,2}" },
{ char: "ll", repl: "l{1,2}"}, { char: "ll", repl: "l{1,2}" },
{ char: "m", repl: "m{1,2}"}, { char: "m", repl: "m{1,2}" },
{ char: "mm", repl: "m{1,2}"}, { char: "mm", repl: "m{1,2}" },
{ char: "h", repl: "k?h?"}, { char: "h", repl: "k?h?" },
{ char: "'", repl: "['|’|`]?"}, { char: "'", repl: "['|’|`]?" },
{ char: "’", repl: "['|’|`]?"}, { char: "’", repl: "['|’|`]?" },
{ char: "`", repl: "['|’|`]?"}, { char: "`", repl: "['|’|`]?" },
]; ];
export const simpleLatinReplacerInfo: IPhoneticsReplacerInfoItem[] = [ export const simpleLatinReplacerInfo: IPhoneticsReplacerInfoItem[] = [
@ -254,65 +264,71 @@ export const simpleLatinReplacerInfo: IPhoneticsReplacerInfoItem[] = [
{ char: "U", repl: ooSoundSimpleLatin }, { char: "U", repl: ooSoundSimpleLatin },
{ char: "o", repl: ooSoundSimpleLatin }, { char: "o", repl: ooSoundSimpleLatin },
{ char: "oo", repl: ooSoundSimpleLatin }, { char: "oo", repl: ooSoundSimpleLatin },
{ char: "i", repl: iSoundSimpleLatin, replWhenBeginning: iSoundSimpleLatinBeginning }, {
{ char: "ey", repl: eySoundSimpleLatin }, char: "i",
{ char: "ee", repl: eySoundSimpleLatin }, repl: iSoundSimpleLatin,
{ char: "uy", repl: eySoundSimpleLatin }, replWhenBeginning: iSoundSimpleLatinBeginning,
},
{ char: "ay", repl: aySoundSimpleLatin },
{ char: "ee", repl: aySoundSimpleLatin },
{ char: "uy", repl: aySoundSimpleLatin },
{ char: "e", repl: middleESoundSimpleLatin }, { char: "e", repl: middleESoundSimpleLatin },
{ char: "w", repl: "(?:w{1,2}?[UÚ]?|b)"}, { char: "w", repl: "(?:w{1,2}?[UÚ]?|b)" },
{ char: "y", repl: "[ií]?y?"}, { char: "y", repl: "[ií]?y?" },
{ char: "ts", repl: "(?:s{1,2}|z{1,2|ts|c)"}, { char: "ts", repl: "(?:s{1,2}|z{1,2|ts|c)" },
{ char: "s", repl: "(?:s{1,2}|z{1,2|ts|c)"}, { char: "s", repl: "(?:s{1,2}|z{1,2|ts|c)" },
{ char: "c", repl: "(?:s{1,2}|z{1,2|ts|c)"}, { char: "c", repl: "(?:s{1,2}|z{1,2|ts|c)" },
{ char: "dz", repl: "(?:dz|z{1,2}|j)"}, { char: "dz", repl: "(?:dz|z{1,2}|j)" },
{ char: "z", repl: "(?:s{1,2}|dz|z{1,2}|ts)"}, { char: "z", repl: "(?:s{1,2}|dz|z{1,2}|ts)" },
{ char: "t", repl: "(?:t{1,2}|T|d{1,2}|D)"}, { char: "t", repl: "(?:t{1,2}|T|d{1,2}|D)" },
{ char: "tt", repl: "(?:t{1,2}|T|d{1,2}|D)"}, { char: "tt", repl: "(?:t{1,2}|T|d{1,2}|D)" },
{ char: "T", repl: "(?:t{1,2}|T|d{1,2}|D)"}, { char: "T", repl: "(?:t{1,2}|T|d{1,2}|D)" },
{ char: "d", repl: "(?:t{1,2}|T|d{1,2}|D)"}, { char: "d", repl: "(?:t{1,2}|T|d{1,2}|D)" },
{ char: "dd", repl: "(?:t{1,2}|T|d{1,2}|D)"}, { char: "dd", repl: "(?:t{1,2}|T|d{1,2}|D)" },
{ char: "D", repl: "(?:t{1,2}|T|d{1,2}|D)"}, { char: "D", repl: "(?:t{1,2}|T|d{1,2}|D)" },
{ char: "r", repl: "(?:R|r{1,2}|N)"}, { char: "r", repl: "(?:R|r{1,2}|N)" },
{ char: "rr", repl: "(?:R|r{1,2}|N)"}, { char: "rr", repl: "(?:R|r{1,2}|N)" },
{ char: "R", repl: "(?:R|r{1,2}|N)"}, { char: "R", repl: "(?:R|r{1,2}|N)" },
{ char: "nb", repl: "(?:nb|mb|nw)"}, { char: "nb", repl: "(?:nb|mb|nw)" },
{ char: "mb", repl: "(?:nb|mb)"}, { char: "mb", repl: "(?:nb|mb)" },
{ char: "n", repl: "(?:n{1,2}|N)"}, { char: "n", repl: "(?:n{1,2}|N)" },
{ char: "N", repl: "(?:R|r{1,2}|N)"}, { char: "N", repl: "(?:R|r{1,2}|N)" },
{ char: "f", repl: "(?:f{1,2}|p{1,2})"}, { char: "f", repl: "(?:f{1,2}|p{1,2})" },
{ char: "ff", repl: "(?:f{1,2}|p{1,2})"}, { char: "ff", repl: "(?:f{1,2}|p{1,2})" },
{ char: "b", repl: "(?:b{1,2}|p{1,2}|w)"}, { char: "b", repl: "(?:b{1,2}|p{1,2}|w)" },
{ char: "bb", repl: "(?:b{1,2}|p{1,2})"}, { char: "bb", repl: "(?:b{1,2}|p{1,2})" },
{ char: "p", repl: "(?:b{1,2}|p{1,2}|f{1,2})"}, { char: "p", repl: "(?:b{1,2}|p{1,2}|f{1,2})" },
{ char: "pp", repl: "(?:b{1,2}|p{1,2}|f{1,2})"}, { char: "pp", repl: "(?:b{1,2}|p{1,2}|f{1,2})" },
{ char: "sh", repl: "(?:x|sh|s`h)"}, { char: "sh", repl: "(?:x|sh|s`h)" },
{ char: "x", repl: "(?:kh|gh|x|h){1,2}"}, { char: "x", repl: "(?:kh|gh|x|h){1,2}" },
{ char: "kh", repl: "(?:kh|gh|x|h){1,2}"}, { char: "kh", repl: "(?:kh|gh|x|h){1,2}" },
{ char: "k", repl: "(?:k{1,2}|q{1,2})"}, { char: "k", repl: "(?:k{1,2}|q{1,2})" },
{ char: "kk", repl: "(?:k{1,2}|q{1,2})"}, { char: "kk", repl: "(?:k{1,2}|q{1,2})" },
{ char: "q", repl: "(?:k{1,2}|q{1,2})"}, { char: "q", repl: "(?:k{1,2}|q{1,2})" },
{ char: "qq", repl: "(?:k{1,2}|q{1,2})"}, { char: "qq", repl: "(?:k{1,2}|q{1,2})" },
{ char: "jz", repl: "(?:G|jz)"}, { char: "jz", repl: "(?:G|jz)" },
{ char: "G", repl: "(?:jz|G|g)"}, { char: "G", repl: "(?:jz|G|g)" },
{ char: "g", repl: "(?:gh?|k{1,2}|G)"}, { char: "g", repl: "(?:gh?|k{1,2}|G)" },
{ char: "gh", repl: "(?:g|gh|kh|G)"}, { char: "gh", repl: "(?:g|gh|kh|G)" },
{ char: "j", repl: "(?:j{1,2}|ch|dz)"}, { char: "j", repl: "(?:j{1,2}|ch|dz)" },
{ char: "ch", repl: "(?:j{1,2}|ch)"}, { char: "ch", repl: "(?:j{1,2}|ch)" },
{ char: "l", repl: "l{1,2}"}, { char: "l", repl: "l{1,2}" },
{ char: "ll", repl: "l{1,2}"}, { char: "ll", repl: "l{1,2}" },
{ char: "m", repl: "m{1,2}"}, { char: "m", repl: "m{1,2}" },
{ char: "mm", repl: "m{1,2}"}, { char: "mm", repl: "m{1,2}" },
{ char: "h", repl: "k?h?"}, { char: "h", repl: "k?h?" },
]; ];
// tslint:disable-next-line // tslint:disable-next-line
export const latinReplacerRegex = /yee|a{1,2}[i|y]|á{1,2}[i|y]|aa|áa|a|ắ|ă|á|U|Ú|u|ú|oo|óo|o|ó|e{1,2}|ée|é|ey|éy|uy|úy|i|í|w|y|q|q|ts|sh|ss|s|dz|z|tt|t|T|dd|d|D|r{1,2}|R|nb|mb|n{1,2}|N|f{1,2}|b{1,2}|p{1,2}|x|kh|q|kk|k|gh|g|G|j|ch|c|ll|l|m{1,2}|h|’|'|`/g; export const latinReplacerRegex =
/yee|a{1,2}[i|y]|á{1,2}[i|y]|aa|áa|a|ắ|ă|á|U|Ú|u|ú|oo|óo|o|ó|e{1,2}|ée|é|ay|áy|uy|úy|i|í|w|y|q|q|ts|sh|ss|s|dz|z|tt|t|T|dd|d|D|r{1,2}|R|nb|mb|n{1,2}|N|f{1,2}|b{1,2}|p{1,2}|x|kh|q|kk|k|gh|g|G|j|ch|c|ll|l|m{1,2}|h|’|'|`/g;
export const simpleLatinReplacerRegex = /yee|a{1,2}[i|y]|aa|a|U|u|oo|o|e{1,2}|ey|uy|i|w|y|q|ts|sh|s|dz|z|tt|t|T|dd|d|D|r{1,2}|R|nb|mb|n{1,2}|N|f{1,2}|b{1,2}|p{1,2}|x|kh|q|k|gh|g|G|j|ch|c|ll|l|m{1,2}|h/g; export const simpleLatinReplacerRegex =
/yee|a{1,2}[i|y]|aa|a|U|u|oo|o|e{1,2}|ay|uy|i|w|y|q|ts|sh|s|dz|z|tt|t|T|dd|d|D|r{1,2}|R|nb|mb|n{1,2}|N|f{1,2}|b{1,2}|p{1,2}|x|kh|q|k|gh|g|G|j|ch|c|ll|l|m{1,2}|h/g;

32
website/src/lib/local-storage.ts
View File

@ -14,7 +14,7 @@ export const userLocalStorageName = "user1";
export function saveOptions(options: Options): void { export function saveOptions(options: Options): void {
localStorage.setItem(optionsLocalStorageName, JSON.stringify(options)); localStorage.setItem(optionsLocalStorageName, JSON.stringify(options));
}; }
export const readOptions = (): undefined | Options => { export const readOptions = (): undefined | Options => {
const optionsRaw = localStorage.getItem(optionsLocalStorageName); const optionsRaw = localStorage.getItem(optionsLocalStorageName);
@ -23,10 +23,6 @@ export const readOptions = (): undefined | Options => {
} }
try { try {
const options = JSON.parse(optionsRaw) as Options; const options = JSON.parse(optionsRaw) as Options;
if (!("searchBarStickyFocus" in options)) {
// compatibility with legacy options
options.searchBarStickyFocus = false;
}
return options; return options;
} catch (e) { } catch (e) {
console.error("error parsing saved state JSON", e); console.error("error parsing saved state JSON", e);
@ -40,18 +36,18 @@ export function saveUser(user: AT.LingdocsUser | undefined): void {
} else { } else {
localStorage.removeItem(userLocalStorageName); localStorage.removeItem(userLocalStorageName);
} }
}; }
export const readUser = (): AT.LingdocsUser | undefined => { export const readUser = (): AT.LingdocsUser | undefined => {
const userRaw = localStorage.getItem(userLocalStorageName); const userRaw = localStorage.getItem(userLocalStorageName);
if (!userRaw) { if (!userRaw) {
return undefined; return undefined;
} }
try { try {
const user = JSON.parse(userRaw) as AT.LingdocsUser; const user = JSON.parse(userRaw) as AT.LingdocsUser;
return user; return user;
} catch (e) { } catch (e) {
console.error("error parsing saved user JSON", e); console.error("error parsing saved user JSON", e);
return undefined; return undefined;
} }
}; };

56
website/src/lib/wee-bit-fuzzy.test.ts
View File

@ -1,40 +1,40 @@
import { makeAWeeBitFuzzy } from "./wee-bit-fuzzy"; import { makeAWeeBitFuzzy } from "./wee-bit-fuzzy";
const pMatches = [ const pMatches = [
["پیټی", "پېټی"], ["پیټی", "پېټی"],
["دوستی", "دوستي"], ["دوستی", "دوستي"],
["پته", "پټه"], ["پته", "پټه"],
["تخلیه", "تحلیه"], ["تخلیه", "تحلیه"],
]; ];
const fMatches = [ const fMatches = [
["tahliya", "takhliya"], ["tahliya", "takhliya"],
["sareyy", "saRey"], ["sarey", "saRay"],
["peyTey", "peTey"], ["peyTey", "peTey"],
]; ];
pMatches.forEach((pair) => { pMatches.forEach((pair) => {
test(`${pair[0]} should match ${pair[1]}`, () => { test(`${pair[0]} should match ${pair[1]}`, () => {
const re = makeAWeeBitFuzzy(pair[0], "p"); const re = makeAWeeBitFuzzy(pair[0], "p");
const result = pair[1].match(new RegExp(re, "i")); const result = pair[1].match(new RegExp(re, "i"));
expect(result).toBeTruthy(); expect(result).toBeTruthy();
}); });
test(`${pair[1]} should match ${pair[0]}`, () => { test(`${pair[1]} should match ${pair[0]}`, () => {
const re = makeAWeeBitFuzzy(pair[1], "p"); const re = makeAWeeBitFuzzy(pair[1], "p");
const result = pair[0].match(new RegExp(re, "i")); const result = pair[0].match(new RegExp(re, "i"));
expect(result).toBeTruthy(); expect(result).toBeTruthy();
}); });
}); });
fMatches.forEach((pair) => { fMatches.forEach((pair) => {
test(`${pair[0]} should match ${pair[1]} both ways`, () => { test(`${pair[0]} should match ${pair[1]} both ways`, () => {
const re = makeAWeeBitFuzzy(pair[0], "f"); const re = makeAWeeBitFuzzy(pair[0], "f");
const result = pair[1].match(new RegExp(re, "i")); const result = pair[1].match(new RegExp(re, "i"));
expect(result).toBeTruthy(); expect(result).toBeTruthy();
}); });
test(`${pair[1]} should match ${pair[0]} both ways`, () => { test(`${pair[1]} should match ${pair[0]} both ways`, () => {
const re = makeAWeeBitFuzzy(pair[1], "f"); const re = makeAWeeBitFuzzy(pair[1], "f");
const result = pair[0].match(new RegExp(re, "i")); const result = pair[0].match(new RegExp(re, "i"));
expect(result).toBeTruthy(); expect(result).toBeTruthy();
}); });
}); });

161
website/src/lib/wee-bit-fuzzy.ts
View File

@ -28,7 +28,7 @@
// R: "[r|R]", // R: "[r|R]",
// }; // };
const fiveYeys = "[ئ|ۍ|ي|ې|ی]"; const fiveYays = "[ئ|ۍ|ي|ې|ی]";
const sSounds = "[س|ص|ث|څ]"; const sSounds = "[س|ص|ث|څ]";
const zSounds = "[ز|ژ|ض|ظ|ذ|ځ]"; const zSounds = "[ز|ژ|ض|ظ|ذ|ځ]";
const tSounds = "[ت|ط|ټ]"; const tSounds = "[ت|ط|ټ]";
@ -39,106 +39,115 @@ const hKhSounds = "[خ|ح|ښ|ه]";
const alef = "[آ|ا]"; const alef = "[آ|ا]";
const pReplacer = { const pReplacer = {
"ی": fiveYeys, ی: fiveYays,
"ي": fiveYeys, ي: fiveYays,
"ۍ": fiveYeys, ۍ: fiveYays,
"ئ": fiveYeys, ئ: fiveYays,
"ې": fiveYeys, ې: fiveYays,
"س": sSounds, س: sSounds,
"ص": sSounds, ص: sSounds,
"ث": sSounds, ث: sSounds,
"څ": sSounds, څ: sSounds,
"ز": zSounds, ز: zSounds,
"ظ": zSounds, ظ: zSounds,
"ذ": zSounds, ذ: zSounds,
"ض": zSounds, ض: zSounds,
"ژ": zSounds, ژ: zSounds,
"ځ": zSounds, ځ: zSounds,
"ت": tSounds, ت: tSounds,
"ط": tSounds, ط: tSounds,
"ټ": tSounds, ټ: tSounds,
"د": dSounds, د: dSounds,
"ډ": dSounds, ډ: dSounds,
"ر": rSounds, ر: rSounds,
"ړ": rSounds, ړ: rSounds,
"ن": nSounds, ن: nSounds,
"ڼ": nSounds, ڼ: nSounds,
"خ": hKhSounds, خ: hKhSounds,
"ح": hKhSounds, ح: hKhSounds,
"ښ": hKhSounds, ښ: hKhSounds,
"ه": hKhSounds, ه: hKhSounds,
"ا": alef, ا: alef,
"آ": alef, آ: alef,
}; };
const fiveYeysF = "(?:eyy|ey|ee|é|e|uy)"; const fiveYaysF = "(?:ey|ay|ee|é|e|uy)";
const hKhF = "(?:kh|h|x)"; const hKhF = "(?:kh|h|x)";
const zSoundsF = "(?:z|dz)"; const zSoundsF = "(?:z|dz)";
const sSoundsF = "(?:ts|s)"; const sSoundsF = "(?:ts|s)";
const fReplacer = { const fReplacer = {
"eyy": fiveYeysF, ey: fiveYaysF,
"ey": fiveYeysF, ay: fiveYaysF,
"uy": fiveYeysF, uy: fiveYaysF,
"ee": fiveYeysF, ee: fiveYaysF,
"e": fiveYeysF, e: fiveYaysF,
"z": zSoundsF, z: zSoundsF,
"dz": zSoundsF, dz: zSoundsF,
"x": hKhF, x: hKhF,
"h": hKhF, h: hKhF,
"kh": hKhF, kh: hKhF,
"ts": sSoundsF, ts: sSoundsF,
"s": sSoundsF, s: sSoundsF,
// only used if ignoring accents // only used if ignoring accents
"a": "[a|á]", a: "[a|á]",
"á": "[a|á|u|ú]", á: "[a|á|u|ú]",
"u": "[u|ú|a|á]", u: "[u|ú|a|á]",
"ú": "[u|ú]", ú: "[u|ú]",
"o": "[o|ó]", o: "[o|ó]",
"ó": "[o|ó]", ó: "[o|ó]",
"i": "[i|í]", i: "[i|í]",
"í": "[i|í]", í: "[i|í]",
"U": "[U|Ú]", U: "[U|Ú]",
"Ú": "[U|Ú]", Ú: "[U|Ú]",
"éy": fiveYeysF, áy: fiveYaysF,
"éyy": fiveYeysF, éy: fiveYaysF,
"úy": fiveYeysF, úy: fiveYaysF,
"ée": fiveYeysF, ée: fiveYaysF,
"é": fiveYeysF, é: fiveYaysF,
}; };
const pRepRegex = new RegExp(Object.keys(pReplacer).join("|"), "g"); const pRepRegex = new RegExp(Object.keys(pReplacer).join("|"), "g");
const fRepRegex = /eyy|ey|uy|ee|e|z|dz|x|kh|h|ts|s/g; const fRepRegex = /ey|ay|uy|ee|e|z|dz|x|kh|h|ts|s/g;
const fRepRegexWAccents = /eyy|éyy|ey|éy|uy|úy|ee|ée|e|é|z|dz|x|ts|s|kh|h|a|á|i|í|o|ó|u|ú|U|Ú/g; const fRepRegexWAccents =
/ey|éy|ay|áy|uy|úy|ee|ée|e|é|z|dz|x|ts|s|kh|h|a|á|i|í|o|ó|u|ú|U|Ú/g;
function makePAWeeBitFuzzy(s: string): string { function makePAWeeBitFuzzy(s: string): string {
// + s.replace(/ /g, "").split("").join(" *"); // + s.replace(/ /g, "").split("").join(" *");
return "^" + s.replace(pRepRegex, mtch => { return (
// @ts-ignore "^" +
return `${pReplacer[mtch]}`; s.replace(pRepRegex, (mtch) => {
}); // @ts-ignore
return `${pReplacer[mtch]}`;
})
);
} }
function makeFAWeeBitFuzzy(s: string, ignoreAccent?: boolean): string { function makeFAWeeBitFuzzy(s: string, ignoreAccent?: boolean): string {
return "^" + s.replace((ignoreAccent ? fRepRegexWAccents : fRepRegex), mtch => { return (
// @ts-ignore "^" +
return fReplacer[mtch]; s.replace(ignoreAccent ? fRepRegexWAccents : fRepRegex, (mtch) => {
}); // @ts-ignore
return fReplacer[mtch];
})
);
} }
export function makeAWeeBitFuzzy(s: string, i: "f" | "p", ignoreAccent?: boolean): string { export function makeAWeeBitFuzzy(
return i === "p" s: string,
? makePAWeeBitFuzzy(s) i: "f" | "p",
: makeFAWeeBitFuzzy(s, ignoreAccent); ignoreAccent?: boolean
} ): string {
return i === "p" ? makePAWeeBitFuzzy(s) : makeFAWeeBitFuzzy(s, ignoreAccent);
}

302
website/src/types/dictionary-types.ts
View File

@ -1,71 +1,88 @@
export type DictionaryStatus = "loading" | "ready" | "updating" | "error loading"; export type DictionaryStatus =
| "loading"
| "ready"
| "updating"
| "error loading";
export type State = { export type State = {
dictionaryStatus: DictionaryStatus, dictionaryStatus: DictionaryStatus;
searchValue: string, showModal: boolean;
options: Options, searchValue: string;
page: number, options: Options;
isolatedEntry: import("@lingdocs/ps-react").Types.DictionaryEntry | undefined, page: number;
results: import("@lingdocs/ps-react").Types.DictionaryEntry[], isolatedEntry: import("@lingdocs/ps-react").Types.DictionaryEntry | undefined;
wordlist: WordlistWord[], results: import("@lingdocs/ps-react").Types.DictionaryEntry[];
reviewTasks: import("./functions-types").ReviewTask[], wordlist: WordlistWord[];
dictionaryInfo: import("@lingdocs/ps-react").Types.DictionaryInfo | undefined, reviewTasks: import("./functions-types").ReviewTask[];
user: undefined | import("./account-types").LingdocsUser, dictionaryInfo: import("@lingdocs/ps-react").Types.DictionaryInfo | undefined;
inflectionSearchResults: undefined | "searching" | { user: undefined | import("./account-types").LingdocsUser;
exact: InflectionSearchResult[], inflectionSearchResults:
fuzzy: InflectionSearchResult[], | undefined
}, | "searching"
} | {
exact: InflectionSearchResult[];
fuzzy: InflectionSearchResult[];
};
};
export type DictionaryAPI = { export type DictionaryAPI = {
initialize: () => Promise<{ initialize: () => Promise<{
response: "loaded first time" | "loaded from saved", response: "loaded first time" | "loaded from saved";
dictionaryInfo: import("@lingdocs/ps-react").Types.DictionaryInfo, dictionaryInfo: import("@lingdocs/ps-react").Types.DictionaryInfo;
}>, }>;
update: (updateComing: () => void) => Promise<{ update: (updateComing: () => void) => Promise<{
response: "no need for update" | "updated" | "unable to check", response: "no need for update" | "updated" | "unable to check";
dictionaryInfo: import("@lingdocs/ps-react").Types.DictionaryInfo, dictionaryInfo: import("@lingdocs/ps-react").Types.DictionaryInfo;
}>, }>;
search: (state: State) => import("@lingdocs/ps-react").Types.DictionaryEntry[], search: (
exactPashtoSearch: (search: string) => import("@lingdocs/ps-react").Types.DictionaryEntry[], state: State
getNewWordsThisMonth: () => import("@lingdocs/ps-react").Types.DictionaryEntry[], ) => import("@lingdocs/ps-react").Types.DictionaryEntry[];
findOneByTs: (ts: number) => import("@lingdocs/ps-react").Types.DictionaryEntry | undefined, exactPashtoSearch: (
findRelatedEntries: (entry: import("@lingdocs/ps-react").Types.DictionaryEntry) => import("@lingdocs/ps-react").Types.DictionaryEntry[], search: string
} ) => import("@lingdocs/ps-react").Types.DictionaryEntry[];
getNewWordsThisMonth: () => import("@lingdocs/ps-react").Types.DictionaryEntry[];
findOneByTs: (
ts: number
) => import("@lingdocs/ps-react").Types.DictionaryEntry | undefined;
findRelatedEntries: (
entry: import("@lingdocs/ps-react").Types.DictionaryEntry
) => import("@lingdocs/ps-react").Types.DictionaryEntry[];
};
export type WordlistWordBase = { export type WordlistWordBase = {
_id: string, _id: string;
/* a backup copy of the full dictionary entry in case it gets deleted from the dictionary */ /* a backup copy of the full dictionary entry in case it gets deleted from the dictionary */
entry: import("@lingdocs/ps-react").Types.DictionaryEntry, entry: import("@lingdocs/ps-react").Types.DictionaryEntry;
/* the notes/context provided by the user for the word in their wordlist */ /* the notes/context provided by the user for the word in their wordlist */
notes: string, notes: string;
supermemo: import("supermemo").SuperMemoItem, supermemo: import("supermemo").SuperMemoItem;
/* rep/stage of warmup stage before moving into supermemo mode */ /* rep/stage of warmup stage before moving into supermemo mode */
warmup: number | "done", warmup: number | "done";
/* date due for review - ISO string */ /* date due for review - ISO string */
dueDate: number, dueDate: number;
} };
export type WordlistAttachmentInfo = { export type WordlistAttachmentInfo = {
imgSize?: { height: number, width: number }, imgSize?: { height: number; width: number };
_attachments: Attachments, _attachments: Attachments;
} };
export type WordlistWordWAttachments = WordlistWordBase & WordlistAttachmentInfo; export type WordlistWordWAttachments = WordlistWordBase &
WordlistAttachmentInfo;
export type WordlistWord = WordlistWordBase | WordlistWordWAttachments; export type WordlistWord = WordlistWordBase | WordlistWordWAttachments;
export type Options = { export type Options = {
language: Language, language: Language;
searchType: SearchType, searchType: SearchType;
theme: Theme, theme: Theme;
textOptionsRecord: TextOptionsRecord, textOptionsRecord: TextOptionsRecord;
wordlistMode: WordlistMode, wordlistMode: WordlistMode;
wordlistReviewLanguage: Language, wordlistReviewLanguage: Language;
wordlistReviewBadge: boolean, wordlistReviewBadge: boolean;
searchBarPosition: SearchBarPosition, searchBarPosition: SearchBarPosition;
searchBarStickyFocus: boolean, searchBarStickyFocus: boolean;
} };
export type Language = "Pashto" | "English"; export type Language = "Pashto" | "English";
export type SearchType = "alphabetical" | "fuzzy"; export type SearchType = "alphabetical" | "fuzzy";
@ -78,84 +95,102 @@ export type SearchBarPosition = "top" | "bottom";
export type WordlistMode = "browse" | "review"; export type WordlistMode = "browse" | "review";
export type TextOptionsRecord = { export type TextOptionsRecord = {
lastModified: import("./account-types").TimeStamp, lastModified: import("./account-types").TimeStamp;
textOptions: import("@lingdocs/ps-react").Types.TextOptions, textOptions: import("@lingdocs/ps-react").Types.TextOptions;
}; };
export type UserLevel = "basic" | "student" | "editor"; export type UserLevel = "basic" | "student" | "editor";
export type OptionsAction = { export type OptionsAction =
type: "toggleSearchType", | {
} | { type: "toggleSearchType";
type: "toggleLanguage", }
} | { | {
type: "changeTheme", type: "toggleLanguage";
payload: Theme, }
} | { | {
type: "changeSearchBarPosition", type: "changeTheme";
payload: SearchBarPosition, payload: Theme;
} | { }
type: "changeWordlistMode", | {
payload: WordlistMode, type: "changeSearchBarPosition";
} | { payload: SearchBarPosition;
type: "changeWordlistReviewLanguage", }
payload: Language, | {
} | { type: "changeWordlistMode";
type: "changeWordlistReviewBadge", payload: WordlistMode;
payload: boolean, }
} | { | {
type: "updateTextOptionsRecord", type: "changeWordlistReviewLanguage";
payload: TextOptionsRecord, payload: Language;
} | { }
type: "changeSearchBarStickyFocus", | {
payload: boolean, type: "changeWordlistReviewBadge";
} | { payload: boolean;
type: "setShowPlayStoreButton", }
payload: boolean, | {
}; type: "updateTextOptionsRecord";
payload: TextOptionsRecord;
}
| {
type: "changeSearchBarStickyFocus";
payload: boolean;
}
| {
type: "setShowPlayStoreButton";
payload: boolean;
};
export type TextOptionsAction = { export type TextOptionsAction =
type: "changePTextSize", | {
payload: PTextSize, type: "changePTextSize";
} | { payload: PTextSize;
type: "changeSpelling", }
payload: import("@lingdocs/ps-react").Types.Spelling, | {
} | { type: "changeSpelling";
type: "changePhonetics", payload: import("@lingdocs/ps-react").Types.Spelling;
payload: "lingdocs" | "ipa" | "alalc" | "none", }
} | { | {
type: "changeDialect", type: "changePhonetics";
payload: "standard" | "peshawer" | "southern", payload: "lingdocs" | "ipa" | "alalc" | "none";
} | { }
type: "changeDiacritics", | {
payload: boolean, type: "changeDialect";
}; payload: "standard" | "peshawer" | "southern";
}
| {
type: "changeDiacritics";
payload: boolean;
};
export type AttachmentToPut = { export type AttachmentToPut = {
content_type: string, content_type: string;
data: string | Blob, data: string | Blob;
}
export type AttachmentWithData = {
content_type: string,
digest: string,
data: string | Blob,
}
export type AttachmentWOutData = {
content_type: string,
digest: string,
stub: true;
}
export type Attachment = AttachmentToPut | AttachmentWithData | AttachmentWOutData
export type AttachmentType = "image" | "audio";
export type Attachments = {
/* only allows one image and one audio attachment - max 2 values */
[filename: string]: Attachment,
}; };
export type WordlistWordDoc = WordlistWord & { _rev: string, _id: string }; export type AttachmentWithData = {
content_type: string;
digest: string;
data: string | Blob;
};
export type AttachmentWOutData = {
content_type: string;
digest: string;
stub: true;
};
export type Attachment =
| AttachmentToPut
| AttachmentWithData
| AttachmentWOutData;
export type AttachmentType = "image" | "audio";
export type Attachments = {
/* only allows one image and one audio attachment - max 2 values */
[filename: string]: Attachment;
};
export type WordlistWordDoc = WordlistWord & { _rev: string; _id: string };
export type InflectionName = "plain" | "1st" | "2nd"; export type InflectionName = "plain" | "1st" | "2nd";
@ -167,15 +202,14 @@ export type PluralInflectionName = "plural" | "2nd";
// the possible matches, and their person/inflection number // the possible matches, and their person/inflection number
export type InflectionSearchResult = { export type InflectionSearchResult = {
entry: import("@lingdocs/ps-react").Types.DictionaryEntry, entry: import("@lingdocs/ps-react").Types.DictionaryEntry;
forms: InflectionFormMatch[], forms: InflectionFormMatch[];
}
export type InflectionFormMatch = {
path: string[],
matches: {
ps: import("@lingdocs/ps-react").Types.PsString,
pos: InflectionName[] | import("@lingdocs/ps-react").Types.Person[] | null,
}[],
}; };
export type InflectionFormMatch = {
path: string[];
matches: {
ps: import("@lingdocs/ps-react").Types.PsString;
pos: InflectionName[] | import("@lingdocs/ps-react").Types.Person[] | null;
}[];
};

8
website/yarn.lock
View File

@ -2349,10 +2349,10 @@
"@jridgewell/resolve-uri" "^3.0.3" "@jridgewell/resolve-uri" "^3.0.3"
"@jridgewell/sourcemap-codec" "^1.4.10" "@jridgewell/sourcemap-codec" "^1.4.10"
"@lingdocs/ps-react@5.10.1": "@lingdocs/ps-react@6.0.0":
version "5.10.1" version "6.0.0"
resolved "https://npm.lingdocs.com/@lingdocs%2fps-react/-/ps-react-5.10.1.tgz#949850aaa3c9de54d4beed1daa9b546bb0a84df9" resolved "https://npm.lingdocs.com/@lingdocs%2fps-react/-/ps-react-6.0.0.tgz#dbdfd1a5afd19253679169eacbf1da5562db5dc3"
integrity sha512-Ro/6Fq9mEdF4/2wJf8USkIlYe+9vWmez/RhoUF0mTjOhmyTGV6cpajK0Qpo1WyCaL5d/6BTI3qVuk5h8pWRQjA== integrity sha512-+j6F65FtmPbeEjjHtE3JqKHtCcUM+cMAN2RMTd8yyacJ4sTJW/oWC+6rAQGQqc1da3lP7tuxt6p+esmFYI9fgQ==
dependencies: dependencies:
"@formkit/auto-animate" "^1.0.0-beta.3" "@formkit/auto-animate" "^1.0.0-beta.3"
classnames "^2.2.6" classnames "^2.2.6"