pashto-dictionary/functions/lib/publishing-helpers.ts

173 lines
5.0 KiB
TypeScript
Raw Normal View History

2024-11-27 17:27:59 +00:00
import { Types as T, validateEntry } from "@lingdocs/inflect";
const title = "LingDocs Pashto Dictionary";
const license = `Copyright © ${new Date().getFullYear()} lingdocs.com All Rights Reserved - Licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License - https://creativecommons.org/licenses/by-nc-sa/4.0/`;
const baseUrl = `https://storage.lingdocs.com/dictionary/`;
2024-11-27 20:10:31 +00:00
export const dictionaryFilename = "dictionary";
export const dictionaryInfoFilename = "dictionary-info";
2024-11-27 17:27:59 +00:00
// const hunspellAffFileFilename = "ps_AFF.aff";
// const hunspellDicFileFilename = "ps_AFF.dic";
2024-11-28 10:34:37 +00:00
// const allWordsJsonFilename = "all-words-dictionary.json";
2024-11-27 17:27:59 +00:00
const url = `${baseUrl}${dictionaryFilename}`;
const infoUrl = `${baseUrl}${dictionaryInfoFilename}`;
export function makeDictionaryObject(
entries: T.DictionaryEntry[]
): T.Dictionary {
return {
info: {
2024-11-28 10:34:37 +00:00
// @ts-ignore
testVal: true,
2024-11-27 17:27:59 +00:00
title,
license,
url,
infoUrl,
release: new Date().getTime(),
numberOfEntries: entries.length,
},
entries,
};
}
export function checkForErrors(
entries: T.DictionaryEntry[]
): (T.DictionaryEntryError | { duplicates: number[] })[] {
// check for duplicates
const tsMap: Record<number, T.DictionaryEntry> = {};
const duplicates: number[] = [];
// making a map here based on ts with the entry will speed up the
// compliment checking process!!
for (var i = 0; i < entries.length; i++) {
const ts = entries[i].ts;
if (ts in tsMap) {
duplicates.push(ts);
} else {
tsMap[ts] = entries[i];
}
}
if (duplicates.length) {
return [{ duplicates }];
}
// check for errors
return entries.reduce(
(errors: T.DictionaryEntryError[], entry: T.DictionaryEntry) => {
const response = validateEntry(entry);
if ("errors" in response && response.errors.length) {
return [...errors, response];
}
if ("checkComplement" in response) {
if (!entry.l) {
const error: T.DictionaryEntryError = {
errors: ["complement needed"],
ts: entry.ts,
p: entry.p,
f: entry.f,
e: entry.e,
erroneousFields: ["l"],
};
return [...errors, error];
}
const complement = tsMap[entry.l];
if (!complement) {
const error: T.DictionaryEntryError = {
errors: ["complement link not found in dictionary"],
ts: entry.ts,
p: entry.p,
f: entry.f,
e: entry.e,
erroneousFields: ["l"],
};
return [...errors, error];
}
if (
!complement.c?.includes("n.") &&
!complement.c?.includes("adj.") &&
!complement.c?.includes("adv.")
) {
const error: T.DictionaryEntryError = {
errors: ["complement link to invalid complement"],
ts: entry.ts,
p: entry.p,
f: entry.f,
e: entry.e,
erroneousFields: ["l"],
};
return [...errors, error];
}
}
return errors;
},
[]
);
}
export function makeSitemap(dictionary: T.Dictionary): string {
function tsToDate(ts: number): string {
if (ts < 10000000000) {
// approximate date for old-style timestamps
return "2021-01-01";
}
return getDateString(new Date(ts));
}
function getDateString(d: Date): string {
return d.toISOString().split("T")[0];
}
const pages = [
"",
"about",
"settings",
"account",
"phrase-builder",
"new-entries",
];
const currentDate = getDateString(new Date());
return `<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
${pages
.map(
(page) =>
`
<url>
<loc>https://dictionary.lingdocs.com/${page}</loc>
<lastmod>${currentDate}</lastmod>
</url>`
)
.join("")}
${dictionary.entries
.map(
(entry) =>
`
<url>
<loc>https://dictionary.lingdocs.com/word?id=${entry.ts}</loc>
<lastmod>${tsToDate(entry.ts)}</lastmod>
</url>`
)
.join("")}
</urlset>
`;
}
// FOR HUNSPELL
// const hunspellAffFileFilename = "ps_AFF.aff";
// const hunspellDicFileFilename = "ps_AFF.dic";
// async function doHunspellEtc(
// info: T.DictionaryInfo,
// entries: T.DictionaryEntry[]
// ) {
// const wordlistResponse = getWordList(entries);
// if (!wordlistResponse.ok) {
// throw new Error(JSON.stringify(wordlistResponse.errors));
// }
// // const hunspell = makeHunspell(wordlistResponse.wordlist);
// // await uploadHunspellToStorage(hunspell);
// await uploadAllWordsToStoarage(info, wordlistResponse.wordlist);
// }
// function makeHunspell(wordlist: string[]) {
// return {
// dicContent: wordlist.reduce((acc, word) => acc + word + "\n", wordlist.length + "\n"),
// affContent: "SET UTF-8\nCOMPLEXPREFIXES\nIGNORE ۱۲۳۴۵۶۷۸۹۰-=ًٌٍَُِّْ؛:؟.،,،؟\n",
// };
// }