pashto-grammar/scripts/get-words.js

86 lines
3.0 KiB
JavaScript
Raw Normal View History

2021-03-06 10:40:31 +00:00
const fs = require("fs");
const fetch = require("node-fetch");
2021-03-16 10:02:53 +00:00
const { readDictionary } = require("@lingdocs/pashto-inflector");
2021-03-06 10:40:31 +00:00
const path = require("path");
2021-10-18 01:12:47 +00:00
const wordsPath = path.join(".", "src", "words");
const wordsFile = "raw-words.ts";
2021-10-18 01:12:47 +00:00
const verbCollectionPath = path.join(wordsPath, "verb-categories");
const nounAdjCollectionPath = path.join(wordsPath, "noun-adj-categories");
const adverbCollectionPath = path.join(wordsPath, "adverbs");
2021-03-06 10:40:31 +00:00
const verbTsFiles = fs.readdirSync(verbCollectionPath);
const nounAdjTsFiles = fs.readdirSync(nounAdjCollectionPath);
const adverbTsFiles = fs.readdirSync(adverbCollectionPath);
2021-03-06 10:40:31 +00:00
2021-10-19 05:17:27 +00:00
const allVerbTsS = verbTsFiles.flatMap(fileName => [
...require(path.join("..", verbCollectionPath, fileName)).map(x => x.ts)
]).filter((v, i, a) => a.findIndex(x => x === v) === i);
2021-03-06 10:40:31 +00:00
2021-10-19 05:17:27 +00:00
const allNounAdjTsS = nounAdjTsFiles.flatMap(fileName => [
...require(path.join("..", nounAdjCollectionPath, fileName)).map(x => x.ts)
]).filter((v, i, a) => a.findIndex(x => x === v) === i);
2021-03-06 10:40:31 +00:00
const allAdverbTsS = adverbTsFiles.flatMap(fileName => [
...require(path.join("..", adverbCollectionPath, fileName)).map(x => x.ts)
]).filter((v, i, a) => a.findIndex(x => x === v) === i);
2021-03-06 10:40:31 +00:00
console.log("getting words from dictionary...");
2021-03-16 10:02:53 +00:00
fetch(process.env.LINGDOCS_DICTIONARY_URL).then(res => res.arrayBuffer()).then(buffer => {
const dictionary = readDictionary(buffer);
const entries = dictionary.entries;
2021-03-06 10:40:31 +00:00
// MAKE VERBS FILE
2021-10-18 01:12:47 +00:00
const allWords = [
...getVerbsFromTsS(entries),
...getNounsAdjsFromTsS(entries),
];
const content = `
// @ts-ignore
const words: Word[] = ${JSON.stringify(allWords)};
export default words;`;
fs.writeFileSync(path.join(wordsPath, wordsFile), content);
2021-03-06 10:40:31 +00:00
});
function getVerbsFromTsS(entries) {
2021-07-08 14:11:06 +00:00
const missingEc = [];
2021-10-19 05:17:27 +00:00
const toReturn = allVerbTsS.map(ts => {
const entry = entries.find(x => ts === x.ts);
2021-03-06 10:40:31 +00:00
if (!entry) {
console.log("couldn't find ts", ts);
return undefined;
}
2021-07-08 14:11:06 +00:00
if (!entry.ec) {
missingEc.push(entry.ts);
}
2021-03-06 10:40:31 +00:00
if (entry.c && entry.c.includes("comp.")) {
const complement = entries.find(x => entry.l === x.ts);
return {
entry,
complement,
};
}
2021-10-18 01:12:47 +00:00
return { entry };
2021-03-06 10:40:31 +00:00
}).filter(x => x);
2021-07-08 14:11:06 +00:00
if (missingEc.length !== 0) {
console.log("Verbs missing ec:", missingEc);
}
return toReturn;
2021-03-06 10:40:31 +00:00
}
function getNounsAdjsFromTsS(entries) {
const b = [...allNounAdjTsS, ...allAdverbTsS].map(ts => {
2021-10-19 05:17:27 +00:00
const entry = entries.find(x => ts === x.ts);
2021-03-06 10:40:31 +00:00
if (!entry) {
2021-10-19 05:17:27 +00:00
console.log("couldn't find ts", ts);
2021-03-06 10:40:31 +00:00
return undefined;
}
2021-10-10 01:45:42 +00:00
// const firstWord = entry.e.split(",")[0].split(";")[0].split("(")[0].trim();
// console.log(firstWord, entry.f, entry.ts);
2021-10-05 01:15:31 +00:00
// if (firstWord.contains(" ")) console.log("SPACE PRESENT");
2021-10-18 01:12:47 +00:00
return entry;
2021-03-06 10:40:31 +00:00
}).filter(x => x);
2021-10-05 01:15:31 +00:00
return b;
// console.log(b.length, "number of nouns/adjs");
2021-03-06 10:40:31 +00:00
}