From 952bca1f807d89700008d3055546e492faabba4c Mon Sep 17 00:00:00 2001 From: Bill D Date: Wed, 10 Mar 2021 15:37:35 +0400 Subject: [PATCH] added protobuf heplers --- package.json | 6 +- src/lib/dictionary-models.js | 118 +++++++++++++++++++++++++++++++++++ src/lib/dictionary.proto | 50 +++++++++++++++ src/lib/protobuf.test.ts | 28 +++++++++ src/lib/protobuf.ts | 30 +++++++++ src/library.ts | 11 ++++ yarn.lock | 27 +++++++- 7 files changed, 267 insertions(+), 3 deletions(-) create mode 100644 src/lib/dictionary-models.js create mode 100644 src/lib/dictionary.proto create mode 100644 src/lib/protobuf.test.ts create mode 100644 src/lib/protobuf.ts diff --git a/package.json b/package.json index d282754..674b92c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@lingdocs/pashto-inflector", - "version": "0.1.7", + "version": "0.1.8", "author": "lingdocs.com", "description": "A Pashto inflection and verb conjugation engine, inculding React components for displaying Pashto text, inflections, and conjugations", "homepage": "https://verbs.lingdocs.com", @@ -23,7 +23,8 @@ "registry": "https://reg.lingdocs.com" }, "dependencies": { - "classnames": "^2.2.6" + "classnames": "^2.2.6", + "pbf": "^3.2.1" }, "devDependencies": { "@fortawesome/fontawesome-free": "^5.15.2", @@ -35,6 +36,7 @@ "@testing-library/user-event": "^12.1.10", "@types/jest": "^26.0.20", "@types/node": "^14.14.32", + "@types/pbf": "^3.0.2", "@types/react": "^17.0.3", "@types/react-dom": "^17.0.2", "bootstrap": "^4.6.0", diff --git a/src/lib/dictionary-models.js b/src/lib/dictionary-models.js new file mode 100644 index 0000000..870a3ed --- /dev/null +++ b/src/lib/dictionary-models.js @@ -0,0 +1,118 @@ +// code generated by pbf v3.2.1 - modified below on line 32 + +// DictionaryInfo ======================================== + +var DictionaryInfo = exports.DictionaryInfo = {}; + +DictionaryInfo.read = function (pbf, end) { + return pbf.readFields(DictionaryInfo._readField, {title: "", license: "", release: 0, numberOfEntries: 0, url: "", infoUrl: ""}, end); +}; +DictionaryInfo._readField = function (tag, obj, pbf) { + if (tag === 1) obj.title = pbf.readString(); + else if (tag === 2) obj.license = pbf.readString(); + else if (tag === 3) obj.release = pbf.readVarint(true); + else if (tag === 4) obj.numberOfEntries = pbf.readVarint(true); + else if (tag === 5) obj.url = pbf.readString(); + else if (tag === 6) obj.infoUrl = pbf.readString(); +}; +DictionaryInfo.write = function (obj, pbf) { + if (obj.title) pbf.writeStringField(1, obj.title); + if (obj.license) pbf.writeStringField(2, obj.license); + if (obj.release) pbf.writeVarintField(3, obj.release); + if (obj.numberOfEntries) pbf.writeVarintField(4, obj.numberOfEntries); + if (obj.url) pbf.writeStringField(5, obj.url); + if (obj.infoUrl) pbf.writeStringField(6, obj.infoUrl); +}; + +// Entry ======================================== + +var Entry = exports.Entry = {}; + +Entry.read = function (pbf, end) { + // modified so that objects do not include non-required fields unless added with Entry._readField below + return pbf.readFields(Entry._readField, {ts: 0, i: 0, p: "", f: "", e: ""}, end); +}; +Entry._readField = function (tag, obj, pbf) { + if (tag === 1) obj.ts = pbf.readVarint(true); + else if (tag === 2) obj.i = pbf.readVarint(true); + else if (tag === 3) obj.p = pbf.readString(); + else if (tag === 4) obj.f = pbf.readString(); + else if (tag === 5) obj.e = pbf.readString(); + else if (tag === 6) obj.c = pbf.readString(); + else if (tag === 7) obj.l = pbf.readString(); + else if (tag === 8) obj.infap = pbf.readString(); + else if (tag === 9) obj.infaf = pbf.readString(); + else if (tag === 11) obj.infbp = pbf.readString(); + else if (tag === 12) obj.infbf = pbf.readString(); + else if (tag === 13) obj.noInf = pbf.readString(); + else if (tag === 14) obj.app = pbf.readString(); + else if (tag === 15) obj.apf = pbf.readString(); + else if (tag === 16) obj.ppp = pbf.readString(); + else if (tag === 17) obj.ppf = pbf.readString(); + else if (tag === 18) obj.psp = pbf.readString(); + else if (tag === 19) obj.psf = pbf.readString(); + else if (tag === 20) obj.ssp = pbf.readString(); + else if (tag === 21) obj.ssf = pbf.readString(); + else if (tag === 22) obj.prp = pbf.readString(); + else if (tag === 23) obj.prf = pbf.readString(); + else if (tag === 24) obj.pprtp = pbf.readString(); + else if (tag === 25) obj.pprtf = pbf.readString(); + else if (tag === 26) obj.tppp = pbf.readString(); + else if (tag === 27) obj.tppf = pbf.readString(); + else if (tag === 28) obj.shortIntrans = pbf.readBoolean(); + else if (tag === 29) obj.noOo = pbf.readBoolean(); + else if (tag === 30) obj.sepOo = pbf.readBoolean(); + else if (tag === 31) obj.separationAtP = pbf.readVarint(true); + else if (tag === 32) obj.separationAtF = pbf.readVarint(true); + else if (tag === 33) obj.diacExcept = pbf.readBoolean(); +}; +Entry.write = function (obj, pbf) { + if (obj.ts) pbf.writeVarintField(1, obj.ts); + if (obj.i) pbf.writeVarintField(2, obj.i); + if (obj.p) pbf.writeStringField(3, obj.p); + if (obj.f) pbf.writeStringField(4, obj.f); + if (obj.e) pbf.writeStringField(5, obj.e); + if (obj.c) pbf.writeStringField(6, obj.c); + if (obj.l) pbf.writeStringField(7, obj.l); + if (obj.infap) pbf.writeStringField(8, obj.infap); + if (obj.infaf) pbf.writeStringField(9, obj.infaf); + if (obj.infbp) pbf.writeStringField(11, obj.infbp); + if (obj.infbf) pbf.writeStringField(12, obj.infbf); + if (obj.noInf) pbf.writeStringField(13, obj.noInf); + if (obj.app) pbf.writeStringField(14, obj.app); + if (obj.apf) pbf.writeStringField(15, obj.apf); + if (obj.ppp) pbf.writeStringField(16, obj.ppp); + if (obj.ppf) pbf.writeStringField(17, obj.ppf); + if (obj.psp) pbf.writeStringField(18, obj.psp); + if (obj.psf) pbf.writeStringField(19, obj.psf); + if (obj.ssp) pbf.writeStringField(20, obj.ssp); + if (obj.ssf) pbf.writeStringField(21, obj.ssf); + if (obj.prp) pbf.writeStringField(22, obj.prp); + if (obj.prf) pbf.writeStringField(23, obj.prf); + if (obj.pprtp) pbf.writeStringField(24, obj.pprtp); + if (obj.pprtf) pbf.writeStringField(25, obj.pprtf); + if (obj.tppp) pbf.writeStringField(26, obj.tppp); + if (obj.tppf) pbf.writeStringField(27, obj.tppf); + if (obj.shortIntrans) pbf.writeBooleanField(28, obj.shortIntrans); + if (obj.noOo) pbf.writeBooleanField(29, obj.noOo); + if (obj.sepOo) pbf.writeBooleanField(30, obj.sepOo); + if (obj.separationAtP) pbf.writeVarintField(31, obj.separationAtP); + if (obj.separationAtF) pbf.writeVarintField(32, obj.separationAtF); + if (obj.diacExcept) pbf.writeBooleanField(33, obj.diacExcept); +}; + +// Dictionary ======================================== + +var Dictionary = exports.Dictionary = {}; + +Dictionary.read = function (pbf, end) { + return pbf.readFields(Dictionary._readField, {info: null, entries: []}, end); +}; +Dictionary._readField = function (tag, obj, pbf) { + if (tag === 1) obj.info = DictionaryInfo.read(pbf, pbf.readVarint() + pbf.pos); + else if (tag === 2) obj.entries.push(Entry.read(pbf, pbf.readVarint() + pbf.pos)); +}; +Dictionary.write = function (obj, pbf) { + if (obj.info) pbf.writeMessage(1, DictionaryInfo.write, obj.info); + if (obj.entries) for (var i = 0; i < obj.entries.length; i++) pbf.writeMessage(2, Entry.write, obj.entries[i]); +}; diff --git a/src/lib/dictionary.proto b/src/lib/dictionary.proto new file mode 100644 index 0000000..589ac58 --- /dev/null +++ b/src/lib/dictionary.proto @@ -0,0 +1,50 @@ +syntax = "proto3"; + +message DictionaryInfo { + string title = 1; + string license = 2; + int32 release = 3; + int32 numberOfEntries = 4; + string url = 5; + string infoUrl = 6; +} + +message Entry { + int32 ts = 1; + int32 i = 2; + string p = 3; + string f = 4; + string e = 5; + string c = 6; + string l = 7; + string infap = 8; + string infaf = 9; + string infbp = 11; + string infbf = 12; + string noInf = 13; + string app = 14; + string apf = 15; + string ppp = 16; + string ppf = 17; + string psp = 18; + string psf = 19; + string ssp = 20; + string ssf = 21; + string prp = 22; + string prf = 23; + string pprtp = 24; + string pprtf = 25; + string tppp = 26; + string tppf = 27; + bool shortIntrans = 28; + bool noOo = 29; + bool sepOo = 30; + int32 separationAtP = 31; + int32 separationAtF = 32; + bool diacExcept = 33; +} + +message Dictionary { + DictionaryInfo info = 1; + repeated Entry entries = 2; +} diff --git a/src/lib/protobuf.test.ts b/src/lib/protobuf.test.ts new file mode 100644 index 0000000..e0f3eea --- /dev/null +++ b/src/lib/protobuf.test.ts @@ -0,0 +1,28 @@ +import { + writeDictionaryInfo, + readDictionaryInfo, + writeDictionary, + readDictionary, +} from "./protobuf"; +import * as T from "../types"; + +const sampleDictionaryInfo: T.DictionaryInfo = { + title: "Sample Dictionary", + license: "none", + url: "https://www.example.com", + infoUrl: "https://www.example.com", + release: 1, + numberOfEntries: 5, +}; + +const sampleDictionary: T.Dictionary = { + info: sampleDictionaryInfo, + entries: [ + {"i":616,"ts":1527813108,"p":"اظهار","f":"izháar","e":"expression, statement, declaration","c":"n. m."}, + ], +} + +test("should encode and decode", () => { + expect(readDictionaryInfo(writeDictionaryInfo(sampleDictionaryInfo))).toEqual(sampleDictionaryInfo); + expect(readDictionary(writeDictionary(sampleDictionary))).toEqual(sampleDictionary); +}); \ No newline at end of file diff --git a/src/lib/protobuf.ts b/src/lib/protobuf.ts new file mode 100644 index 0000000..ff16ccc --- /dev/null +++ b/src/lib/protobuf.ts @@ -0,0 +1,30 @@ +import * as T from "../types"; +// @ts-ignore +import * as protoModels from "./dictionary-models.js"; +import Pbf from "pbf"; + +export function writeDictionary(dictionary: T.Dictionary): Uint8Array { + const pbfDict = new Pbf(); + protoModels.Dictionary.write(dictionary, pbfDict); + const buffer = pbfDict.finish(); + return buffer; +} + +export function readDictionary(buffer: Uint8Array): T.Dictionary { + const pbf = new Pbf(buffer); + const dictionary = protoModels.Dictionary.read(pbf) as T.Dictionary; + return dictionary; +} + +export function writeDictionaryInfo(dictionary: T.DictionaryInfo): Uint8Array { + const pbfDict = new Pbf(); + protoModels.DictionaryInfo.write(dictionary, pbfDict); + const buffer = pbfDict.finish(); + return buffer; +} + +export function readDictionaryInfo(buffer: Uint8Array): T.DictionaryInfo { + const pbf = new Pbf(buffer); + const dictionaryInfo = protoModels.DictionaryInfo.read(pbf) as T.DictionaryInfo; + return dictionaryInfo; +} \ No newline at end of file diff --git a/src/library.ts b/src/library.ts index 7c2bb0b..0e80144 100644 --- a/src/library.ts +++ b/src/library.ts @@ -44,6 +44,12 @@ import { import { validateEntry } from "./lib/validate-entry"; +import { + readDictionary, + writeDictionary, + readDictionaryInfo, + writeDictionaryInfo, +} from "./lib/protobuf"; import defaultTextOptions from "./lib/default-text-options"; import * as grammarUnits from "./lib/grammar-units"; import * as Types from "./types"; @@ -61,6 +67,11 @@ export { convertAfToPkSpelling, convertPkToAfSpelling, validateEntry, + // protobuf helpers + readDictionary, + writeDictionary, + readDictionaryInfo, + writeDictionaryInfo, // COMPONENTS ConjugationViewer, Examples, diff --git a/yarn.lock b/yarn.lock index 674ef8b..a228abd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1849,6 +1849,11 @@ resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0" integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA== +"@types/pbf@^3.0.2": + version "3.0.2" + resolved "https://registry.yarnpkg.com/@types/pbf/-/pbf-3.0.2.tgz#8d291ad68b4b8c533e96c174a2e3e6399a59ed61" + integrity sha512-EDrLIPaPXOZqDjrkzxxbX7UlJSeQVgah3i0aA4pOSzmK9zq3BIh7/MZIQxED7slJByvKM4Gc6Hypyu2lJzh3SQ== + "@types/prettier@^2.0.0": version "2.2.2" resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.2.2.tgz#e2280c89ddcbeef340099d6968d8c86ba155fdf6" @@ -5758,7 +5763,7 @@ identity-obj-proxy@3.0.0: dependencies: harmony-reflect "^1.4.6" -ieee754@^1.1.4: +ieee754@^1.1.12, ieee754@^1.1.4: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== @@ -8109,6 +8114,14 @@ path-type@^4.0.0: resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== +pbf@^3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/pbf/-/pbf-3.2.1.tgz#b4c1b9e72af966cd82c6531691115cc0409ffe2a" + integrity sha512-ClrV7pNOn7rtmoQVF4TS1vyU0WhYRnP92fzbfF75jAIwpnzdJXf8iTd4CMEqO4yUenH6NDqLiwjqlh6QgZzgLQ== + dependencies: + ieee754 "^1.1.12" + resolve-protobuf-schema "^2.1.0" + pbkdf2@^3.0.3: version "3.1.1" resolved "https://registry.yarnpkg.com/pbkdf2/-/pbkdf2-3.1.1.tgz#cb8724b0fada984596856d1a6ebafd3584654b94" @@ -8969,6 +8982,11 @@ prop-types@^15.6.2, prop-types@^15.7.2: object-assign "^4.1.1" react-is "^16.8.1" +protocol-buffers-schema@^3.3.1: + version "3.5.1" + resolved "https://registry.yarnpkg.com/protocol-buffers-schema/-/protocol-buffers-schema-3.5.1.tgz#8388e768d383ac8cbea23e1280dfadb79f4122ad" + integrity sha512-YVCvdhxWNDP8/nJDyXLuM+UFsuPk4+1PB7WGPVDzm3HTHbzFLxQYeW2iZpS4mmnXrQJGBzt230t/BbEb7PrQaw== + proxy-addr@~2.0.5: version "2.0.6" resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.6.tgz#fdc2336505447d3f2f2c638ed272caf614bbb2bf" @@ -9612,6 +9630,13 @@ resolve-from@^5.0.0: resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-5.0.0.tgz#c35225843df8f776df21c57557bc087e9dfdfc69" integrity sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw== +resolve-protobuf-schema@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/resolve-protobuf-schema/-/resolve-protobuf-schema-2.1.0.tgz#9ca9a9e69cf192bbdaf1006ec1973948aa4a3758" + integrity sha512-kI5ffTiZWmJaS/huM8wZfEMer1eRd7oJQhDuxeCLe3t7N7mX3z94CN0xPxBQxFYQTSNz9T0i+v6inKqSdK8xrQ== + dependencies: + protocol-buffers-schema "^3.3.1" + resolve-url-loader@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/resolve-url-loader/-/resolve-url-loader-3.1.2.tgz#235e2c28e22e3e432ba7a5d4e305c59a58edfc08"