try moving functions around

This commit is contained in:
adueck 2024-11-27 22:27:59 +05:00
parent f9afbd017c
commit 75be56b3f8
39 changed files with 4150 additions and 13195 deletions

View File

@ -1,5 +0,0 @@
{
"projects": {
"default": "lingdocs"
}
}

View File

@ -1,19 +0,0 @@
name: Deploy Hono
on:
push:
branches:
- master
jobs:
deploy:
runs-on: ubuntu-latest
name: Deploy
steps:
- uses: actions/checkout@v4
- name: Deploy
uses: cloudflare/wrangler-action@v3
with:
apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
workingDirectory: "new-functions"

View File

@ -9,6 +9,7 @@
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"@googleapis/sheets": "^9.3.1",
"@lingdocs/inflect": "7.7.1",
"base64url": "^3.0.1",
"bcryptjs": "^2.4.3",
@ -19,6 +20,7 @@
"ejs": "^3.1.6",
"express": "^4.17.2",
"express-session": "^1.17.2",
"googleapis": "^144.0.0",
"lokijs": "^1.5.12",
"nano": "^9.0.3",
"next": "^13.4.12",
@ -509,6 +511,18 @@
"node": ">=18"
}
},
"node_modules/@googleapis/sheets": {
"version": "9.3.1",
"resolved": "https://registry.npmjs.org/@googleapis/sheets/-/sheets-9.3.1.tgz",
"integrity": "sha512-nPgzOiDs/FSFhE+dX2KfkmsmkXM3WfXYP06FoW8cXvHshwxHSI3FbXwe5XJYstDAWXP9YA7AMSvmwnuD4OAl2w==",
"license": "Apache-2.0",
"dependencies": {
"googleapis-common": "^7.0.0"
},
"engines": {
"node": ">=12.0.0"
}
},
"node_modules/@jridgewell/resolve-uri": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz",
@ -1010,6 +1024,41 @@
"node": ">=0.4.0"
}
},
"node_modules/agent-base": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz",
"integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==",
"license": "MIT",
"dependencies": {
"debug": "^4.3.4"
},
"engines": {
"node": ">= 14"
}
},
"node_modules/agent-base/node_modules/debug": {
"version": "4.3.7",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz",
"integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==",
"license": "MIT",
"dependencies": {
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/agent-base/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"license": "MIT"
},
"node_modules/ansi-styles": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
@ -1123,6 +1172,26 @@
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
"integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
},
"node_modules/base64-js": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
"integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"license": "MIT"
},
"node_modules/base64url": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/base64url/-/base64url-3.0.1.tgz",
@ -1136,6 +1205,15 @@
"resolved": "https://registry.npmjs.org/bcryptjs/-/bcryptjs-2.4.3.tgz",
"integrity": "sha1-mrVie5PmBiH/fNrF2pczAn3x0Ms= sha512-V/Hy/X9Vt7f3BbPJEi8BdVFMByHi+jNXrYkW3huaybV/kQ0KJg0Y6PkEMbn+zeT+i+SiKZ/HMqJGIIt4LZDqNQ=="
},
"node_modules/bignumber.js": {
"version": "9.1.2",
"resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz",
"integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==",
"license": "MIT",
"engines": {
"node": "*"
}
},
"node_modules/binary-extensions": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
@ -1191,6 +1269,12 @@
"node": ">=8"
}
},
"node_modules/buffer-equal-constant-time": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
"integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==",
"license": "BSD-3-Clause"
},
"node_modules/buffer-from": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
@ -1466,6 +1550,15 @@
"xtend": "^4.0.0"
}
},
"node_modules/ecdsa-sig-formatter": {
"version": "1.0.11",
"resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz",
"integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==",
"license": "Apache-2.0",
"dependencies": {
"safe-buffer": "^5.0.1"
}
},
"node_modules/ee-first": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@ -1620,6 +1713,12 @@
"node": ">= 0.8"
}
},
"node_modules/extend": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==",
"license": "MIT"
},
"node_modules/filelist": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.2.tgz",
@ -1750,6 +1849,48 @@
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
"integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
},
"node_modules/gaxios": {
"version": "6.7.1",
"resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz",
"integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==",
"license": "Apache-2.0",
"dependencies": {
"extend": "^3.0.2",
"https-proxy-agent": "^7.0.1",
"is-stream": "^2.0.0",
"node-fetch": "^2.6.9",
"uuid": "^9.0.1"
},
"engines": {
"node": ">=14"
}
},
"node_modules/gaxios/node_modules/uuid": {
"version": "9.0.1",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz",
"integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==",
"funding": [
"https://github.com/sponsors/broofa",
"https://github.com/sponsors/ctavan"
],
"license": "MIT",
"bin": {
"uuid": "dist/bin/uuid"
}
},
"node_modules/gcp-metadata": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz",
"integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==",
"license": "Apache-2.0",
"dependencies": {
"gaxios": "^6.0.0",
"json-bigint": "^1.0.0"
},
"engines": {
"node": ">=14"
}
},
"node_modules/get-intrinsic": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz",
@ -1813,11 +1954,84 @@
"resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz",
"integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw=="
},
"node_modules/google-auth-library": {
"version": "9.15.0",
"resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.0.tgz",
"integrity": "sha512-7ccSEJFDFO7exFbO6NRyC+xH8/mZ1GZGG2xxx9iHxZWcjUjJpjWxIMw3cofAKcueZ6DATiukmmprD7yavQHOyQ==",
"license": "Apache-2.0",
"dependencies": {
"base64-js": "^1.3.0",
"ecdsa-sig-formatter": "^1.0.11",
"gaxios": "^6.1.1",
"gcp-metadata": "^6.1.0",
"gtoken": "^7.0.0",
"jws": "^4.0.0"
},
"engines": {
"node": ">=14"
}
},
"node_modules/googleapis": {
"version": "144.0.0",
"resolved": "https://registry.npmjs.org/googleapis/-/googleapis-144.0.0.tgz",
"integrity": "sha512-ELcWOXtJxjPX4vsKMh+7V+jZvgPwYMlEhQFiu2sa9Qmt5veX8nwXPksOWGGN6Zk4xCiLygUyaz7xGtcMO+Onxw==",
"license": "Apache-2.0",
"dependencies": {
"google-auth-library": "^9.0.0",
"googleapis-common": "^7.0.0"
},
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/googleapis-common": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/googleapis-common/-/googleapis-common-7.2.0.tgz",
"integrity": "sha512-/fhDZEJZvOV3X5jmD+fKxMqma5q2Q9nZNSF3kn1F18tpxmA86BcTxAGBQdM0N89Z3bEaIs+HVznSmFJEAmMTjA==",
"license": "Apache-2.0",
"dependencies": {
"extend": "^3.0.2",
"gaxios": "^6.0.3",
"google-auth-library": "^9.7.0",
"qs": "^6.7.0",
"url-template": "^2.0.8",
"uuid": "^9.0.0"
},
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/googleapis-common/node_modules/uuid": {
"version": "9.0.1",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz",
"integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==",
"funding": [
"https://github.com/sponsors/broofa",
"https://github.com/sponsors/ctavan"
],
"license": "MIT",
"bin": {
"uuid": "dist/bin/uuid"
}
},
"node_modules/graceful-fs": {
"version": "4.2.8",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz",
"integrity": "sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg=="
},
"node_modules/gtoken": {
"version": "7.1.0",
"resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz",
"integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==",
"license": "MIT",
"dependencies": {
"gaxios": "^6.0.0",
"jws": "^4.0.0"
},
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/has": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
@ -1877,6 +2091,42 @@
"node": ">= 0.6"
}
},
"node_modules/https-proxy-agent": {
"version": "7.0.5",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz",
"integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==",
"license": "MIT",
"dependencies": {
"agent-base": "^7.0.2",
"debug": "4"
},
"engines": {
"node": ">= 14"
}
},
"node_modules/https-proxy-agent/node_modules/debug": {
"version": "4.3.7",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz",
"integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==",
"license": "MIT",
"dependencies": {
"ms": "^2.1.3"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/https-proxy-agent/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"license": "MIT"
},
"node_modules/iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
@ -2009,6 +2259,18 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/is-stream": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
"integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==",
"license": "MIT",
"engines": {
"node": ">=8"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/is-typedarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
@ -2042,6 +2304,15 @@
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
"peer": true
},
"node_modules/json-bigint": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz",
"integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==",
"license": "MIT",
"dependencies": {
"bignumber.js": "^9.0.0"
}
},
"node_modules/jsonfile": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz",
@ -2059,6 +2330,27 @@
"promise": "^7.0.1"
}
},
"node_modules/jwa": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz",
"integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==",
"license": "MIT",
"dependencies": {
"buffer-equal-constant-time": "1.0.1",
"ecdsa-sig-formatter": "1.0.11",
"safe-buffer": "^5.0.1"
}
},
"node_modules/jws": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz",
"integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==",
"license": "MIT",
"dependencies": {
"jwa": "^2.0.0",
"safe-buffer": "^5.0.1"
}
},
"node_modules/kruptein": {
"version": "2.2.3",
"resolved": "https://registry.npmjs.org/kruptein/-/kruptein-2.2.3.tgz",
@ -2287,9 +2579,10 @@
}
},
"node_modules/node-fetch": {
"version": "2.6.7",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz",
"integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
"license": "MIT",
"dependencies": {
"whatwg-url": "^5.0.0"
},
@ -3372,6 +3665,12 @@
"node": ">= 0.8"
}
},
"node_modules/url-template": {
"version": "2.0.8",
"resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz",
"integrity": "sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw==",
"license": "BSD"
},
"node_modules/utils-merge": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
@ -3711,6 +4010,14 @@
"dev": true,
"optional": true
},
"@googleapis/sheets": {
"version": "9.3.1",
"resolved": "https://registry.npmjs.org/@googleapis/sheets/-/sheets-9.3.1.tgz",
"integrity": "sha512-nPgzOiDs/FSFhE+dX2KfkmsmkXM3WfXYP06FoW8cXvHshwxHSI3FbXwe5XJYstDAWXP9YA7AMSvmwnuD4OAl2w==",
"requires": {
"googleapis-common": "^7.0.0"
}
},
"@jridgewell/resolve-uri": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz",
@ -4114,6 +4421,29 @@
"integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==",
"dev": true
},
"agent-base": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz",
"integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==",
"requires": {
"debug": "^4.3.4"
},
"dependencies": {
"debug": {
"version": "4.3.7",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz",
"integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==",
"requires": {
"ms": "^2.1.3"
}
},
"ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
}
}
},
"ansi-styles": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
@ -4210,6 +4540,11 @@
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
"integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
},
"base64-js": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
"integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="
},
"base64url": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/base64url/-/base64url-3.0.1.tgz",
@ -4220,6 +4555,11 @@
"resolved": "https://registry.npmjs.org/bcryptjs/-/bcryptjs-2.4.3.tgz",
"integrity": "sha1-mrVie5PmBiH/fNrF2pczAn3x0Ms= sha512-V/Hy/X9Vt7f3BbPJEi8BdVFMByHi+jNXrYkW3huaybV/kQ0KJg0Y6PkEMbn+zeT+i+SiKZ/HMqJGIIt4LZDqNQ=="
},
"bignumber.js": {
"version": "9.1.2",
"resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz",
"integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug=="
},
"binary-extensions": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
@ -4266,6 +4606,11 @@
"fill-range": "^7.0.1"
}
},
"buffer-equal-constant-time": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz",
"integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA=="
},
"buffer-from": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
@ -4479,6 +4824,14 @@
"xtend": "^4.0.0"
}
},
"ecdsa-sig-formatter": {
"version": "1.0.11",
"resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz",
"integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==",
"requires": {
"safe-buffer": "^5.0.1"
}
},
"ee-first": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@ -4603,6 +4956,11 @@
}
}
},
"extend": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="
},
"filelist": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.2.tgz",
@ -4693,6 +5051,34 @@
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
"integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
},
"gaxios": {
"version": "6.7.1",
"resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz",
"integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==",
"requires": {
"extend": "^3.0.2",
"https-proxy-agent": "^7.0.1",
"is-stream": "^2.0.0",
"node-fetch": "^2.6.9",
"uuid": "^9.0.1"
},
"dependencies": {
"uuid": {
"version": "9.0.1",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz",
"integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="
}
}
},
"gcp-metadata": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz",
"integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==",
"requires": {
"gaxios": "^6.0.0",
"json-bigint": "^1.0.0"
}
},
"get-intrinsic": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz",
@ -4740,11 +5126,62 @@
"resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz",
"integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw=="
},
"google-auth-library": {
"version": "9.15.0",
"resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.0.tgz",
"integrity": "sha512-7ccSEJFDFO7exFbO6NRyC+xH8/mZ1GZGG2xxx9iHxZWcjUjJpjWxIMw3cofAKcueZ6DATiukmmprD7yavQHOyQ==",
"requires": {
"base64-js": "^1.3.0",
"ecdsa-sig-formatter": "^1.0.11",
"gaxios": "^6.1.1",
"gcp-metadata": "^6.1.0",
"gtoken": "^7.0.0",
"jws": "^4.0.0"
}
},
"googleapis": {
"version": "144.0.0",
"resolved": "https://registry.npmjs.org/googleapis/-/googleapis-144.0.0.tgz",
"integrity": "sha512-ELcWOXtJxjPX4vsKMh+7V+jZvgPwYMlEhQFiu2sa9Qmt5veX8nwXPksOWGGN6Zk4xCiLygUyaz7xGtcMO+Onxw==",
"requires": {
"google-auth-library": "^9.0.0",
"googleapis-common": "^7.0.0"
}
},
"googleapis-common": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/googleapis-common/-/googleapis-common-7.2.0.tgz",
"integrity": "sha512-/fhDZEJZvOV3X5jmD+fKxMqma5q2Q9nZNSF3kn1F18tpxmA86BcTxAGBQdM0N89Z3bEaIs+HVznSmFJEAmMTjA==",
"requires": {
"extend": "^3.0.2",
"gaxios": "^6.0.3",
"google-auth-library": "^9.7.0",
"qs": "^6.7.0",
"url-template": "^2.0.8",
"uuid": "^9.0.0"
},
"dependencies": {
"uuid": {
"version": "9.0.1",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz",
"integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="
}
}
},
"graceful-fs": {
"version": "4.2.8",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz",
"integrity": "sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg=="
},
"gtoken": {
"version": "7.1.0",
"resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz",
"integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==",
"requires": {
"gaxios": "^6.0.0",
"jws": "^4.0.0"
}
},
"has": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
@ -4783,6 +5220,30 @@
"toidentifier": "1.0.1"
}
},
"https-proxy-agent": {
"version": "7.0.5",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz",
"integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==",
"requires": {
"agent-base": "^7.0.2",
"debug": "4"
},
"dependencies": {
"debug": {
"version": "4.3.7",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz",
"integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==",
"requires": {
"ms": "^2.1.3"
}
},
"ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
}
}
},
"iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
@ -4882,6 +5343,11 @@
"has-tostringtag": "^1.0.0"
}
},
"is-stream": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
"integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg=="
},
"is-typedarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
@ -4909,6 +5375,14 @@
"integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==",
"peer": true
},
"json-bigint": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz",
"integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==",
"requires": {
"bignumber.js": "^9.0.0"
}
},
"jsonfile": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz",
@ -4926,6 +5400,25 @@
"promise": "^7.0.1"
}
},
"jwa": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz",
"integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==",
"requires": {
"buffer-equal-constant-time": "1.0.1",
"ecdsa-sig-formatter": "1.0.11",
"safe-buffer": "^5.0.1"
}
},
"jws": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz",
"integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==",
"requires": {
"jwa": "^2.0.0",
"safe-buffer": "^5.0.1"
}
},
"kruptein": {
"version": "2.2.3",
"resolved": "https://registry.npmjs.org/kruptein/-/kruptein-2.2.3.tgz",
@ -5074,9 +5567,9 @@
}
},
"node-fetch": {
"version": "2.6.7",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz",
"integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==",
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
"requires": {
"whatwg-url": "^5.0.0"
}
@ -5857,6 +6350,11 @@
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
"integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw= sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="
},
"url-template": {
"version": "2.0.8",
"resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz",
"integrity": "sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw=="
},
"utils-merge": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",

View File

@ -12,6 +12,7 @@
"author": "lingdocs.com",
"license": "ISC",
"dependencies": {
"@googleapis/sheets": "^9.3.1",
"@lingdocs/inflect": "7.7.1",
"base64url": "^3.0.1",
"bcryptjs": "^2.4.3",
@ -22,6 +23,7 @@
"ejs": "^3.1.6",
"express": "^4.17.2",
"express-session": "^1.17.2",
"googleapis": "^144.0.0",
"lokijs": "^1.5.12",
"nano": "^9.0.3",
"next": "^13.4.12",

View File

@ -9,6 +9,7 @@ import inProd from "./lib/inProd";
import feedbackRouter from "./routers/feedback-router";
import paymentRouter from "./routers/payment-router";
import dictionaryRouter from "./routers/dictionary-router";
import submissionsRouter from "./routers/submissions-router";
const sameOriginCorsOpts = {
origin: inProd ? /\.lingdocs\.com$/ : "*",
@ -43,6 +44,7 @@ app.use("/", cors(sameOriginCorsOpts), authRouter(passport));
// REST API - returning json
app.use("/api", cors(sameOriginCorsOpts), apiRouter);
app.use("/feedback", cors(sameOriginCorsOpts), feedbackRouter);
app.use("/submissions", cors(sameOriginCorsOpts), submissionsRouter);
// TODO: check - does this work with the cors ?
app.use("/payment", cors(sameOriginCorsOpts), paymentRouter);

View File

@ -1,3 +1,5 @@
// TODO: REDO THIS THIS IS UGLY
const names = [
"LINGDOCS_EMAIL_HOST",
"LINGDOCS_EMAIL_USER",
@ -12,11 +14,18 @@ const names = [
"STRIPE_SECRET_KEY",
"STRIPE_WEBHOOK_SECRET",
"NTFY_TOPIC",
];
"LINGDOCS_SERVICE_ACCOUNT_KEY",
"LINGDOCS_SERVICE_ACCOUNT_EMAIL",
"LINGDOCS_DICTIONARY_SPREADSHEET",
"LINGDOCS_DICTIONARY_SHEET_ID",
] as const;
const values = names.map((name) => ({
name,
value: process.env[name] || "",
value:
name === "LINGDOCS_SERVICE_ACCOUNT_KEY"
? Buffer.from(process.env[name] || "").toString("base64")
: process.env[name] || "",
}));
const missing = values.filter((v) => !v.value);
@ -42,4 +51,8 @@ export default {
stripeSecretKey: values[10].value,
stripeWebhookSecret: values[11].value,
ntfyTopic: values[12].value,
lingdocsServiceAccountKey: values[13].value,
lingdocsServiceAccountEmail: values[14].value,
lingdocsDictionarySpreadsheet: values[15].value,
lingdocsDictionarySheetId: values[16].value,
};

View File

@ -1,22 +1,48 @@
import Nano from "nano";
import * as FT from "../../website/src/types/functions-types";
// import * as functions from "firebase-functions/v2";
// @ts-ignore
import { defineString } from "firebase-functions/params";
import * as FT from "../../../website/src/types/functions-types";
import {
addDictionaryEntries,
deleteEntry,
Sheets,
updateDictionaryEntries,
} from "../../../functions/lib/spreadsheet-tools";
import { google } from "googleapis";
import env from "./env-vars";
// Define some parameters
// // import {
// // addDictionaryEntries,
// // deleteEntry,
// // updateDictionaryEntries,
// // } from "./tools/spreadsheet-tools";
const couchdbUrl = defineString("ABC");
console.log({ couchdb: couchdbUrl });
const nano = Nano("");
const sheetId = parseInt(env.lingdocsDictionarySheetId);
if (isNaN(sheetId)) {
console.error("Invalid SheetID for LINGDOCS_DICTIONARY_SHEET_ID env var");
process.exit(1);
}
const nano = Nano(env.couchDbURL);
const reviewTasksDb = nano.db.use("review-tasks");
// TODO: get new env vars on server (remember base64 for key)
const auth = new google.auth.GoogleAuth({
credentials: {
// IMPORTANT!! have to have key stored in Base64 because of the
// weirdness of node handling spaces in the key (at least there was on AWS)
private_key: Buffer.from(
env.lingdocsServiceAccountEmail,
"base64"
).toString("ascii"),
client_email: env.lingdocsServiceAccountKey,
},
scopes: [
"https://www.googleapis.com/auth/spreadsheets",
"https://www.googleapis.com/auth/drive.file",
],
});
const { spreadsheets } = google.sheets({
version: "v4",
auth,
});
const sheets: Sheets = {
spreadsheetId: env.lingdocsDictionarySpreadsheet,
spreadsheets,
};
export async function receiveSubmissions(
e: FT.SubmissionsRequest,
editor: boolean
@ -39,12 +65,12 @@ export async function receiveSubmissions(
}
if (edits.length && editor) {
// const { newEntries, entryEdits, entryDeletions } = sortEdits(edits);
// await updateDictionaryEntries(entryEdits);
// for (const ed of entryDeletions) {
// await deleteEntry(ed);
// }
// await addDictionaryEntries(newEntries);
const { newEntries, entryEdits, entryDeletions } = sortEdits(edits);
await updateDictionaryEntries(sheets, entryEdits);
for (const ed of entryDeletions) {
await deleteEntry(sheets, sheetId, ed);
}
await addDictionaryEntries(sheets, newEntries);
}
return {

View File

@ -0,0 +1,31 @@
import express, { Response } from "express";
import * as T from "../../../website/src/types/account-types";
import { receiveSubmissions } from "../lib/submissions";
// TODO: ADD PROPER ERROR HANDLING THAT WILL RETURN JSON ALWAYS
function sendResponse(res: Response, payload: T.APIResponse) {
return res.send(payload);
}
const submissionsRouter = express.Router();
// Guard all api with authentication
submissionsRouter.use((req, res, next) => {
if (req.isAuthenticated()) {
return next();
}
const r: T.APIResponse = { ok: false, error: "401 Unauthorized" };
return res.status(401).send(r);
});
/**
* Receive a submissions request
*/
submissionsRouter.post("/", async (req, res, next) => {
if (!req.user) return next("user not found");
const r = await receiveSubmissions(req.body, !!req.user.admin);
sendResponse(res, r);
});
export default submissionsRouter;

View File

@ -1,19 +0,0 @@
{
"functions": {
"predeploy": "cp .npmrc functions && cat .npmrc | envsubst > functions/.npmrc && cd functions && npm --prefix \"$RESOURCE_DIR\" run build",
"postdeploy": "rm functions/.npmrc"
},
"hosting": {
"public": "public",
"rewrites": [
{
"source": "/publishDictionary",
"function": "/publishDictionary"
},
{
"source": "/submissions",
"function": "/submissions"
}
]
}
}

24
functions/.gitignore vendored
View File

@ -1,17 +1,9 @@
# Debug
ui-debug.log
*.js
!jest.config.js
*.d.ts
node_modules
!lib
# Compiled JavaScript files
lib
# TypeScript v1 declaration files
typings/
# Node.js dependency directory
node_modules/
# File with private NPM token(s) inserted for deploying function
.npmrc
# Firebase functions config/env for running functions locally
.runtimeconfig.json
# CDK asset staging directory
.cdk.staging
cdk.out

6
functions/.npmignore Normal file
View File

@ -0,0 +1,6 @@
*.ts
!*.d.ts
# CDK asset staging directory
.cdk.staging
cdk.out

14
functions/README.md Normal file
View File

@ -0,0 +1,14 @@
# Publish Dictionary Function
This is a CDK / AWS Lambda project for the publish dictionary function
The `cdk.json` file tells the CDK Toolkit how to execute your app.
## Useful commands
- `npm run build` compile typescript to js
- `npm run watch` watch for changes and compile
- `npm run test` perform the jest unit tests
- `npx cdk deploy` deploy this stack to your default AWS account/region
- `npx cdk diff` compare deployed stack with current state
- `npx cdk synth` emits the synthesized CloudFormation template

View File

@ -0,0 +1,20 @@
#!/usr/bin/env node
import * as cdk from 'aws-cdk-lib';
import { FunctionsStack } from '../lib/functions-stack';
const app = new cdk.App();
new FunctionsStack(app, 'FunctionsStack', {
/* If you don't specify 'env', this stack will be environment-agnostic.
* Account/Region-dependent features and context lookups will not work,
* but a single synthesized template can be deployed anywhere. */
/* Uncomment the next line to specialize this stack for the AWS Account
* and Region that are implied by the current CLI configuration. */
// env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION },
/* Uncomment the next line if you know exactly what Account and Region you
* want to deploy the stack to. */
// env: { account: '123456789012', region: 'us-east-1' },
/* For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html */
});

80
functions/cdk.json Normal file
View File

@ -0,0 +1,80 @@
{
"app": "npx ts-node --prefer-ts-exts bin/functions.ts",
"watch": {
"include": [
"**"
],
"exclude": [
"README.md",
"cdk*.json",
"**/*.d.ts",
"**/*.js",
"tsconfig.json",
"package*.json",
"yarn.lock",
"node_modules",
"test"
]
},
"context": {
"@aws-cdk/aws-lambda:recognizeLayerVersion": true,
"@aws-cdk/core:checkSecretUsage": true,
"@aws-cdk/core:target-partitions": [
"aws",
"aws-cn"
],
"@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true,
"@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true,
"@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true,
"@aws-cdk/aws-iam:minimizePolicies": true,
"@aws-cdk/core:validateSnapshotRemovalPolicy": true,
"@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true,
"@aws-cdk/aws-s3:createDefaultLoggingPolicy": true,
"@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true,
"@aws-cdk/aws-apigateway:disableCloudWatchRole": true,
"@aws-cdk/core:enablePartitionLiterals": true,
"@aws-cdk/aws-events:eventsTargetQueueSameAccount": true,
"@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true,
"@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true,
"@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true,
"@aws-cdk/aws-route53-patters:useCertificate": true,
"@aws-cdk/customresources:installLatestAwsSdkDefault": false,
"@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true,
"@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true,
"@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true,
"@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true,
"@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true,
"@aws-cdk/aws-redshift:columnId": true,
"@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true,
"@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true,
"@aws-cdk/aws-apigateway:requestValidatorUniqueId": true,
"@aws-cdk/aws-kms:aliasNameRef": true,
"@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true,
"@aws-cdk/core:includePrefixInUniqueNameGeneration": true,
"@aws-cdk/aws-efs:denyAnonymousAccess": true,
"@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true,
"@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true,
"@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true,
"@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true,
"@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true,
"@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true,
"@aws-cdk/aws-codepipeline-actions:useNewDefaultBranchForCodeCommitSource": true,
"@aws-cdk/aws-cloudwatch-actions:changeLambdaPermissionLogicalIdForLambdaAction": true,
"@aws-cdk/aws-codepipeline:crossAccountKeysDefaultValueToFalse": true,
"@aws-cdk/aws-codepipeline:defaultPipelineTypeToV2": true,
"@aws-cdk/aws-kms:reduceCrossAccountRegionPolicyScope": true,
"@aws-cdk/aws-eks:nodegroupNameAttribute": true,
"@aws-cdk/aws-ec2:ebsDefaultGp3Volume": true,
"@aws-cdk/aws-ecs:removeDefaultDeploymentAlarm": true,
"@aws-cdk/custom-resources:logApiResponseDataPropertyTrueDefault": false,
"@aws-cdk/aws-s3:keepNotificationInImportedBucket": false,
"@aws-cdk/aws-ecs:reduceEc2FargateCloudWatchPermissions": true,
"@aws-cdk/aws-dynamodb:resourcePolicyPerReplica": true,
"@aws-cdk/aws-ec2:ec2SumTImeoutEnabled": true,
"@aws-cdk/aws-appsync:appSyncGraphQLAPIScopeLambdaPermission": true,
"@aws-cdk/aws-rds:setCorrectValueForDatabaseInstanceReadReplicaInstanceResourceId": true,
"@aws-cdk/core:cfnIncludeRejectComplexResourceUpdateCreatePolicyIntrinsics": true,
"@aws-cdk/aws-lambda-nodejs:sdkV3ExcludeSmithyPackages": true,
"@aws-cdk/aws-stepfunctions-tasks:fixRunEcsTaskPolicy": true
}
}

View File

@ -1,5 +1,8 @@
/** @type {import('ts-jest').JestConfigWithTsJest} */
module.exports = {
preset: 'ts-jest',
testEnvironment: 'node',
roots: ['<rootDir>/test'],
testMatch: ['**/*.test.ts'],
transform: {
'^.+\\.tsx?$': 'ts-jest'
}
};

98
functions/lambda/index.ts Normal file
View File

@ -0,0 +1,98 @@
import { Hono } from "hono";
import { handle } from "hono/aws-lambda";
// wish we could tree shake this!
import { google } from "googleapis";
import { sheets } from "@googleapis/sheets";
import { getEntriesFromSheet } from "../lib/spreadsheet-tools";
import {
checkForErrors,
dictionaryFilename,
dictionaryInfoFilename,
makeDictionaryObject,
makeSitemap,
} from "../lib/publishing-helpers";
import { uploader } from "../lib/uploader";
import { S3Client } from "@aws-sdk/client-s3";
import { getEnv } from "../lib/env-helper";
// import { getWordList } from "../lib/word-list-maker";
// import { Types as T } from "@lingdocs/inflect";
// const allWordsJsonFilename = "all-words-dictionary2.json";
const app = new Hono();
app.get("/publish", async (c) => {
const vars = getEnv(c);
const auth = new google.auth.GoogleAuth({
credentials: {
// IMPORTANT!! have to have key stored in Base64 because of the
// weirdness of node handling spaces in the key
private_key: Buffer.from(
vars.LINGDOCS_SERVICE_ACCOUNT_KEY,
"base64"
).toString("ascii"),
client_email: vars.LINGDOCS_SERVICE_ACCOUNT_EMAIL,
},
scopes: [
"https://www.googleapis.com/auth/spreadsheets",
"https://www.googleapis.com/auth/drive.file",
],
});
const { spreadsheets } = sheets({
version: "v4",
auth,
});
const entries = await getEntriesFromSheet({
spreadsheets,
spreadsheetId: vars.LINGDOCS_DICTIONARY_SPREADSHEET,
});
const errors = checkForErrors(entries);
if (errors.length) {
return c.json({
ok: false,
errors,
});
}
const dictionary = makeDictionaryObject(entries);
const sitemap = makeSitemap(dictionary);
// const wordListRes = getWordList(dictionary.entries);
// if (!wordListRes.ok) {
// return c.json({
// ok: false,
// error: "error(s) in creating inflections",
// errors: wordListRes.errors,
// });
// }
// const wordList: T.AllWordsWithInflections = {
// info: dictionary.info,
// words: wordListRes.wordlist,
// };
// got dictionary, now upload it to storage
const s3Client = new S3Client({
region: "auto",
endpoint: vars.DICT_R2_ENDPOINT,
credentials: {
accessKeyId: vars.DICT_R2_KEY_ID,
secretAccessKey: vars.DICT_R2_KEY_SECRET,
},
});
const upload = uploader(vars.DICT_R2_BUCKET, s3Client);
const uploadResult = await Promise.all([
upload(JSON.stringify(dictionary), `${dictionaryFilename}.json`),
upload(JSON.stringify(dictionary.info), `${dictionaryInfoFilename}.json`),
upload(sitemap, `sitemap2.xml`),
// upload(JSON.stringify(wordList), allWordsJsonFilename),
]);
if (uploadResult.some((res) => res.output.$metadata.httpStatusCode !== 200)) {
return c.json({
ok: false,
error: "error uploading file(s)",
uploadResult,
});
}
return c.json({
ok: true,
info: dictionary.info,
});
});
export const handler = handle(app);

View File

@ -0,0 +1,39 @@
import { Context } from "hono";
import { env } from "hono/adapter";
export type FEnvironment = {
LINGDOCS_DICTIONARY_SPREADSHEET: string;
LINGDOCS_DICTIONARY_SHEET_ID: string;
LINGDOCS_SERVICE_ACCOUNT_EMAIL: string;
LINGDOCS_SERVICE_ACCOUNT_KEY: string;
DICT_R2_ENDPOINT: string;
DICT_R2_KEY_ID: string;
DICT_R2_KEY_SECRET: string;
DICT_R2_BUCKET: string;
};
export const environment: FEnvironment = {
LINGDOCS_DICTIONARY_SPREADSHEET:
process.env.LINGDOCS_DICTIONARY_SPREADSHEET || "",
LINGDOCS_DICTIONARY_SHEET_ID: process.env.LINGDOCS_DICTIONARY_SHEET_ID || "",
LINGDOCS_SERVICE_ACCOUNT_EMAIL:
process.env.LINGDOCS_SERVICE_ACCOUNT_EMAIL || "",
LINGDOCS_SERVICE_ACCOUNT_KEY: Buffer.from(
process.env.LINGDOCS_SERVICE_ACCOUNT_KEY || ""
).toString("base64"),
DICT_R2_ENDPOINT: process.env.DICT_R2_ENDPOINT || "",
DICT_R2_KEY_ID: process.env.DICT_R2_KEY_ID || "",
DICT_R2_KEY_SECRET: process.env.DICT_R2_KEY_SECRET || "",
DICT_R2_BUCKET: process.env.DICT_R2_BUCKET || "",
};
Object.entries(environment).forEach(([key, value]) => {
if (value === "") {
console.log(`Missing env var for ${key}`);
process.exit(1);
}
});
export function getEnv(c: Context) {
return env<FEnvironment>(c);
}

View File

@ -0,0 +1,23 @@
import * as cdk from "aws-cdk-lib";
import { Construct } from "constructs";
// import * as sqs from 'aws-cdk-lib/aws-sqs';
import * as lambda from "aws-cdk-lib/aws-lambda";
import { NodejsFunction } from "aws-cdk-lib/aws-lambda-nodejs";
import { environment } from "./env-helper";
export class FunctionsStack extends cdk.Stack {
constructor(scope: Construct, id: string, props?: cdk.StackProps) {
super(scope, id, props);
const fn = new NodejsFunction(this, "lambda", {
entry: "lambda/index.ts",
handler: "handler",
runtime: lambda.Runtime.NODEJS_22_X,
timeout: cdk.Duration.seconds(30),
memorySize: 1028,
environment,
});
fn.addFunctionUrl({
authType: lambda.FunctionUrlAuthType.NONE,
});
}
}

View File

@ -0,0 +1,170 @@
import { Types as T, validateEntry } from "@lingdocs/inflect";
const title = "LingDocs Pashto Dictionary";
const license = `Copyright © ${new Date().getFullYear()} lingdocs.com All Rights Reserved - Licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License - https://creativecommons.org/licenses/by-nc-sa/4.0/`;
const baseUrl = `https://storage.lingdocs.com/dictionary/`;
export const dictionaryFilename = "dictionary2";
export const dictionaryInfoFilename = "dictionary-info2";
// const hunspellAffFileFilename = "ps_AFF.aff";
// const hunspellDicFileFilename = "ps_AFF.dic";
const allWordsJsonFilename = "all-words-dictionary.json";
const url = `${baseUrl}${dictionaryFilename}`;
const infoUrl = `${baseUrl}${dictionaryInfoFilename}`;
export function makeDictionaryObject(
entries: T.DictionaryEntry[]
): T.Dictionary {
return {
info: {
title,
license,
url,
infoUrl,
release: new Date().getTime(),
numberOfEntries: entries.length,
},
entries,
};
}
export function checkForErrors(
entries: T.DictionaryEntry[]
): (T.DictionaryEntryError | { duplicates: number[] })[] {
// check for duplicates
const tsMap: Record<number, T.DictionaryEntry> = {};
const duplicates: number[] = [];
// making a map here based on ts with the entry will speed up the
// compliment checking process!!
for (var i = 0; i < entries.length; i++) {
const ts = entries[i].ts;
if (ts in tsMap) {
duplicates.push(ts);
} else {
tsMap[ts] = entries[i];
}
}
if (duplicates.length) {
return [{ duplicates }];
}
// check for errors
return entries.reduce(
(errors: T.DictionaryEntryError[], entry: T.DictionaryEntry) => {
const response = validateEntry(entry);
if ("errors" in response && response.errors.length) {
return [...errors, response];
}
if ("checkComplement" in response) {
if (!entry.l) {
const error: T.DictionaryEntryError = {
errors: ["complement needed"],
ts: entry.ts,
p: entry.p,
f: entry.f,
e: entry.e,
erroneousFields: ["l"],
};
return [...errors, error];
}
const complement = tsMap[entry.l];
if (!complement) {
const error: T.DictionaryEntryError = {
errors: ["complement link not found in dictionary"],
ts: entry.ts,
p: entry.p,
f: entry.f,
e: entry.e,
erroneousFields: ["l"],
};
return [...errors, error];
}
if (
!complement.c?.includes("n.") &&
!complement.c?.includes("adj.") &&
!complement.c?.includes("adv.")
) {
const error: T.DictionaryEntryError = {
errors: ["complement link to invalid complement"],
ts: entry.ts,
p: entry.p,
f: entry.f,
e: entry.e,
erroneousFields: ["l"],
};
return [...errors, error];
}
}
return errors;
},
[]
);
}
export function makeSitemap(dictionary: T.Dictionary): string {
function tsToDate(ts: number): string {
if (ts < 10000000000) {
// approximate date for old-style timestamps
return "2021-01-01";
}
return getDateString(new Date(ts));
}
function getDateString(d: Date): string {
return d.toISOString().split("T")[0];
}
const pages = [
"",
"about",
"settings",
"account",
"phrase-builder",
"new-entries",
];
const currentDate = getDateString(new Date());
return `<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
${pages
.map(
(page) =>
`
<url>
<loc>https://dictionary.lingdocs.com/${page}</loc>
<lastmod>${currentDate}</lastmod>
</url>`
)
.join("")}
${dictionary.entries
.map(
(entry) =>
`
<url>
<loc>https://dictionary.lingdocs.com/word?id=${entry.ts}</loc>
<lastmod>${tsToDate(entry.ts)}</lastmod>
</url>`
)
.join("")}
</urlset>
`;
}
// FOR HUNSPELL
// const hunspellAffFileFilename = "ps_AFF.aff";
// const hunspellDicFileFilename = "ps_AFF.dic";
// async function doHunspellEtc(
// info: T.DictionaryInfo,
// entries: T.DictionaryEntry[]
// ) {
// const wordlistResponse = getWordList(entries);
// if (!wordlistResponse.ok) {
// throw new Error(JSON.stringify(wordlistResponse.errors));
// }
// // const hunspell = makeHunspell(wordlistResponse.wordlist);
// // await uploadHunspellToStorage(hunspell);
// await uploadAllWordsToStoarage(info, wordlistResponse.wordlist);
// }
// function makeHunspell(wordlist: string[]) {
// return {
// dicContent: wordlist.reduce((acc, word) => acc + word + "\n", wordlist.length + "\n"),
// affContent: "SET UTF-8\nCOMPLEXPREFIXES\nIGNORE ۱۲۳۴۵۶۷۸۹۰-=ًٌٍَُِّْ؛:؟.،,،؟\n",
// };
// }

View File

@ -1,57 +1,92 @@
import { google } from "googleapis";
import { Types as T } from "@lingdocs/inflect";
import * as FT from "../../../website/src/types/functions-types";
import * as FT from "../../website/src/types/functions-types";
import { standardizeEntry } from "@lingdocs/inflect";
import type { sheets_v4 } from "@googleapis/sheets";
import {
dictionaryEntryBooleanFields,
dictionaryEntryNumberFields,
dictionaryEntryTextFields,
simplifyPhonetics,
standardizePashto,
} from "@lingdocs/inflect";
import * as functions from "firebase-functions";
const spreadsheetId = functions.config().sheet.id;
const sheetId = 51288491;
const validFields = [
...dictionaryEntryTextFields,
...dictionaryEntryBooleanFields,
...dictionaryEntryNumberFields,
];
const SCOPES = [
"https://www.googleapis.com/auth/spreadsheets",
"https://www.googleapis.com/auth/drive.file",
];
export type Sheets = {
spreadsheetId: string;
spreadsheets: sheets_v4.Resource$Spreadsheets;
};
const auth = new google.auth.GoogleAuth({
credentials: {
private_key: functions.config().serviceacct.key,
client_email: functions.config().serviceacct.email,
},
scopes: SCOPES,
});
const { spreadsheets } = google.sheets({
version: "v4",
auth,
});
async function getTsIndex(): Promise<number[]> {
const values = await getRange("A2:A");
async function getTsIndex(sheets: Sheets): Promise<number[]> {
const values = await getRange(sheets, "A2:A");
return values.map((r) => parseInt(r[0]));
}
async function getFirstEmptyRow(): Promise<number> {
const values = await getRange("A2:A");
async function getFirstEmptyRow(sheets: Sheets): Promise<number> {
const values = await getRange(sheets, "A2:A");
return values.length + 2;
}
export async function updateDictionaryEntries(edits: FT.EntryEdit[]) {
export async function getEntriesFromSheet({
spreadsheets,
spreadsheetId,
}: Sheets): Promise<T.DictionaryEntry[]> {
const keyInfo = await getKeyInfo({ spreadsheets, spreadsheetId });
const { data } = await spreadsheets.values.get({
spreadsheetId,
range: `A2:${keyInfo.lastCol}`,
});
if (!data.values) {
throw new Error("data not found");
}
function processRow(row: string[]) {
// TODO: optimize this
const processedRow = row.flatMap<
[keyof T.DictionaryEntry, string | boolean | number]
>((x, i) => {
if (x === "") {
return [];
}
const k = keyInfo.keyRow[i];
// @ts-expect-error
if (dictionaryEntryNumberFields.includes(k)) {
return [[k, parseInt(x)]];
}
// @ts-expect-error
if (dictionaryEntryBooleanFields.includes(k)) {
return [[k, x.toLowerCase() === "true"]];
}
return [[k, k.endsWith("p") ? standardizePashto(x.trim()) : x.trim()]];
});
return processedRow;
}
const entries = data.values.map(processRow).map((pr) => {
return Object.fromEntries(pr) as T.DictionaryEntry;
});
entries.sort((a, b) => a.p.localeCompare(b.p, "ps"));
const entriesLength = entries.length;
// add index and g
for (let i = 0; i < entriesLength; i++) {
entries[i].i = i;
entries[i].g = simplifyPhonetics(entries[i].f);
}
return entries;
}
export async function updateDictionaryEntries(
{ spreadsheets, spreadsheetId }: Sheets,
edits: FT.EntryEdit[]
) {
if (edits.length === 0) {
return;
}
const entries = edits.map((e) => e.entry);
const tsIndex = await getTsIndex();
const { keyRow, lastCol } = await getKeyInfo();
const tsIndex = await getTsIndex({ spreadsheets, spreadsheetId });
const { keyRow, lastCol } = await getKeyInfo({ spreadsheets, spreadsheetId });
function entryToRowArray(e: T.DictionaryEntry): any[] {
return keyRow.slice(1).map((k) => e[k] || "");
}
@ -64,7 +99,7 @@ export async function updateDictionaryEntries(edits: FT.EntryEdit[]) {
const values = [entryToRowArray(entry)];
return [
{
range: `B${rowNum}:${lastCol}${rowNum}`,
q: `B${rowNum}:${lastCol}${rowNum}`,
values,
},
];
@ -78,13 +113,16 @@ export async function updateDictionaryEntries(edits: FT.EntryEdit[]) {
});
}
export async function addDictionaryEntries(additions: FT.NewEntry[]) {
export async function addDictionaryEntries(
{ spreadsheets, spreadsheetId }: Sheets,
additions: FT.NewEntry[]
) {
if (additions.length === 0) {
return;
}
const entries = additions.map((x) => standardizeEntry(x.entry));
const endRow = await getFirstEmptyRow();
const { keyRow, lastCol } = await getKeyInfo();
const endRow = await getFirstEmptyRow({ spreadsheets, spreadsheetId });
const { keyRow, lastCol } = await getKeyInfo({ spreadsheets, spreadsheetId });
const ts = Date.now();
function entryToRowArray(e: T.DictionaryEntry): any[] {
return keyRow.slice(1).map((k) => e[k] || "");
@ -105,10 +143,11 @@ export async function addDictionaryEntries(additions: FT.NewEntry[]) {
}
export async function updateDictionaryFields(
{ spreadsheets, spreadsheetId }: Sheets,
edits: { ts: number; col: keyof T.DictionaryEntry; val: any }[]
) {
const tsIndex = await getTsIndex();
const { colMap } = await getKeyInfo();
const tsIndex = await getTsIndex({ spreadsheets, spreadsheetId });
const { colMap } = await getKeyInfo({ spreadsheets, spreadsheetId });
const data = edits.flatMap((edit) => {
const rowNum = getRowNumFromTs(tsIndex, edit.ts);
if (rowNum === undefined) {
@ -132,8 +171,12 @@ export async function updateDictionaryFields(
});
}
export async function deleteEntry(ed: FT.EntryDeletion) {
const tsIndex = await getTsIndex();
export async function deleteEntry(
{ spreadsheets, spreadsheetId }: Sheets,
sheetId: number,
ed: FT.EntryDeletion
) {
const tsIndex = await getTsIndex({ spreadsheets, spreadsheetId });
const row = getRowNumFromTs(tsIndex, ed.ts);
if (!row) {
console.error(`${ed.ts} not found to do delete`);
@ -169,28 +212,35 @@ function getRowNumFromTs(tsIndex: number[], ts: number): number | undefined {
return res + 2;
}
async function getKeyInfo(): Promise<{
async function getKeyInfo(sheets: Sheets): Promise<{
colMap: Record<keyof T.DictionaryEntry, string>;
colMapN: Record<keyof T.DictionaryEntry, number>;
keyRow: (keyof T.DictionaryEntry)[];
lastCol: string;
}> {
const headVals = await getRange("A1:1");
const headVals = await getRange(sheets, "A1:1");
const headRow: string[] = headVals[0];
const colMap: any = {};
const colMap: Record<any, string> = {};
const colMapN: Record<any, number> = {};
headRow.forEach((c, i) => {
if (validFields.every((v) => c !== v)) {
throw new Error(`Invalid spreadsheet field ${c}`);
}
colMap[c] = getColumnLetters(i);
colMapN[c] = i;
});
return {
colMap: colMap as Record<keyof T.DictionaryEntry, string>,
colMapN: colMapN as Record<keyof T.DictionaryEntry, number>,
keyRow: headRow as (keyof T.DictionaryEntry)[],
lastCol: getColumnLetters(headRow.length - 1),
};
}
async function getRange(range: string): Promise<any[][]> {
async function getRange(
{ spreadsheets, spreadsheetId }: Sheets,
range: string
): Promise<any[][]> {
const { data } = await spreadsheets.values.get({
spreadsheetId,
range,

39
functions/lib/uploader.ts Normal file
View File

@ -0,0 +1,39 @@
import {
S3Client,
PutObjectCommand,
PutBucketAclCommandOutput,
} from "@aws-sdk/client-s3";
import * as zlib from "node:zlib";
export const uploader =
(bucket: string, s3Client: S3Client) => (content: string, filename: string) =>
new Promise<{ filename: string; output: PutBucketAclCommandOutput }>(
(resolve, reject) => {
// upload to r2 (new destination)
zlib.gzip(content, (err, buffer) => {
if (err) {
console.error(err);
reject(err);
}
const putObjectCommand = new PutObjectCommand({
Bucket: bucket,
Key: `dictionary/${filename}`,
CacheControl: "no-cache",
Body: buffer,
ContentEncoding: "gzip",
ContentType: filename.endsWith(".json")
? "application/json"
: filename.endsWith(".xml")
? "application/xml"
: "text/plain; charset=UTF-8",
});
s3Client
.send(putObjectCommand)
.then((output) => resolve({ filename, output }))
.catch((err) => {
console.error(err);
reject(err);
});
});
}
);

13964
functions/package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -1,41 +1,32 @@
{
"name": "functions",
"version": "0.1.0",
"bin": {
"functions": "bin/functions.js"
},
"scripts": {
"build": "tsc",
"serve": "npm run build && firebase emulators:start --only functions",
"shell": "npm run build && firebase functions:shell",
"start": "npm run shell",
"deploy": "firebase deploy --only functions",
"logs": "firebase functions:log",
"test": "jest"
},
"engines": {
"node": "20"
},
"main": "lib/functions/src/index.js",
"dependencies": {
"@aws-sdk/client-s3": "^3.474.0",
"@lingdocs/inflect": "7.7.1",
"@types/cors": "^2.8.10",
"@types/google-spreadsheet": "^3.0.2",
"@types/react": "^18.0.21",
"cors": "^2.8.5",
"firebase-admin": "^13.0.1",
"firebase-functions": "^6.1.1",
"googleapis": "^144.0.0",
"nano": "^9.0.3",
"node-fetch": "^2.6.1",
"react": "^17.0.1",
"react-bootstrap": "^1.5.1",
"react-dom": "^17.0.1"
"watch": "tsc -w",
"test": "jest",
"cdk": "cdk"
},
"devDependencies": {
"@types/jest": "^26.0.20",
"@types/node-fetch": "^2.5.12",
"firebase-functions-test": "^0.2.0",
"jest": "^29.3.1",
"ts-jest": "^29.0.5",
"ts-node": "^10.9.1",
"typescript": "^4.6.3"
"@types/jest": "^29.5.14",
"@types/node": "22.7.9",
"aws-cdk": "2.171.0",
"jest": "^29.7.0",
"ts-jest": "^29.2.5",
"ts-node": "^10.9.2",
"typescript": "~5.6.3"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.701.0",
"@googleapis/sheets": "^9.3.1",
"@lingdocs/inflect": "^7.7.1",
"aws-cdk-lib": "2.171.0",
"constructs": "^10.0.0",
"google-auth-library": "^9.15.0",
"googleapis": "^144.0.0",
"hono": "^4.6.12"
}
}

View File

@ -1,63 +0,0 @@
import * as functions from "firebase-functions/v2";
import * as FT from "../../website/src/types/functions-types";
import { receiveSubmissions } from "./submissions";
import lingdocsAuth from "./middleware/lingdocs-auth";
import publish from "./publish";
const couchdbUrl = functions.params.defineString("ABC");
console.log({ couchdb: couchdbUrl.value() });
export const publishDictionary = functions.https.onRequest(
{
timeoutSeconds: 525,
memory: "2GiB",
},
lingdocsAuth(
async (
req,
res // : functions.Response<FT.PublishDictionaryResponse | FT.FunctionError>
) => {
if (req.user.level !== "editor") {
res.status(403).send({ ok: false, error: "403 forbidden" });
return;
}
try {
const response = await publish();
res.send(response);
} catch (e) {
// @ts-ignore
res.status(500).send({ ok: false, error: e.message });
}
}
)
);
export const submissions = functions.https.onRequest(
{
timeoutSeconds: 60,
memory: "1GiB",
},
lingdocsAuth(
async (
req,
res // : functions.Response<FT.SubmissionsResponse | FT.FunctionError>
) => {
if (!Array.isArray(req.body)) {
res.status(400).send({
ok: false,
error: "invalid submission",
});
return;
}
const suggestions = req.body as FT.SubmissionsRequest;
try {
const response = await receiveSubmissions(suggestions, true); // req.user.level === "editor");
// TODO: WARN IF ANY OF THE EDITS DIDN'T HAPPEN
res.send(response);
} catch (e) {
// @ts-ignore
res.status(500).send({ ok: false, error: e.message });
}
}
)
);

View File

@ -1,63 +0,0 @@
import cors from "cors";
import fetch from "node-fetch";
// unfortunately have to comment out all this typing because the new version
// of firebase-functions doesn't include it?
// import type { https, Response } from "firebase-functions";
// import * as FT from "../../../website/src/types/functions-types";
// import type { LingdocsUser } from "../../../website/src/types/account-types";
const useCors = cors({ credentials: true, origin: /\.lingdocs\.com$/ });
// interface ReqWUser extends https.Request {
// user: LingdocsUser;
// }
/**
* creates a handler to pass to a firebase https.onRequest function
*
*/
export default function makeHandler(
toRun: (
req: any, //ReqWUser,
res: any /*Response<FT.FunctionResponse> */
) => any | Promise<any>
) {
return function (
reqPlain: any /* https.Request */,
resPlain: any /* Response<any> */
) {
useCors(reqPlain, resPlain, async () => {
const { req, res } = await authorize(reqPlain, resPlain);
if (!req) {
res.status(401).send({ ok: false, error: "unauthorized" });
return;
}
toRun(req, res);
return;
});
};
}
async function authorize(
req: any /* https.Request*/,
res: any /*Response<any>*/
): Promise<{
req: any; // ReqWUser | null;
res: any /*Response<FT.FunctionResponse>*/;
}> {
const {
headers: { cookie },
} = req;
if (!cookie) {
return { req: null, res };
}
const r = await fetch("https://account.lingdocs.com/api/user", {
headers: { cookie },
});
const { ok, user } = await r.json();
if (ok === true && user) {
req.user = user;
return { req: req /* as ReqWUser*/, res };
}
return { req: null, res };
}

View File

@ -1,351 +0,0 @@
import { GoogleSpreadsheet } from "google-spreadsheet";
import * as functions from "firebase-functions";
import {
Types as T,
dictionaryEntryBooleanFields,
dictionaryEntryNumberFields,
dictionaryEntryTextFields,
validateEntry,
simplifyPhonetics,
standardizeEntry,
} from "@lingdocs/inflect";
import { getWordList } from "./word-list-maker";
import { PublishDictionaryResponse } from "../../website/src/types/functions-types";
import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3";
import zlib from "zlib";
const s3Client = new S3Client({
region: "auto",
endpoint: functions.config().r2.endpoint,
credentials: {
accessKeyId: functions.config().r2.access_key_id,
secretAccessKey: functions.config().r2.secret_access_key,
},
});
const title = "LingDocs Pashto Dictionary";
const license = `Copyright © ${new Date().getFullYear()} lingdocs.com All Rights Reserved - Licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License - https://creativecommons.org/licenses/by-nc-sa/4.0/`;
const baseUrl = `https://storage.lingdocs.com/dictionary/`;
const dictionaryFilename = "dictionary";
const dictionaryInfoFilename = "dictionary-info";
// const hunspellAffFileFilename = "ps_AFF.aff";
// const hunspellDicFileFilename = "ps_AFF.dic";
const allWordsJsonFilename = "all-words-dictionary.json";
const url = `${baseUrl}${dictionaryFilename}`;
const infoUrl = `${baseUrl}${dictionaryInfoFilename}`;
// TODO: Create a seperate function for publishing the Hunspell that can run after the publish function?
// to keep the publish function time down
export default async function publish(): Promise<PublishDictionaryResponse> {
const entries = await getRawEntries();
const errors = checkForErrors(entries);
if (errors.length) {
return { ok: false, errors };
}
// const duplicates = findDuplicates(entries);
// duplicates.forEach((duplicate) => {
// const index = entries.findIndex(e => e.ts === duplicate.ts);
// if (index > -1) entries.splice(index, 1);
// })
const dictionary: T.Dictionary = {
info: {
title,
license,
url,
infoUrl,
release: new Date().getTime(),
numberOfEntries: entries.length,
},
entries,
};
uploadDictionaryToStorage(dictionary).catch(console.error);
uploadSitemap(dictionary).catch(console.error);
// TODO: make this async and run after publish response
doHunspellEtc(dictionary.info, entries).catch(console.error);
return {
ok: true,
info: dictionary.info,
};
}
async function doHunspellEtc(
info: T.DictionaryInfo,
entries: T.DictionaryEntry[]
) {
const wordlistResponse = getWordList(entries);
if (!wordlistResponse.ok) {
throw new Error(JSON.stringify(wordlistResponse.errors));
}
// const hunspell = makeHunspell(wordlistResponse.wordlist);
// await uploadHunspellToStorage(hunspell);
await uploadAllWordsToStoarage(info, wordlistResponse.wordlist);
}
/**
* Gets the entries from the spreadsheet, and also deletes duplicate
* entries that are sometimes annoyingly created by the GoogleSheets API
* when adding entries programmatically
*
* @returns
*
*/
async function getRows() {
const doc = new GoogleSpreadsheet(functions.config().sheet.id);
await doc.useServiceAccountAuth({
client_email: functions.config().serviceacct.email,
private_key: functions.config().serviceacct.key,
});
await doc.loadInfo();
const sheet = doc.sheetsByIndex[0];
const rows = await sheet.getRows();
rows.sort((a, b) => (a.ts > b.ts ? -1 : a.ts < b.ts ? 1 : 0));
return rows;
}
async function getRawEntries(): Promise<T.DictionaryEntry[]> {
const rows = await getRows();
// async function deleteRow(i: number) {
// console.log("WILL NOT DELETE ROW", rows[i].p, rows[i].ts, rows[i].f);
// // await rows[i].delete();
// }
const entries: T.DictionaryEntry[] = [];
// let sheetIndex = 0;
// get the rows in order of ts for easy detection of duplicate entries
const duplicates: Set<number> = new Set();
for (let i = 0; i < rows.length; i++) {
// function sameEntry(a: any, b: any): boolean {
// return a.p === b.p && a.f === b.f && a.e === b.e;
// }
// sheetIndex++;
const row = rows[i];
const nextRow = rows[i + 1] || undefined;
if (row.ts === nextRow?.ts) {
// if (sameEntry(row, nextRow)) {
// // this looks like a duplicate entry made by the sheets api
// // delete it and keep going
// await deleteRow(sheetIndex);
// sheetIndex--;
// continue;
// } else {
duplicates.add(row.ts);
// }
}
const e: T.DictionaryEntry = {
i: 1,
ts: parseInt(row.ts),
p: row.p,
f: row.f,
g: simplifyPhonetics(row.f),
e: row.e,
};
dictionaryEntryNumberFields.forEach(
(field: T.DictionaryEntryNumberField) => {
if (row[field]) e[field] = parseInt(row[field]);
}
);
dictionaryEntryTextFields.forEach((field: T.DictionaryEntryTextField) => {
if (row[field]) e[field] = row[field].trim();
});
dictionaryEntryBooleanFields.forEach(
(field: T.DictionaryEntryBooleanField) => {
if (row[field]) e[field] = true;
}
);
entries.push(standardizeEntry(e));
}
if (duplicates.size) {
throw new Error(
`ts ${Array.from(duplicates).join(
", "
)} is a duplicate ts of a different entry`
);
}
// make alphabetical index
entries.sort((a, b) => a.p.localeCompare(b.p, "ps"));
const entriesLength = entries.length;
// add index
for (let i = 0; i < entriesLength; i++) {
entries[i].i = i;
}
return entries;
}
function checkForErrors(
entries: T.DictionaryEntry[]
): T.DictionaryEntryError[] {
return entries.reduce(
(errors: T.DictionaryEntryError[], entry: T.DictionaryEntry) => {
const response = validateEntry(entry);
if ("errors" in response && response.errors.length) {
return [...errors, response];
}
if ("checkComplement" in response) {
const complement = entries.find((e) => e.ts === entry.l);
if (!complement) {
const error: T.DictionaryEntryError = {
errors: ["complement link not found in dictonary"],
ts: entry.ts,
p: entry.p,
f: entry.f,
e: entry.e,
erroneousFields: ["l"],
};
return [...errors, error];
}
if (
!complement.c?.includes("n.") &&
!complement.c?.includes("adj.") &&
!complement.c?.includes("adv.")
) {
const error: T.DictionaryEntryError = {
errors: ["complement link to invalid complement"],
ts: entry.ts,
p: entry.p,
f: entry.f,
e: entry.e,
erroneousFields: ["l"],
};
return [...errors, error];
}
}
return errors;
},
[]
);
}
// function findDuplicates(entries: T.DictionaryEntry[]): T.DictionaryEntry[] {
// const tsSoFar = new Set();
// const duplicates: T.DictionaryEntry[] = [];
// // tslint:disable-next-line: prefer-for-of
// for (let i = 0; i < entries.length; i++) {
// const ts = entries[i].ts;
// if (tsSoFar.has(ts)) {
// duplicates.push(entries[i]);
// }
// tsSoFar.add(ts);
// }
// return duplicates;
// }
async function upload(content: Buffer | string, filename: string) {
const isBuffer = typeof content !== "string";
// upload to r2 (new destination)
if (isBuffer) {
const putObjectCommand = new PutObjectCommand({
Bucket: functions.config().r2.bucket_name,
Key: `dictionary/${filename}`,
Body: content,
CacheControl: "no-cache",
ContentType: "application/octet-stream",
});
await s3Client.send(putObjectCommand);
} else {
zlib.gzip(content, (err, buffer) => {
if (err) {
console.error(err);
}
const putObjectCommand = new PutObjectCommand({
Bucket: functions.config().r2.bucket_name,
Key: `dictionary/${filename}`,
CacheControl: "no-cache",
Body: buffer,
ContentEncoding: "gzip",
ContentType: filename.endsWith(".json")
? "application/json"
: filename.endsWith(".xml")
? "application/xml"
: "text/plain; charset=UTF-8",
});
s3Client.send(putObjectCommand).catch(console.error);
});
}
}
// async function uploadHunspellToStorage(wordlist: {
// affContent: string,
// dicContent: string,
// }) {
// await Promise.all([
// upload(wordlist.affContent, hunspellAffFileFilename),
// upload(wordlist.dicContent, hunspellDicFileFilename),
// ]);
// }
async function uploadAllWordsToStoarage(
info: T.DictionaryInfo,
words: T.PsString[]
) {
await upload(
JSON.stringify({ info, words } as T.AllWordsWithInflections),
allWordsJsonFilename
);
}
async function uploadSitemap(dictionary: T.Dictionary) {
await upload(makeSitemap(dictionary), "sitemap.xml");
}
async function uploadDictionaryToStorage(dictionary: T.Dictionary) {
await Promise.all([
upload(JSON.stringify(dictionary), `${dictionaryFilename}.json`),
upload(
JSON.stringify(dictionary.info, null, "\t"),
`${dictionaryInfoFilename}.json`
),
]);
}
function makeSitemap(dictionary: T.Dictionary): string {
function tsToDate(ts: number): string {
if (ts < 10000000000) {
// approximate date for old-style timestamps
return "2021-01-01";
}
return getDateString(new Date(ts));
}
function getDateString(d: Date): string {
return d.toISOString().split("T")[0];
}
const pages = [
"",
"about",
"settings",
"account",
"phrase-builder",
"new-entries",
];
const currentDate = getDateString(new Date());
return `<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
${pages
.map(
(page) =>
`
<url>
<loc>https://dictionary.lingdocs.com/${page}</loc>
<lastmod>${currentDate}</lastmod>
</url>`
)
.join("")}
${dictionary.entries
.map(
(entry) =>
`
<url>
<loc>https://dictionary.lingdocs.com/word?id=${entry.ts}</loc>
<lastmod>${tsToDate(entry.ts)}</lastmod>
</url>`
)
.join("")}
</urlset>
`;
}
// function makeHunspell(wordlist: string[]) {
// return {
// dicContent: wordlist.reduce((acc, word) => acc + word + "\n", wordlist.length + "\n"),
// affContent: "SET UTF-8\nCOMPLEXPREFIXES\nIGNORE ۱۲۳۴۵۶۷۸۹۰-=ًٌٍَُِّْ؛:؟.،,،؟\n",
// };
// }

View File

@ -0,0 +1,17 @@
// import * as cdk from 'aws-cdk-lib';
// import { Template } from 'aws-cdk-lib/assertions';
// import * as Functions from '../lib/functions-stack';
// example test. To run these tests, uncomment this file along with the
// example resource in lib/functions-stack.ts
test('SQS Queue Created', () => {
// const app = new cdk.App();
// // WHEN
// const stack = new Functions.FunctionsStack(app, 'MyTestStack');
// // THEN
// const template = Template.fromStack(stack);
// template.hasResourceProperties('AWS::SQS::Queue', {
// VisibilityTimeout: 300
// });
});

View File

@ -1,17 +1,31 @@
{
"compilerOptions": {
"target": "ES2020",
"module": "commonjs",
"noImplicitReturns": true,
"noUnusedLocals": true,
"outDir": "lib",
"sourceMap": true,
"lib": [
"es2020",
"dom"
],
"declaration": true,
"strict": true,
"target": "es2017",
"skipLibCheck": true,
"esModuleInterop": true,
"noImplicitAny": true,
"strictNullChecks": true,
"noImplicitThis": true,
"alwaysStrict": true,
"noUnusedLocals": false,
"noUnusedParameters": false,
"noImplicitReturns": true,
"noFallthroughCasesInSwitch": false,
"inlineSourceMap": true,
"inlineSources": true,
"experimentalDecorators": true,
"strictPropertyInitialization": false,
"typeRoots": [
"./node_modules/@types"
]
},
"compileOnSave": true,
"include": [
"src"
"exclude": [
"node_modules",
"cdk.out"
]
}

View File

@ -1,33 +0,0 @@
# prod
dist/
# dev
.yarn/
!.yarn/releases
.vscode/*
!.vscode/launch.json
!.vscode/*.code-snippets
.idea/workspace.xml
.idea/usage.statistics.xml
.idea/shelf
# deps
node_modules/
.wrangler
# env
.env
.env.production
.dev.vars
# logs
logs/
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
pnpm-debug.log*
lerna-debug.log*
# misc
.DS_Store

View File

@ -1,8 +0,0 @@
```
npm install
npm run dev
```
```
npm run deploy
```

File diff suppressed because it is too large Load Diff

View File

@ -1,14 +0,0 @@
{
"name": "new-functions",
"scripts": {
"dev": "wrangler dev",
"deploy": "wrangler deploy --minify"
},
"dependencies": {
"hono": "^4.6.12"
},
"devDependencies": {
"@cloudflare/workers-types": "^4.20241112.0",
"wrangler": "^3.88.0"
}
}

View File

@ -1,17 +0,0 @@
import { Hono } from "hono";
import { cors } from "hono/cors";
import { authMiddleware } from "./middleware/lingdocs-auth";
const app = new Hono();
app.use(cors());
app.get("/", (c) => {
// c.env.LINGDOCS_COUCHDB
return c.text("Hi from hono updated");
});
app.get("/wa", authMiddleware, async (c) => {
return c.json({ name: c.var.user?.name, admin: c.var.user?.admin });
});
export default app;

View File

@ -1,20 +0,0 @@
import { createMiddleware } from "hono/factory";
import type { LingdocsUser } from "../../../website/src/types/account-types";
export const authMiddleware = createMiddleware<{
Variables: {
user: LingdocsUser | undefined;
};
}>(async (c, next) => {
const cookie = c.req.header("Cookie") || "";
const r = await fetch("https://account.lingdocs.com/api/user", {
headers: { cookie },
});
const res = (await r.json()) as { ok: boolean; user: LingdocsUser };
if (res.ok) {
c.set("user", res.user);
} else {
c.set("user", undefined);
}
await next();
});

View File

@ -1,17 +0,0 @@
{
"compilerOptions": {
"target": "ESNext",
"module": "ESNext",
"moduleResolution": "Bundler",
"strict": true,
"skipLibCheck": true,
"lib": [
"ESNext"
],
"types": [
"@cloudflare/workers-types/2023-07-01"
],
"jsx": "react-jsx",
"jsxImportSource": "hono/jsx"
},
}

View File

@ -1,28 +0,0 @@
name = "new-functions"
main = "src/index.ts"
compatibility_date = "2024-11-26"
# compatibility_flags = [ "nodejs_compat" ]
# [vars]
# MY_VAR = "my-variable"
# [[kv_namespaces]]
# binding = "MY_KV_NAMESPACE"
# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
# [[r2_buckets]]
# binding = "MY_BUCKET"
# bucket_name = "my-bucket"
# [[d1_databases]]
# binding = "DB"
# database_name = "my-database"
# database_id = ""
# [ai]
# binding = "AI"
# [observability]
# enabled = true
# head_sampling_rate = 1